Skip to content

Commit ccd9835

Browse files
rust-bors[bot]rust-bors[bot]
authored
Auto merge of #149808 - Kobzol:opt-dist-reuse-pgo, r=<try>
WIP: Try to reuse PGO profiles in `opt-dist`
2 parents 018d269 + 5a5444a commit ccd9835

File tree

3 files changed

+195
-6
lines changed

3 files changed

+195
-6
lines changed

‎Cargo.lock‎

Lines changed: 144 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,12 @@ version = "0.21.7"
266266
source = "registry+https://github.com/rust-lang/crates.io-index"
267267
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
268268

269+
[[package]]
270+
name = "base64"
271+
version = "0.22.1"
272+
source = "registry+https://github.com/rust-lang/crates.io-index"
273+
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
274+
269275
[[package]]
270276
name = "basic-toml"
271277
version = "0.1.10"
@@ -382,6 +388,12 @@ version = "0.6.9"
382388
source = "registry+https://github.com/rust-lang/crates.io-index"
383389
checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
384390

391+
[[package]]
392+
name = "bytes"
393+
version = "1.11.0"
394+
source = "registry+https://github.com/rust-lang/crates.io-index"
395+
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
396+
385397
[[package]]
386398
name = "camino"
387399
version = "1.2.1"
@@ -1726,6 +1738,22 @@ dependencies = [
17261738
"match_token",
17271739
]
17281740

1741+
[[package]]
1742+
name = "http"
1743+
version = "1.4.0"
1744+
source = "registry+https://github.com/rust-lang/crates.io-index"
1745+
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
1746+
dependencies = [
1747+
"bytes",
1748+
"itoa",
1749+
]
1750+
1751+
[[package]]
1752+
name = "httparse"
1753+
version = "1.10.1"
1754+
source = "registry+https://github.com/rust-lang/crates.io-index"
1755+
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
1756+
17291757
[[package]]
17301758
name = "humansize"
17311759
version = "2.1.3"
@@ -2762,6 +2790,7 @@ dependencies = [
27622790
"tabled",
27632791
"tar",
27642792
"tempfile",
2793+
"ureq",
27652794
"xz2",
27662795
]
27672796

@@ -3311,6 +3340,20 @@ dependencies = [
33113340
"walkdir",
33123341
]
33133342

3343+
[[package]]
3344+
name = "ring"
3345+
version = "0.17.14"
3346+
source = "registry+https://github.com/rust-lang/crates.io-index"
3347+
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
3348+
dependencies = [
3349+
"cc",
3350+
"cfg-if",
3351+
"getrandom 0.2.16",
3352+
"libc",
3353+
"untrusted",
3354+
"windows-sys 0.52.0",
3355+
]
3356+
33143357
[[package]]
33153358
name = "run_make_support"
33163359
version = "0.0.0"
@@ -4865,7 +4908,7 @@ version = "0.0.0"
48654908
dependencies = [
48664909
"arrayvec",
48674910
"askama",
4868-
"base64",
4911+
"base64 0.21.7",
48694912
"expect-test",
48704913
"indexmap",
48714914
"itertools",
@@ -4982,6 +5025,41 @@ dependencies = [
49825025
"windows-sys 0.61.2",
49835026
]
49845027

5028+
[[package]]
5029+
name = "rustls"
5030+
version = "0.23.35"
5031+
source = "registry+https://github.com/rust-lang/crates.io-index"
5032+
checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
5033+
dependencies = [
5034+
"log",
5035+
"once_cell",
5036+
"ring",
5037+
"rustls-pki-types",
5038+
"rustls-webpki",
5039+
"subtle",
5040+
"zeroize",
5041+
]
5042+
5043+
[[package]]
5044+
name = "rustls-pki-types"
5045+
version = "1.13.1"
5046+
source = "registry+https://github.com/rust-lang/crates.io-index"
5047+
checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c"
5048+
dependencies = [
5049+
"zeroize",
5050+
]
5051+
5052+
[[package]]
5053+
name = "rustls-webpki"
5054+
version = "0.103.8"
5055+
source = "registry+https://github.com/rust-lang/crates.io-index"
5056+
checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
5057+
dependencies = [
5058+
"ring",
5059+
"rustls-pki-types",
5060+
"untrusted",
5061+
]
5062+
49855063
[[package]]
49865064
name = "rustversion"
49875065
version = "1.0.22"
@@ -5389,6 +5467,12 @@ dependencies = [
53895467
"syn 1.0.109",
53905468
]
53915469

5470+
[[package]]
5471+
name = "subtle"
5472+
version = "2.6.1"
5473+
source = "registry+https://github.com/rust-lang/crates.io-index"
5474+
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
5475+
53925476
[[package]]
53935477
name = "syn"
53945478
version = "1.0.109"
@@ -6065,6 +6149,41 @@ dependencies = [
60656149
"tidy",
60666150
]
60676151

6152+
[[package]]
6153+
name = "untrusted"
6154+
version = "0.9.0"
6155+
source = "registry+https://github.com/rust-lang/crates.io-index"
6156+
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
6157+
6158+
[[package]]
6159+
name = "ureq"
6160+
version = "3.1.4"
6161+
source = "registry+https://github.com/rust-lang/crates.io-index"
6162+
checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a"
6163+
dependencies = [
6164+
"base64 0.22.1",
6165+
"flate2",
6166+
"log",
6167+
"percent-encoding",
6168+
"rustls",
6169+
"rustls-pki-types",
6170+
"ureq-proto",
6171+
"utf-8",
6172+
"webpki-roots",
6173+
]
6174+
6175+
[[package]]
6176+
name = "ureq-proto"
6177+
version = "0.5.3"
6178+
source = "registry+https://github.com/rust-lang/crates.io-index"
6179+
checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
6180+
dependencies = [
6181+
"base64 0.22.1",
6182+
"http",
6183+
"httparse",
6184+
"log",
6185+
]
6186+
60686187
[[package]]
60696188
name = "url"
60706189
version = "2.5.7"
@@ -6345,6 +6464,15 @@ dependencies = [
63456464
"wasm-bindgen",
63466465
]
63476466

6467+
[[package]]
6468+
name = "webpki-roots"
6469+
version = "1.0.4"
6470+
source = "registry+https://github.com/rust-lang/crates.io-index"
6471+
checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e"
6472+
dependencies = [
6473+
"rustls-pki-types",
6474+
]
6475+
63486476
[[package]]
63496477
name = "winapi"
63506478
version = "0.3.9"
@@ -6526,6 +6654,15 @@ dependencies = [
65266654
"windows-link 0.2.1",
65276655
]
65286656

6657+
[[package]]
6658+
name = "windows-sys"
6659+
version = "0.52.0"
6660+
source = "registry+https://github.com/rust-lang/crates.io-index"
6661+
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
6662+
dependencies = [
6663+
"windows-targets 0.52.6",
6664+
]
6665+
65296666
[[package]]
65306667
name = "windows-sys"
65316668
version = "0.59.0"
@@ -6863,6 +7000,12 @@ dependencies = [
68637000
"synstructure",
68647001
]
68657002

7003+
[[package]]
7004+
name = "zeroize"
7005+
version = "1.8.2"
7006+
source = "registry+https://github.com/rust-lang/crates.io-index"
7007+
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
7008+
68667009
[[package]]
68677010
name = "zerotrie"
68687011
version = "0.2.3"

‎src/tools/opt-dist/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,4 @@ tempfile = "3.5"
2121
derive_builder = "0.20"
2222
clap = { version = "4", features = ["derive"] }
2323
tabled = { version = "0.15", default-features = false, features = ["std"] }
24+
ureq = "3.1.4"

‎src/tools/opt-dist/src/main.rs‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1+
#![allow(unused, warnings)]
2+
13
use anyhow::Context;
4+
use build_helper::ci::CiEnv;
5+
use build_helper::git::{GitConfig, get_closest_upstream_commit};
26
use camino::{Utf8Path, Utf8PathBuf};
37
use clap::Parser;
48
use log::LevelFilter;
9+
use ureq::http::StatusCode;
510
use utils::io;
611

712
use crate::bolt::{bolt_optimize, with_bolt_instrumented};
@@ -10,11 +15,11 @@ use crate::exec::{Bootstrap, cmd};
1015
use crate::tests::run_tests;
1116
use crate::timer::Timer;
1217
use crate::training::{
13-
gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks,
14-
rustc_benchmarks,
18+
BoltProfile, LlvmPGOProfile, RustcPGOProfile, gather_bolt_profiles, gather_llvm_profiles,
19+
gather_rustc_profiles, llvm_benchmarks, rustc_benchmarks,
1520
};
1621
use crate::utils::artifact_size::print_binary_sizes;
17-
use crate::utils::io::{copy_directory, reset_directory};
22+
use crate::utils::io::{copy_directory, find_file_in_dir, reset_directory};
1823
use crate::utils::{
1924
clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group,
2025
write_timer_to_summary,
@@ -464,15 +469,55 @@ fn main() -> anyhow::Result<()> {
464469

465470
let mut timer = Timer::new();
466471

467-
let result = execute_pipeline(&env, &mut timer, build_args);
472+
let parent_sha = get_closest_upstream_commit(
473+
None,
474+
&GitConfig { nightly_branch: "main", git_merge_commit_email: "[email protected]" },
475+
CiEnv::GitHubActions,
476+
)
477+
.unwrap()
478+
.unwrap();
479+
480+
let url = format!(
481+
"https://ci-artifacts.rust-lang.org/rustc-builds/{parent_sha}/reproducible-artifacts-nightly-x86_64-unknown-linux-gnu.tar.xz"
482+
);
483+
let response = ureq::get(url).call()?;
484+
if !response.status().is_success() {
485+
return Err(anyhow::anyhow!("Error while downloading PGO: {}", response.status()));
486+
}
487+
let mut body = response.into_body();
488+
let mut archive = tar::Archive::new(xz::read::XzDecoder::new(body.as_reader()));
489+
490+
let pgo_dir = Utf8Path::new("pgo-profiles");
491+
archive.unpack(pgo_dir)?;
492+
let dir = find_file_in_dir(pgo_dir, "reproducible-artifacts", "")?;
493+
let dir = dir.join("reproducible-artifacts");
494+
let llvm_pgo_profile = LlvmPGOProfile(dir.join("llvm-pgo.profdata"));
495+
let rustc_pgo_profile = RustcPGOProfile(dir.join("rustc-pgo.profdata"));
496+
let bolt_profiles =
497+
[BoltProfile(dir.join("LLVM-bolt.profdata")), BoltProfile(dir.join("rustc-bolt.profdata"))];
498+
499+
let mut dist = Bootstrap::dist(&env, &build_args)
500+
.llvm_pgo_optimize(Some(&llvm_pgo_profile))
501+
.rustc_pgo_optimize(&rustc_pgo_profile);
502+
503+
// for bolt_profile in bolt_profiles {
504+
// dist = dist.with_bolt_profile(Some(bolt_profile));
505+
// }
506+
// dist = dist.with_llvm_bolt_ldflags().with_rustc_bolt_ldflags();
507+
508+
// Final stage: Assemble the dist artifacts
509+
// The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused.
510+
timer.section("Stage 5 (final build)", |stage| dist.run(stage))?;
511+
512+
// let result = execute_pipeline(&env, &mut timer, build_args);
468513
log::info!("Timer results\n{}", timer.format_stats());
469514

470515
if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") {
471516
write_timer_to_summary(&summary_path, &timer)?;
472517
}
473518

474519
print_free_disk_space()?;
475-
result.context("Optimized build pipeline has failed")?;
520+
// result.context("Optimized build pipeline has failed")?;
476521
print_binary_sizes(&env)?;
477522

478523
Ok(())

0 commit comments

Comments
 (0)