Auto merge of #63470 - Mark-Simulacrum:rustc-depdep, r=alexcrichton

Utilize -Zbinary-dep-depinfo in rustbuild

We no longer utilize stamp-file mtimes at all inside rustbuild, and a future PR may be able to entirely eliminate them by eagerly copying to the appropriate sysroot. The only mtime-based dependency tracking left is for documentation because we lie to Cargo about the rustdoc binary, so Cargo does not track changes to the real binary, and codegen-backends because binary-dep-depinfo does not emit that information into the depfiles.

Both of these are fixable in the longer term but this existing patch gives us the following benefits:
 * We no longer delete Cargo target directories manually within a stage. Cross-stage, changes to codegen backends will still clear out target directories. This means that incremental state persists across individual steps (e.g., rebuilding libstd does not clear out librustc incremental state). Fixes #54712.
 * Dependency tracking across steps within a given stage is now fully precise. We will not clear out all codegen backend dependencies due to changes in librustc_driver, for example, only deleting the final librustc_codegen_llvm crate. Fixes #54008, fixes #50481.
 * We properly track codegen backends as a dependency (equivalent to rustc) across changes. Fixes #53284, and fixes #52719.
 * Cross-stage dependency tracking of crates is also much more accurate and reliable. Most likely fixes #49979 (but no reproduction steps in that issue). Fixes #59105.

cc #63012

Author: bors

src/bootstrap/builder.rs

@@ -754,76 +754,20 @@ impl<'a> Builder<'a> {
         let mut cargo = Command::new(&self.initial_cargo);
         let out_dir = self.stage_out(compiler, mode);
 
-        // command specific path, we call clear_if_dirty with this
-        let mut my_out = match cmd {
-            "build" => self.cargo_out(compiler, mode, target),
-
-            // This is the intended out directory for crate documentation.
-            "doc" | "rustdoc" =>  self.crate_doc_out(target),
-
-            _ => self.stage_out(compiler, mode),
-        };
-
-        // This is for the original compiler, but if we're forced to use stage 1, then
-        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
-        // we copy the libs forward.
-        let cmp = self.compiler_for(compiler.stage, compiler.host, target);
-
-        let libstd_stamp = match cmd {
-            "check" | "clippy" | "fix" => check::libstd_stamp(self, cmp, target),
-            _ => compile::libstd_stamp(self, cmp, target),
-        };
-
-        let libtest_stamp = match cmd {
-            "check" | "clippy" | "fix" => check::libtest_stamp(self, cmp, target),
-            _ => compile::libtest_stamp(self, cmp, target),
-        };
-
-        let librustc_stamp = match cmd {
-            "check" | "clippy" | "fix" => check::librustc_stamp(self, cmp, target),
-            _ => compile::librustc_stamp(self, cmp, target),
-        };
+        // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
+        // so we need to explicitly clear out if they've been updated.
+        for backend in self.codegen_backends(compiler) {
+            self.clear_if_dirty(&out_dir, &backend);
+        }
 
         if cmd == "doc" || cmd == "rustdoc" {
-            if mode == Mode::Rustc || mode == Mode::ToolRustc || mode == Mode::Codegen {
+            let my_out = match mode {
                 // This is the intended out directory for compiler documentation.
-                my_out = self.compiler_doc_out(target);
-            }
+                Mode::Rustc | Mode::ToolRustc | Mode::Codegen => self.compiler_doc_out(target),
+                _ => self.crate_doc_out(target),
+            };
             let rustdoc = self.rustdoc(compiler);
             self.clear_if_dirty(&my_out, &rustdoc);
-        } else if cmd != "test" {
-            match mode {
-                Mode::Std => {
-                    self.clear_if_dirty(&my_out, &self.rustc(compiler));
-                    for backend in self.codegen_backends(compiler) {
-                        self.clear_if_dirty(&my_out, &backend);
-                    }
-                },
-                Mode::Test => {
-                    self.clear_if_dirty(&my_out, &libstd_stamp);
-                },
-                Mode::Rustc => {
-                    self.clear_if_dirty(&my_out, &self.rustc(compiler));
-                    self.clear_if_dirty(&my_out, &libstd_stamp);
-                    self.clear_if_dirty(&my_out, &libtest_stamp);
-                },
-                Mode::Codegen => {
-                    self.clear_if_dirty(&my_out, &librustc_stamp);
-                },
-                Mode::ToolBootstrap => { },
-                Mode::ToolStd => {
-                    self.clear_if_dirty(&my_out, &libstd_stamp);
-                },
-                Mode::ToolTest => {
-                    self.clear_if_dirty(&my_out, &libstd_stamp);
-                    self.clear_if_dirty(&my_out, &libtest_stamp);
-                },
-                Mode::ToolRustc => {
-                    self.clear_if_dirty(&my_out, &libstd_stamp);
-                    self.clear_if_dirty(&my_out, &libtest_stamp);
-                    self.clear_if_dirty(&my_out, &librustc_stamp);
-                },
-            }
         }
 
         cargo
@@ -861,6 +805,19 @@ impl<'a> Builder<'a> {
             },
         }
 
+        // This tells Cargo (and in turn, rustc) to output more complete
+        // dependency information.  Most importantly for rustbuild, this
+        // includes sysroot artifacts, like libstd, which means that we don't
+        // need to track those in rustbuild (an error prone process!). This
+        // feature is currently unstable as there may be some bugs and such, but
+        // it represents a big improvement in rustbuild's reliability on
+        // rebuilds, so we're using it here.
+        //
+        // For some additional context, see #63470 (the PR originally adding
+        // this), as well as #63012 which is the tracking issue for this
+        // feature on the rustc side.
+        cargo.arg("-Zbinary-dep-depinfo");
+
         cargo.arg("-j").arg(self.jobs().to_string());
         // Remove make-related flags to ensure Cargo can correctly set things up
         cargo.env_remove("MAKEFLAGS");

src/bootstrap/check.rs

@@ -245,7 +245,6 @@ impl Step for Rustdoc {
         let libdir = builder.sysroot_libdir(compiler, target);
         let hostdir = builder.sysroot_libdir(compiler, compiler.host);
         add_to_sysroot(&builder, &libdir, &hostdir, &rustdoc_stamp(builder, compiler, target));
-        builder.cargo(compiler, Mode::ToolRustc, target, "clean");
     }
 }
 

src/bootstrap/compile.rs

@@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio, exit};
 use std::str;
 
-use build_helper::{output, mtime, t, up_to_date};
+use build_helper::{output, t, up_to_date};
 use filetime::FileTime;
 use serde::Deserialize;
 use serde_json;
@@ -274,8 +274,6 @@ impl Step for StdLink {
             // for reason why the sanitizers are not built in stage0.
             copy_apple_sanitizer_dylibs(builder, &builder.native_dir(target), "osx", &libdir);
         }
-
-        builder.cargo(target_compiler, Mode::ToolStd, target, "clean");
     }
 }
 
@@ -480,8 +478,6 @@ impl Step for TestLink {
             &builder.sysroot_libdir(target_compiler, compiler.host),
             &libtest_stamp(builder, compiler, target)
         );
-
-        builder.cargo(target_compiler, Mode::ToolTest, target, "clean");
     }
 }
 
@@ -639,7 +635,6 @@ impl Step for RustcLink {
             &builder.sysroot_libdir(target_compiler, compiler.host),
             &librustc_stamp(builder, compiler, target)
         );
-        builder.cargo(target_compiler, Mode::ToolRustc, target, "clean");
     }
 }
 
@@ -1202,41 +1197,13 @@ pub fn run_cargo(builder: &Builder<'_>,
         deps.push((path_to_add.into(), false));
     }
 
-    // Now we want to update the contents of the stamp file, if necessary. First
-    // we read off the previous contents along with its mtime. If our new
-    // contents (the list of files to copy) is different or if any dep's mtime
-    // is newer then we rewrite the stamp file.
     deps.sort();
-    let stamp_contents = fs::read(stamp);
-    let stamp_mtime = mtime(&stamp);
     let mut new_contents = Vec::new();
-    let mut max = None;
-    let mut max_path = None;
     for (dep, proc_macro) in deps.iter() {
-        let mtime = mtime(dep);
-        if Some(mtime) > max {
-            max = Some(mtime);
-            max_path = Some(dep.clone());
-        }
         new_contents.extend(if *proc_macro { b"h" } else { b"t" });
         new_contents.extend(dep.to_str().unwrap().as_bytes());
         new_contents.extend(b"\0");
     }
-    let max = max.unwrap();
-    let max_path = max_path.unwrap();
-    let contents_equal = stamp_contents
-        .map(|contents| contents == new_contents)
-        .unwrap_or_default();
-    if contents_equal && max <= stamp_mtime {
-        builder.verbose(&format!("not updating {:?}; contents equal and {:?} <= {:?}",
-                stamp, max, stamp_mtime));
-        return deps.into_iter().map(|(d, _)| d).collect()
-    }
-    if max > stamp_mtime {
-        builder.verbose(&format!("updating {:?} as {:?} changed", stamp, max_path));
-    } else {
-        builder.verbose(&format!("updating {:?} as deps changed", stamp));
-    }
     t!(fs::write(&stamp, &new_contents));
     deps.into_iter().map(|(d, _)| d).collect()
 }