From 8e0007f829661e57d008d2e908c95f6e84b04b25 Mon Sep 17 00:00:00 2001

From: bors <bors@rust-lang.org>

Date: Thu, 24 Oct 2019 07:27:00 +0000

Subject: [PATCH] Auto merge of #65474 - Mark-Simulacrum:rustc-dev-split,

 r=pietroalbini

Split the rustc target libraries into separate rustc-dev component

This is re-applies a squashed version of #64823 as well as including #65337 to fix bugs noted after merging the first PR.

The second PR is confirmed as fixing windows-gnu, and presumably also fixes other platforms, such as musl (i.e. #65335 should be fixed); `RUSTUP_DIST_SERVER=https://dev-static.rust-lang.org rustup toolchain install nightly-2019-10-16` can be installed to confirm that this is indeed the case.

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs

index b8071b98f707..2748903f2d47 100644

--- a/src/bootstrap/builder.rs

+++ b/src/bootstrap/builder.rs

@@ -443,6 +443,7 @@ impl<'a> Builder<'a> {

                 dist::Rustc,

                 dist::DebuggerScripts,

                 dist::Std,

+                dist::RustcDev,

                 dist::Analysis,

                 dist::Src,

                 dist::PlainSourceTarball,

diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs

index cadb9a7e441f..df1c72575846 100644

--- a/src/bootstrap/check.rs

+++ b/src/bootstrap/check.rs

@@ -55,6 +55,7 @@ impl Step for Std {

                   cargo,

                   args(builder.kind),

                   &libstd_stamp(builder, compiler, target),

+                  vec![],

                   true);

         let libdir = builder.sysroot_libdir(compiler, target);

@@ -103,6 +104,7 @@ impl Step for Rustc {

                   cargo,

                   args(builder.kind),

                   &librustc_stamp(builder, compiler, target),

+                  vec![],

                   true);

         let libdir = builder.sysroot_libdir(compiler, target);

@@ -155,6 +157,7 @@ impl Step for CodegenBackend {

                   cargo,

                   args(builder.kind),

                   &codegen_backend_stamp(builder, compiler, target, backend),

+                  vec![],

                   true);

     }

 }

@@ -199,6 +202,7 @@ impl Step for Rustdoc {

                   cargo,

                   args(builder.kind),

                   &rustdoc_stamp(builder, compiler, target),

+                  vec![],

                   true);

         let libdir = builder.sysroot_libdir(compiler, target);

diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs

index 5074b035789a..da8d43ed49b7 100644

--- a/src/bootstrap/compile.rs

+++ b/src/bootstrap/compile.rs

@@ -69,7 +69,7 @@ impl Step for Std {

             return;

         }

-        builder.ensure(StartupObjects { compiler, target });

+        let mut target_deps = builder.ensure(StartupObjects { compiler, target });

         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);

         if compiler_to_use != compiler {

@@ -91,7 +91,7 @@ impl Step for Std {

             return;

         }

-        copy_third_party_objects(builder, &compiler, target);

+        target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter());

         let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");

         std_cargo(builder, &compiler, target, &mut cargo);

@@ -102,6 +102,7 @@ impl Step for Std {

                   cargo,

                   vec![],

                   &libstd_stamp(builder, compiler, target),

+                  target_deps,

                   false);

         builder.ensure(StdLink {

@@ -113,9 +114,22 @@ impl Step for Std {

 }

 /// Copies third pary objects needed by various targets.

-fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>) {

+fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>)

+    -> Vec<PathBuf>

+{

     let libdir = builder.sysroot_libdir(*compiler, target);

+    let mut target_deps = vec![];

+

+    let mut copy_and_stamp = |sourcedir: &Path, name: &str| {

+        let target = libdir.join(name);

+        builder.copy(

+            &sourcedir.join(name),

+            &target,

+        );

+        target_deps.push(target);

+    };

+

     // Copies the crt(1,i,n).o startup objects

     //

     // Since musl supports fully static linking, we can cross link for it even

@@ -123,19 +137,13 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:

     // files. As those shipped with glibc won't work, copy the ones provided by

     // musl so we have them on linux-gnu hosts.

     if target.contains("musl") {

+        let srcdir = builder.musl_root(target).unwrap().join("lib");

         for &obj in &["crt1.o", "crti.o", "crtn.o"] {

-            builder.copy(

-                &builder.musl_root(target).unwrap().join("lib").join(obj),

-                &libdir.join(obj),

-            );

+            copy_and_stamp(&srcdir, obj);

         }

     } else if target.ends_with("-wasi") {

-        for &obj in &["crt1.o"] {

-            builder.copy(

-                &builder.wasi_root(target).unwrap().join("lib/wasm32-wasi").join(obj),

-                &libdir.join(obj),

-            );

-        }

+        let srcdir = builder.wasi_root(target).unwrap().join("lib/wasm32-wasi");

+        copy_and_stamp(&srcdir, "crt1.o");

     }

     // Copies libunwind.a compiled to be linked wit x86_64-fortanix-unknown-sgx.

@@ -145,11 +153,11 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:

     // which is provided by std for this target.

     if target == "x86_64-fortanix-unknown-sgx" {

         let src_path_env = "X86_FORTANIX_SGX_LIBS";

-        let obj = "libunwind.a";

         let src = env::var(src_path_env).expect(&format!("{} not found in env", src_path_env));

-        let src = Path::new(&src).join(obj);

-        builder.copy(&src, &libdir.join(obj));

+        copy_and_stamp(Path::new(&src), "libunwind.a");

     }

+

+    target_deps

 }

 /// Configure cargo to compile the standard library, adding appropriate env vars

@@ -306,7 +314,7 @@ pub struct StartupObjects {

 }

 impl Step for StartupObjects {

-    type Output = ();

+    type Output = Vec<PathBuf>;

     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {

         run.path("src/rtstartup")

@@ -325,13 +333,15 @@ impl Step for StartupObjects {

     /// They don't require any library support as they're just plain old object

     /// files, so we just use the nightly snapshot compiler to always build them (as

     /// no other compilers are guaranteed to be available).

-    fn run(self, builder: &Builder<'_>) {

+    fn run(self, builder: &Builder<'_>) -> Vec<PathBuf> {

         let for_compiler = self.compiler;

         let target = self.target;

         if !target.contains("windows-gnu") {

-            return

+            return vec![]

         }

+        let mut target_deps = vec![];

+

         let src_dir = &builder.src.join("src/rtstartup");

         let dst_dir = &builder.native_dir(target).join("rtstartup");

         let sysroot_dir = &builder.sysroot_libdir(for_compiler, target);

@@ -350,7 +360,9 @@ impl Step for StartupObjects {

                             .arg(src_file));

             }

-            builder.copy(dst_file, &sysroot_dir.join(file.to_string() + ".o"));

+            let target = sysroot_dir.join(file.to_string() + ".o");

+            builder.copy(dst_file, &target);

+            target_deps.push(target);

         }

         for obj in ["crt2.o", "dllcrt2.o"].iter() {

@@ -358,8 +370,12 @@ impl Step for StartupObjects {

                                     builder.cc(target),

                                     target,

                                     obj);

-            builder.copy(&src, &sysroot_dir.join(obj));

+            let target = sysroot_dir.join(obj);

+            builder.copy(&src, &target);

+            target_deps.push(target);

         }

+

+        target_deps

     }

 }

@@ -437,6 +453,7 @@ impl Step for Rustc {

                   cargo,

                   vec![],

                   &librustc_stamp(builder, compiler, target),

+                  vec![],

                   false);

         builder.ensure(RustcLink {

@@ -585,7 +602,7 @@ impl Step for CodegenBackend {

         let tmp_stamp = out_dir.join(".tmp.stamp");

-        let files = run_cargo(builder, cargo, vec![], &tmp_stamp, false);

+        let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false);

         if builder.config.dry_run {

             return;

         }

@@ -941,6 +958,7 @@ pub fn run_cargo(builder: &Builder<'_>,

                  cargo: Cargo,

                  tail_args: Vec<String>,

                  stamp: &Path,

+                 additional_target_deps: Vec<PathBuf>,

                  is_check: bool)

     -> Vec<PathBuf>

 {

@@ -1057,6 +1075,7 @@ pub fn run_cargo(builder: &Builder<'_>,

         deps.push((path_to_add.into(), false));

     }

+    deps.extend(additional_target_deps.into_iter().map(|d| (d, false)));

     deps.sort();

     let mut new_contents = Vec::new();

     for (dep, proc_macro) in deps.iter() {

diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs

index 514ad1144491..93143570b0fe 100644

--- a/src/bootstrap/dist.rs

+++ b/src/bootstrap/dist.rs

@@ -637,6 +637,28 @@ impl Step for DebuggerScripts {

     }

 }

+fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {

+    // The only true set of target libraries came from the build triple, so

+    // let's reduce redundant work by only producing archives from that host.

+    if compiler.host != builder.config.build {

+        builder.info("\tskipping, not a build host");

+        true

+    } else {

+        false

+    }

+}

+

+/// Copy stamped files into an image's `target/lib` directory.

+fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {

+    let dst = image.join("lib/rustlib").join(target).join("lib");

+    t!(fs::create_dir_all(&dst);;

+    for (path, host) in builder.read_stamp_file(stamp) {

+        if !host || builder.config.build == target {

+            builder.copy(&path, &dst.join(path.file_name().unwrap()));

+        }

+    }

+}

+

 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]

 pub struct Std {

     pub compiler: Compiler,

@@ -667,44 +689,19 @@ impl Step for Std {

         let target = self.target;

         let name = pkgname(builder, "rust-std");

-

-        // The only true set of target libraries came from the build triple, so

-        // let's reduce redundant work by only producing archives from that host.

-        if compiler.host != builder.config.build {

-            builder.info("\tskipping, not a build host");

-            return distdir(builder).join(format!("{}-{}.tar.gz", name, target));

+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));

+        if skip_host_target_lib(builder, compiler) {

+            return archive;

         }

-        // We want to package up as many target libraries as possible

-        // for the `rust-std` package, so if this is a host target we

-        // depend on librustc and otherwise we just depend on libtest.

-        if builder.hosts.iter().any(|t| t == target) {

-            builder.ensure(compile::Rustc { compiler, target });

-        } else {

-            builder.ensure(compile::Std { compiler, target });

-        }

+        builder.ensure(compile::Std { compiler, target });

         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));

         let _ = fs::remove_dir_all(&image);

-        let dst = image.join("lib/rustlib").join(target);

-        t!(fs::create_dir_all(&dst);;

-        let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();

-        src.pop(); // Remove the trailing /lib folder from the sysroot_libdir

-        builder.cp_filtered(&src, &dst, &|path| {

-            if let Some(name) = path.file_name().and_then(|s| s.to_str()) {

-                if name == builder.config.rust_codegen_backends_dir.as_str() {

-                    return false

-                }

-                if name == "bin" {

-                    return false

-                }

-                if name.contains("LLVM") {

-                    return false

-                }

-            }

-            true

-        });

+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);

+        let stamp = compile::libstd_stamp(builder, compiler_to_use, target);

+        copy_target_libs(builder, &target, &image, &stamp);

         let mut cmd = rust_installer(builder);

         cmd.arg("generate")

@@ -723,7 +720,73 @@ impl Step for Std {

         let _time = timeit(builder);

         builder.run(&mut cmd);

         builder.remove_dir(&image);

-        distdir(builder).join(format!("{}-{}.tar.gz", name, target))

+        archive

+    }

+}

+

+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]

+pub struct RustcDev {

+    pub compiler: Compiler,

+    pub target: Interned<String>,

+}

+

+impl Step for RustcDev {

+    type Output = PathBuf;

+    const DEFAULT: bool = true;

+    const ONLY_HOSTS: bool = true;

+

+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {

+        run.path("rustc-dev")

+    }

+

+    fn make_run(run: RunConfig<'_>) {

+        run.builder.ensure(RustcDev {

+            compiler: run.builder.compiler_for(

+                run.builder.top_stage,

+                run.builder.config.build,

+                run.target,

+            ),

+            target: run.target,

+        });

+    }

+

+    fn run(self, builder: &Builder<'_>) -> PathBuf {

+        let compiler = self.compiler;

+        let target = self.target;

+

+        let name = pkgname(builder, "rustc-dev");

+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));

+        if skip_host_target_lib(builder, compiler) {

+            return archive;

+        }

+

+        builder.ensure(compile::Rustc { compiler, target });

+

+        let image = tmpdir(builder).join(format!("{}-{}-image", name, target));

+        let _ = fs::remove_dir_all(&image);

+

+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);

+        let stamp = compile::librustc_stamp(builder, compiler_to_use, target);

+        copy_target_libs(builder, &target, &image, &stamp);

+

+        let mut cmd = rust_installer(builder);

+        cmd.arg("generate")

+           .arg("--product-name=Rust")

+           .arg("--rel-manifest-dir=rustlib")

+           .arg("--success-message=Rust-is-ready-to-develop.")

+           .arg("--image-dir").arg(&image)

+           .arg("--work-dir").arg(&tmpdir(builder))

+           .arg("--output-dir").arg(&distdir(builder))

+           .arg(format!("--package-name={}-{}", name, target))

+           .arg(format!("--component-name=rustc-dev-{}", target))

+           .arg("--legacy-manifest-dirs=rustlib,cargo");

+

+        builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",

+            compiler.stage, &compiler.host, target));

+        let _time = timeit(builder);

+        builder.run(&mut cmd);

+        builder.remove_dir(&image);

+        archive

     }

 }

diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs

index a182405f3b2d..d1cf1cbca784 100644

--- a/src/bootstrap/lib.rs

+++ b/src/bootstrap/lib.rs

@@ -1137,6 +1137,7 @@ impl Build {

     pub fn copy(&self, src: &Path, dst: &Path) {

         if self.config.dry_run { return; }

         self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst));

+        if src == dst { return; }

         let _ = fs::remove_file(&dst);

         let metadata = t!(src.symlink_metadata());

         if metadata.file_type().is_symlink() {

diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs

index f41e7dd17ede..c0d2deab2f8b 100644

--- a/src/tools/build-manifest/src/main.rs

+++ b/src/tools/build-manifest/src/main.rs

@@ -399,6 +399,7 @@ impl Builder {

     fn add_packages_to(&mut self, manifest: &mut Manifest) {

         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);

         package("rustc", HOSTS);

+        package("rustc-dev", HOSTS);

         package("cargo", HOSTS);

         package("rust-mingw", MINGW);

         package("rust-std", TARGETS);

@@ -426,6 +427,13 @@ impl Builder {

             "rls-preview", "rust-src", "llvm-tools-preview",

             "lldb-preview", "rust-analysis", "miri-preview"

         ]);

+

+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently

+        // split out of `rust-std`. We'll include it by default as a transition for nightly users.

+        if self.rust_release == "nightly" {

+            self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]);

+            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);

+        }

     }

     fn add_renames_to(&self, manifest: &mut Manifest) {

@@ -481,6 +489,15 @@ impl Builder {

             components.push(host_component("rust-mingw"));

         }

+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently

+        // split out of `rust-std`. We'll include it by default as a transition for nightly users,

+        // but ship it as an optional component on the beta and stable channels.

+        if self.rust_release == "nightly" {

+            components.push(host_component("rustc-dev"));

+        } else {

+            extensions.push(host_component("rustc-dev"));

+        }

+

         // Tools are always present in the manifest,

         // but might be marked as unavailable if they weren't built.

         extensions.extend(vec![

@@ -498,6 +515,11 @@ impl Builder {

                 .filter(|&&target| target != host)

                 .map(|target| Component::from_str("rust-std", target))

         );

+        extensions.extend(

+            HOSTS.iter()

+                .filter(|&&target| target != host)

+                .map(|target| Component::from_str("rustc-dev", target))

+        );

         extensions.push(Component::from_str("rust-src", "*"));

         // If the components/extensions don't actually exist for this

@@ -534,6 +556,14 @@ impl Builder {

         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());

     }

+    fn extend_profile(&mut self,

+               profile_name: &str,

+               dst: &mut BTreeMap<String, Vec<String>>,

+               pkgs: &[&str]) {

+        dst.get_mut(profile_name).expect("existing profile")

+            .extend(pkgs.iter().map(|s| (*s).to_owned()));

+    }

+

     fn package(&mut self,

                pkgname: &str,

                dst: &mut BTreeMap<String, Package>,