From 8e0007f829661e57d008d2e908c95f6e84b04b25 Mon Sep 17 00:00:00 2001
From: bors <bors@rust-lang.org>
Date: Thu, 24 Oct 2019 07:27:00 +0000
Subject: [PATCH] Auto merge of #65474 - Mark-Simulacrum:rustc-dev-split,
r=pietroalbini
Split the rustc target libraries into separate rustc-dev component
This is re-applies a squashed version of #64823 as well as including #65337 to fix bugs noted after merging the first PR.
The second PR is confirmed as fixing windows-gnu, and presumably also fixes other platforms, such as musl (i.e. #65335 should be fixed); `RUSTUP_DIST_SERVER=https://dev-static.rust-lang.org rustup toolchain install nightly-2019-10-16` can be installed to confirm that this is indeed the case.
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index b8071b98f707..2748903f2d47 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -443,6 +443,7 @@ impl<'a> Builder<'a> {
dist::Rustc,
dist::DebuggerScripts,
dist::Std,
+ dist::RustcDev,
dist::Analysis,
dist::Src,
dist::PlainSourceTarball,
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index cadb9a7e441f..df1c72575846 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -55,6 +55,7 @@ impl Step for Std {
cargo,
args(builder.kind),
&libstd_stamp(builder, compiler, target),
+ vec![],
true);
let libdir = builder.sysroot_libdir(compiler, target);
@@ -103,6 +104,7 @@ impl Step for Rustc {
cargo,
args(builder.kind),
&librustc_stamp(builder, compiler, target),
+ vec![],
true);
let libdir = builder.sysroot_libdir(compiler, target);
@@ -155,6 +157,7 @@ impl Step for CodegenBackend {
cargo,
args(builder.kind),
&codegen_backend_stamp(builder, compiler, target, backend),
+ vec![],
true);
}
}
@@ -199,6 +202,7 @@ impl Step for Rustdoc {
cargo,
args(builder.kind),
&rustdoc_stamp(builder, compiler, target),
+ vec![],
true);
let libdir = builder.sysroot_libdir(compiler, target);
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 5074b035789a..da8d43ed49b7 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -69,7 +69,7 @@ impl Step for Std {
return;
}
- builder.ensure(StartupObjects { compiler, target });
+ let mut target_deps = builder.ensure(StartupObjects { compiler, target });
let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
if compiler_to_use != compiler {
@@ -91,7 +91,7 @@ impl Step for Std {
return;
}
- copy_third_party_objects(builder, &compiler, target);
+ target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter());
let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
std_cargo(builder, &compiler, target, &mut cargo);
@@ -102,6 +102,7 @@ impl Step for Std {
cargo,
vec![],
&libstd_stamp(builder, compiler, target),
+ target_deps,
false);
builder.ensure(StdLink {
@@ -113,9 +114,22 @@ impl Step for Std {
}
/// Copies third pary objects needed by various targets.
-fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>) {
+fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>)
+ -> Vec<PathBuf>
+{
let libdir = builder.sysroot_libdir(*compiler, target);
+ let mut target_deps = vec![];
+
+ let mut copy_and_stamp = |sourcedir: &Path, name: &str| {
+ let target = libdir.join(name);
+ builder.copy(
+ &sourcedir.join(name),
+ &target,
+ );
+ target_deps.push(target);
+ };
+
// Copies the crt(1,i,n).o startup objects
//
// Since musl supports fully static linking, we can cross link for it even
@@ -123,19 +137,13 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
// files. As those shipped with glibc won't work, copy the ones provided by
// musl so we have them on linux-gnu hosts.
if target.contains("musl") {
+ let srcdir = builder.musl_root(target).unwrap().join("lib");
for &obj in &["crt1.o", "crti.o", "crtn.o"] {
- builder.copy(
- &builder.musl_root(target).unwrap().join("lib").join(obj),
- &libdir.join(obj),
- );
+ copy_and_stamp(&srcdir, obj);
}
} else if target.ends_with("-wasi") {
- for &obj in &["crt1.o"] {
- builder.copy(
- &builder.wasi_root(target).unwrap().join("lib/wasm32-wasi").join(obj),
- &libdir.join(obj),
- );
- }
+ let srcdir = builder.wasi_root(target).unwrap().join("lib/wasm32-wasi");
+ copy_and_stamp(&srcdir, "crt1.o");
}
// Copies libunwind.a compiled to be linked wit x86_64-fortanix-unknown-sgx.
@@ -145,11 +153,11 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
// which is provided by std for this target.
if target == "x86_64-fortanix-unknown-sgx" {
let src_path_env = "X86_FORTANIX_SGX_LIBS";
- let obj = "libunwind.a";
let src = env::var(src_path_env).expect(&format!("{} not found in env", src_path_env));
- let src = Path::new(&src).join(obj);
- builder.copy(&src, &libdir.join(obj));
+ copy_and_stamp(Path::new(&src), "libunwind.a");
}
+
+ target_deps
}
/// Configure cargo to compile the standard library, adding appropriate env vars
@@ -306,7 +314,7 @@ pub struct StartupObjects {
}
impl Step for StartupObjects {
- type Output = ();
+ type Output = Vec<PathBuf>;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/rtstartup")
@@ -325,13 +333,15 @@ impl Step for StartupObjects {
/// They don't require any library support as they're just plain old object
/// files, so we just use the nightly snapshot compiler to always build them (as
/// no other compilers are guaranteed to be available).
- fn run(self, builder: &Builder<'_>) {
+ fn run(self, builder: &Builder<'_>) -> Vec<PathBuf> {
let for_compiler = self.compiler;
let target = self.target;
if !target.contains("windows-gnu") {
- return
+ return vec![]
}
+ let mut target_deps = vec![];
+
let src_dir = &builder.src.join("src/rtstartup");
let dst_dir = &builder.native_dir(target).join("rtstartup");
let sysroot_dir = &builder.sysroot_libdir(for_compiler, target);
@@ -350,7 +360,9 @@ impl Step for StartupObjects {
.arg(src_file));
}
- builder.copy(dst_file, &sysroot_dir.join(file.to_string() + ".o"));
+ let target = sysroot_dir.join(file.to_string() + ".o");
+ builder.copy(dst_file, &target);
+ target_deps.push(target);
}
for obj in ["crt2.o", "dllcrt2.o"].iter() {
@@ -358,8 +370,12 @@ impl Step for StartupObjects {
builder.cc(target),
target,
obj);
- builder.copy(&src, &sysroot_dir.join(obj));
+ let target = sysroot_dir.join(obj);
+ builder.copy(&src, &target);
+ target_deps.push(target);
}
+
+ target_deps
}
}
@@ -437,6 +453,7 @@ impl Step for Rustc {
cargo,
vec![],
&librustc_stamp(builder, compiler, target),
+ vec![],
false);
builder.ensure(RustcLink {
@@ -585,7 +602,7 @@ impl Step for CodegenBackend {
let tmp_stamp = out_dir.join(".tmp.stamp");
- let files = run_cargo(builder, cargo, vec![], &tmp_stamp, false);
+ let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false);
if builder.config.dry_run {
return;
}
@@ -941,6 +958,7 @@ pub fn run_cargo(builder: &Builder<'_>,
cargo: Cargo,
tail_args: Vec<String>,
stamp: &Path,
+ additional_target_deps: Vec<PathBuf>,
is_check: bool)
-> Vec<PathBuf>
{
@@ -1057,6 +1075,7 @@ pub fn run_cargo(builder: &Builder<'_>,
deps.push((path_to_add.into(), false));
}
+ deps.extend(additional_target_deps.into_iter().map(|d| (d, false)));
deps.sort();
let mut new_contents = Vec::new();
for (dep, proc_macro) in deps.iter() {
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 514ad1144491..93143570b0fe 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -637,6 +637,28 @@ impl Step for DebuggerScripts {
}
}
+fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
+ // The only true set of target libraries came from the build triple, so
+ // let's reduce redundant work by only producing archives from that host.
+ if compiler.host != builder.config.build {
+ builder.info("\tskipping, not a build host");
+ true
+ } else {
+ false
+ }
+}
+
+/// Copy stamped files into an image's `target/lib` directory.
+fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
+ let dst = image.join("lib/rustlib").join(target).join("lib");
+ t!(fs::create_dir_all(&dst));
+ for (path, host) in builder.read_stamp_file(stamp) {
+ if !host || builder.config.build == target {
+ builder.copy(&path, &dst.join(path.file_name().unwrap()));
+ }
+ }
+}
+
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Std {
pub compiler: Compiler,
@@ -667,44 +689,19 @@ impl Step for Std {
let target = self.target;
let name = pkgname(builder, "rust-std");
-
- // The only true set of target libraries came from the build triple, so
- // let's reduce redundant work by only producing archives from that host.
- if compiler.host != builder.config.build {
- builder.info("\tskipping, not a build host");
- return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
+ let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
+ if skip_host_target_lib(builder, compiler) {
+ return archive;
}
- // We want to package up as many target libraries as possible
- // for the `rust-std` package, so if this is a host target we
- // depend on librustc and otherwise we just depend on libtest.
- if builder.hosts.iter().any(|t| t == target) {
- builder.ensure(compile::Rustc { compiler, target });
- } else {
- builder.ensure(compile::Std { compiler, target });
- }
+ builder.ensure(compile::Std { compiler, target });
let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
let _ = fs::remove_dir_all(&image);
- let dst = image.join("lib/rustlib").join(target);
- t!(fs::create_dir_all(&dst));
- let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
- src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
- builder.cp_filtered(&src, &dst, &|path| {
- if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
- if name == builder.config.rust_codegen_backends_dir.as_str() {
- return false
- }
- if name == "bin" {
- return false
- }
- if name.contains("LLVM") {
- return false
- }
- }
- true
- });
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
+ copy_target_libs(builder, &target, &image, &stamp);
let mut cmd = rust_installer(builder);
cmd.arg("generate")
@@ -723,7 +720,73 @@ impl Step for Std {
let _time = timeit(builder);
builder.run(&mut cmd);
builder.remove_dir(&image);
- distdir(builder).join(format!("{}-{}.tar.gz", name, target))
+ archive
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDev {
+ pub compiler: Compiler,
+ pub target: Interned<String>,
+}
+
+impl Step for RustcDev {
+ type Output = PathBuf;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("rustc-dev")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustcDev {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ let name = pkgname(builder, "rustc-dev");
+ let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
+ if skip_host_target_lib(builder, compiler) {
+ return archive;
+ }
+
+ builder.ensure(compile::Rustc { compiler, target });
+
+ let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
+ let _ = fs::remove_dir_all(&image);
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
+ copy_target_libs(builder, &target, &image, &stamp);
+
+ let mut cmd = rust_installer(builder);
+ cmd.arg("generate")
+ .arg("--product-name=Rust")
+ .arg("--rel-manifest-dir=rustlib")
+ .arg("--success-message=Rust-is-ready-to-develop.")
+ .arg("--image-dir").arg(&image)
+ .arg("--work-dir").arg(&tmpdir(builder))
+ .arg("--output-dir").arg(&distdir(builder))
+ .arg(format!("--package-name={}-{}", name, target))
+ .arg(format!("--component-name=rustc-dev-{}", target))
+ .arg("--legacy-manifest-dirs=rustlib,cargo");
+
+ builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
+ compiler.stage, &compiler.host, target));
+ let _time = timeit(builder);
+ builder.run(&mut cmd);
+ builder.remove_dir(&image);
+ archive
}
}
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index a182405f3b2d..d1cf1cbca784 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -1137,6 +1137,7 @@ impl Build {
pub fn copy(&self, src: &Path, dst: &Path) {
if self.config.dry_run { return; }
self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst));
+ if src == dst { return; }
let _ = fs::remove_file(&dst);
let metadata = t!(src.symlink_metadata());
if metadata.file_type().is_symlink() {
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index f41e7dd17ede..c0d2deab2f8b 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -399,6 +399,7 @@ impl Builder {
fn add_packages_to(&mut self, manifest: &mut Manifest) {
let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
package("rustc", HOSTS);
+ package("rustc-dev", HOSTS);
package("cargo", HOSTS);
package("rust-mingw", MINGW);
package("rust-std", TARGETS);
@@ -426,6 +427,13 @@ impl Builder {
"rls-preview", "rust-src", "llvm-tools-preview",
"lldb-preview", "rust-analysis", "miri-preview"
]);
+
+ // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+ // split out of `rust-std`. We'll include it by default as a transition for nightly users.
+ if self.rust_release == "nightly" {
+ self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]);
+ self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+ }
}
fn add_renames_to(&self, manifest: &mut Manifest) {
@@ -481,6 +489,15 @@ impl Builder {
components.push(host_component("rust-mingw"));
}
+ // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+ // split out of `rust-std`. We'll include it by default as a transition for nightly users,
+ // but ship it as an optional component on the beta and stable channels.
+ if self.rust_release == "nightly" {
+ components.push(host_component("rustc-dev"));
+ } else {
+ extensions.push(host_component("rustc-dev"));
+ }
+
// Tools are always present in the manifest,
// but might be marked as unavailable if they weren't built.
extensions.extend(vec![
@@ -498,6 +515,11 @@ impl Builder {
.filter(|&&target| target != host)
.map(|target| Component::from_str("rust-std", target))
);
+ extensions.extend(
+ HOSTS.iter()
+ .filter(|&&target| target != host)
+ .map(|target| Component::from_str("rustc-dev", target))
+ );
extensions.push(Component::from_str("rust-src", "*"));
// If the components/extensions don't actually exist for this
@@ -534,6 +556,14 @@ impl Builder {
dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
}
+ fn extend_profile(&mut self,
+ profile_name: &str,
+ dst: &mut BTreeMap<String, Vec<String>>,
+ pkgs: &[&str]) {
+ dst.get_mut(profile_name).expect("existing profile")
+ .extend(pkgs.iter().map(|s| (*s).to_owned()));
+ }
+
fn package(&mut self,
pkgname: &str,
dst: &mut BTreeMap<String, Package>,