From a6db05986e4326ab428cc2ce08cf3499f93617f3 Mon Sep 17 00:00:00 2001 From: RussellBentley Date: Thu, 20 May 2021 20:44:13 -0400 Subject: [PATCH 001/138] Update Build::cpp documentation, remove documentation link to non-public function --- src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0f330ebd9..4bfe9de8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -592,6 +592,12 @@ impl Build { /// /// The other `cpp_*` options will only become active if this is set to /// `true`. + /// + /// The name of the C++ standard library to link is decided by: + /// 1. If [cpp_link_stdlib](Build::cpp_link_stdlib) is set, use its value. + /// 2. Else if the `CXXSTDLIB` environment variable is set, use its value. + /// 3. Else the default is `libc++` for OS X and BSDs, `libc++_shared` for Android, + /// `None` for MSVC and `libstdc++` for anything else. pub fn cpp(&mut self, cpp: bool) -> &mut Build { self.cpp = cpp; self @@ -684,8 +690,6 @@ impl Build { /// Set the standard library to link against when compiling with C++ /// support. /// - /// See [`get_cpp_link_stdlib`](cc::Build::get_cpp_link_stdlib) documentation - /// for the default value. /// If the `CXXSTDLIB` environment variable is set, its value will /// override the default value, but not the value explicitly set by calling /// this function. From dddc9e580a599b603f64c0db411dafc549addfc2 Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Thu, 17 Jun 2021 15:18:45 +0100 Subject: [PATCH 002/138] [watchos] Add support for watchOS --- src/lib.rs | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b62a1da04..cedeec759 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1416,7 +1416,7 @@ impl Build { cmd.push_opt_unless_duplicate("-DANDROID".into()); } - if !target.contains("apple-ios") { + if !target.contains("apple-ios") && !target.contains("apple-watchos") { cmd.push_cc_arg("-ffunction-sections".into()); cmd.push_cc_arg("-fdata-sections".into()); } @@ -1484,6 +1484,19 @@ impl Build { .into(), ); } + } else if target.contains("watchos-sim") { + if let Some(arch) = + map_darwin_target_from_rust_to_compiler_architecture(target) + { + let deployment_target = env::var("WATCHOS_DEPLOYMENT_TARGET") + .unwrap_or_else(|_| "5.0".into()); + cmd.args.push( + format!( + "--target={}-apple-watchos{}-simulator", + arch, deployment_target + ).into(), + ); + } } else if target.starts_with("riscv64gc-") { cmd.args.push( format!("--target={}", target.replace("riscv64gc", "riscv64")).into(), @@ -1720,6 +1733,10 @@ impl Build { self.ios_flags(cmd)?; } + if target.contains("apple-watchos") { + self.watchos_flags(cmd)?; + } + if self.static_flag.unwrap_or(false) { cmd.args.push("-static".into()); } @@ -1984,6 +2001,74 @@ impl Build { Ok(()) } + fn watchos_flags(&self, cmd: &mut Tool) -> Result<(), Error> { + enum ArchSpec { + Device(&'static str), + Simulator(&'static str), + } + + let target = self.get_target()?; + let arch = target.split('-').nth(0).ok_or_else(|| { + Error::new( + ErrorKind::ArchitectureInvalid, + "Unknown architecture for watchOS target.", + ) + })?; + + let arch = match arch { + "armv7k" => ArchSpec::Device("armv7k"), + "arm64_32" => ArchSpec::Device("arm64_32"), + "i386" | "i686" => ArchSpec::Simulator("-m32"), + "x86_64" => ArchSpec::Simulator("-m64"), + _ => { + return Err(Error::new( + ErrorKind::ArchitectureInvalid, + "Unknown architecture for watchOS target.", + )); + } + + }; + + let min_version = + std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into()); + + + let sdk = match arch { + ArchSpec::Device(arch) => { + cmd.args.push("-arch".into()); + cmd.args.push(arch.into()); + cmd.args + .push(format!("-mwatchos-version-min={}", min_version).into()); + "watchos" + } + ArchSpec::Simulator(arch) => { + cmd.args.push(arch.into()); + cmd.args + .push(format!("-mwatch-simulator-version-min={}", min_version).into()); + "watchsimulator" + } + }; + + self.print(&format!("Detecting watchOS SDK path for {}", sdk)); + let sdk_path = self.apple_sdk_root(sdk)?; + cmd.args.push("-isysroot".into()); + cmd.args.push(sdk_path); + cmd.args.push("-fembed-bitcode".into()); + /* + * TODO we probably ultimately want the -fembed-bitcode-marker flag + * but can't have it now because of an issue in LLVM: + * https://github.com/alexcrichton/cc-rs/issues/301 + * https://github.com/rust-lang/rust/pull/48896#comment-372192660 + */ + /* + if self.get_opt_level()? == "0" { + cmd.args.push("-fembed-bitcode-marker".into()); + } + */ + + Ok(()) + } + fn cmd>(&self, prog: P) -> Command { let mut cmd = Command::new(prog); for &(ref a, ref b) in self.env.iter() { @@ -2067,6 +2152,8 @@ impl Build { } } else if target.contains("apple-ios") { clang.to_string() + } else if target.contains("apple-watchos") { + clang.to_string() } else if target.contains("android") { autodetect_android_compiler(&target, &host, gnu, clang) } else if target.contains("cloudabi") { @@ -2630,7 +2717,7 @@ impl Build { Err(_) => { return Err(Error::new( ErrorKind::IOError, - "Unable to determine iOS SDK path.", + "Unable to determine Apple SDK path.", )); } }; From 2a22393362530b9591bbfc55a726bb8403e2a828 Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Thu, 17 Jun 2021 17:33:30 +0100 Subject: [PATCH 003/138] [watch_os] lifetime fix --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cedeec759..8ace735da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -165,7 +165,7 @@ impl From for Error { } impl Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}: {}", self.kind, self.message) } } @@ -2044,7 +2044,7 @@ impl Build { ArchSpec::Simulator(arch) => { cmd.args.push(arch.into()); cmd.args - .push(format!("-mwatch-simulator-version-min={}", min_version).into()); + .push(format!("-mwatchsimulator-version-min={}", min_version).into()); "watchsimulator" } }; From 19525e20db749a6e51f56869d1bfda4743da9942 Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Thu, 17 Jun 2021 17:33:30 +0100 Subject: [PATCH 004/138] [watch_os] lifetime fix --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fee2c0f49..4112c7411 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,7 +168,7 @@ impl From for Error { } impl Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}: {}", self.kind, self.message) } } @@ -2207,7 +2207,7 @@ impl Build { ArchSpec::Simulator(arch) => { cmd.args.push(arch.into()); cmd.args - .push(format!("-mwatch-simulator-version-min={}", min_version).into()); + .push(format!("-mwatchsimulator-version-min={}", min_version).into()); "watchsimulator" } }; From 73f8fe4b5c0024d855398e4fa4ae68b5b82221f2 Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Tue, 8 Mar 2022 11:50:04 +0000 Subject: [PATCH 005/138] [watch_os] fix formatting --- src/lib.rs | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4112c7411..f76053209 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1592,7 +1592,7 @@ impl Build { } } else if target.contains("watchos-sim") { if let Some(arch) = - map_darwin_target_from_rust_to_compiler_architecture(target) + map_darwin_target_from_rust_to_compiler_architecture(target) { let deployment_target = env::var("WATCHOS_DEPLOYMENT_TARGET") .unwrap_or_else(|_| "5.0".into()); @@ -1600,7 +1600,8 @@ impl Build { format!( "--target={}-apple-watchos{}-simulator", arch, deployment_target - ).into(), + ) + .into(), ); } } else if target.starts_with("riscv64gc-") { @@ -2179,23 +2180,21 @@ impl Build { })?; let arch = match arch { - "armv7k" => ArchSpec::Device("armv7k"), - "arm64_32" => ArchSpec::Device("arm64_32"), - "i386" | "i686" => ArchSpec::Simulator("-m32"), - "x86_64" => ArchSpec::Simulator("-m64"), - _ => { - return Err(Error::new( - ErrorKind::ArchitectureInvalid, - "Unknown architecture for watchOS target.", - )); - } - + "armv7k" => ArchSpec::Device("armv7k"), + "arm64_32" => ArchSpec::Device("arm64_32"), + "i386" | "i686" => ArchSpec::Simulator("-m32"), + "x86_64" => ArchSpec::Simulator("-m64"), + _ => { + return Err(Error::new( + ErrorKind::ArchitectureInvalid, + "Unknown architecture for watchOS target.", + )); + } }; let min_version = std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into()); - let sdk = match arch { ArchSpec::Device(arch) => { cmd.args.push("-arch".into()); From d5a711e72ecc71cc932e9e9a27a004f7a72a3ce3 Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Tue, 8 Mar 2022 17:56:33 +0000 Subject: [PATCH 006/138] [watchos] Make a single flags function for ios and watchos --- src/lib.rs | 118 +++++++++++++++++------------------------------------ 1 file changed, 38 insertions(+), 80 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f76053209..d19c4a7bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,7 +59,7 @@ use std::collections::HashMap; use std::env; use std::ffi::{OsStr, OsString}; -use std::fmt::{self, Display}; +use std::fmt::{self, Display, Formatter}; use std::fs; use std::io::{self, BufRead, BufReader, Read, Write}; use std::path::{Component, Path, PathBuf}; @@ -1864,11 +1864,11 @@ impl Build { } if target.contains("apple-ios") { - self.ios_flags(cmd)?; + self.ios_watchos_flags(cmd)?; } if target.contains("apple-watchos") { - self.watchos_flags(cmd)?; + self.ios_watchos_flags(cmd)?; } if self.static_flag.unwrap_or(false) { @@ -2061,18 +2061,37 @@ impl Build { Ok(()) } - fn ios_flags(&self, cmd: &mut Tool) -> Result<(), Error> { + fn ios_watchos_flags(&self, cmd: &mut Tool) -> Result<(), Error> { enum ArchSpec { Device(&'static str), Simulator(&'static str), Catalyst(&'static str), } + enum Os { + Ios, + WatchOs, + } + impl Display for Os { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Os::Ios => f.write_str("iOS"), + Os::WatchOs => f.write_str("WatchOS"), + } + } + } + let target = self.get_target()?; + let os = if target.contains("-watchos") { + Os::WatchOs + } else { + Os::Ios + }; + let arch = target.split('-').nth(0).ok_or_else(|| { Error::new( ErrorKind::ArchitectureInvalid, - "Unknown architecture for iOS target.", + format!("Unknown architecture for {} target.", os).as_str(), ) })?; @@ -2101,6 +2120,7 @@ impl Build { } else if is_sim { match arch { "arm64" | "aarch64" => ArchSpec::Simulator("-arch arm64"), + "x86_64" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, @@ -2111,108 +2131,46 @@ impl Build { } else { match arch { "arm" | "armv7" | "thumbv7" => ArchSpec::Device("armv7"), + "armv7k" => ArchSpec::Device("armv7k"), "armv7s" | "thumbv7s" => ArchSpec::Device("armv7s"), "arm64e" => ArchSpec::Device("arm64e"), "arm64" | "aarch64" => ArchSpec::Device("arm64"), + "arm64_32" => ArchSpec::Device("arm64_32"), "i386" | "i686" => ArchSpec::Simulator("-m32"), "x86_64" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, - "Unknown architecture for iOS target.", + format!("Unknown architecture for {} target.", os).as_str(), )); } } }; - let min_version = - std::env::var("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "7.0".into()); - - let sdk = match arch { - ArchSpec::Device(arch) => { - cmd.args.push("-arch".into()); - cmd.args.push(arch.into()); - cmd.args - .push(format!("-miphoneos-version-min={}", min_version).into()); - "iphoneos" - } - ArchSpec::Simulator(arch) => { - cmd.args.push(arch.into()); - cmd.args - .push(format!("-mios-simulator-version-min={}", min_version).into()); - "iphonesimulator" - } - ArchSpec::Catalyst(_) => "macosx", - }; - - self.print(&format!("Detecting iOS SDK path for {}", sdk)); - let sdk_path = self.apple_sdk_root(sdk)?; - cmd.args.push("-isysroot".into()); - cmd.args.push(sdk_path); - cmd.args.push("-fembed-bitcode".into()); - /* - * TODO we probably ultimately want the -fembed-bitcode-marker flag - * but can't have it now because of an issue in LLVM: - * https://github.com/alexcrichton/cc-rs/issues/301 - * https://github.com/rust-lang/rust/pull/48896#comment-372192660 - */ - /* - if self.get_opt_level()? == "0" { - cmd.args.push("-fembed-bitcode-marker".into()); - } - */ - - Ok(()) - } - - fn watchos_flags(&self, cmd: &mut Tool) -> Result<(), Error> { - enum ArchSpec { - Device(&'static str), - Simulator(&'static str), - } - - let target = self.get_target()?; - let arch = target.split('-').nth(0).ok_or_else(|| { - Error::new( - ErrorKind::ArchitectureInvalid, - "Unknown architecture for watchOS target.", - ) - })?; - - let arch = match arch { - "armv7k" => ArchSpec::Device("armv7k"), - "arm64_32" => ArchSpec::Device("arm64_32"), - "i386" | "i686" => ArchSpec::Simulator("-m32"), - "x86_64" => ArchSpec::Simulator("-m64"), - _ => { - return Err(Error::new( - ErrorKind::ArchitectureInvalid, - "Unknown architecture for watchOS target.", - )); - } + let (sdk_prefix, sim_prefix, min_version) = match os { + Os::Ios => ("iphone", "ios-", std::env::var("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "7.0".into())), + Os::WatchOs => ("watch", "watch", std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into())), }; - let min_version = - std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into()); - let sdk = match arch { ArchSpec::Device(arch) => { cmd.args.push("-arch".into()); cmd.args.push(arch.into()); cmd.args - .push(format!("-mwatchos-version-min={}", min_version).into()); - "watchos" + .push(format!("-m{}os-version-min={}", sdk_prefix, min_version).into()); + format!("{}os", sdk_prefix) } ArchSpec::Simulator(arch) => { cmd.args.push(arch.into()); cmd.args - .push(format!("-mwatchsimulator-version-min={}", min_version).into()); - "watchsimulator" + .push(format!("-m{}simulator-version-min={}", sim_prefix, min_version).into()); + format!("{}simulator", sdk_prefix) } + ArchSpec::Catalyst(_) => "macosx".to_owned(), }; - self.print(&format!("Detecting watchOS SDK path for {}", sdk)); - let sdk_path = self.apple_sdk_root(sdk)?; + self.print(&format!("Detecting {} SDK path for {}", os, sdk)); + let sdk_path = self.apple_sdk_root(sdk.as_str())?; cmd.args.push("-isysroot".into()); cmd.args.push(sdk_path); cmd.args.push("-fembed-bitcode".into()); From ceb03741e2973651ce8ada26a0250ac32f64eb6a Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Tue, 8 Mar 2022 17:59:36 +0000 Subject: [PATCH 007/138] [watchos] fix format --- src/lib.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d19c4a7bc..97d27373f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2148,8 +2148,16 @@ impl Build { }; let (sdk_prefix, sim_prefix, min_version) = match os { - Os::Ios => ("iphone", "ios-", std::env::var("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "7.0".into())), - Os::WatchOs => ("watch", "watch", std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into())), + Os::Ios => ( + "iphone", + "ios-", + std::env::var("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "7.0".into()), + ), + Os::WatchOs => ( + "watch", + "watch", + std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into()), + ), }; let sdk = match arch { From f461a550c3dd0c94a308a47720989650d75db0ff Mon Sep 17 00:00:00 2001 From: Vladimir Michael Eatwell Date: Tue, 8 Mar 2022 18:27:29 +0000 Subject: [PATCH 008/138] [watchos] remove redundant if statement --- src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 97d27373f..7101cf50c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1863,11 +1863,7 @@ impl Build { } } - if target.contains("apple-ios") { - self.ios_watchos_flags(cmd)?; - } - - if target.contains("apple-watchos") { + if target.contains("apple-ios") || target.contains("apple-watchos") { self.ios_watchos_flags(cmd)?; } From f5484fba9f76c0cc51049e04aa8995672b83d27d Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Sat, 30 Apr 2022 15:32:11 +0200 Subject: [PATCH 009/138] Update CUDA Linux GPG repository key [and compiler version]. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42f358875..62fa522e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,16 +92,16 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Install cuda-minimal-build-11-4 + - name: Install cuda-minimal-build-11-6 shell: bash run: | # https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=deb_network wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" sudo apt-get update - sudo apt-get -y install cuda-minimal-build-11-4 + sudo apt-get -y install cuda-minimal-build-11-6 - name: Test 'cudart' feature shell: bash run: env PATH=/usr/local/cuda/bin:$PATH cargo test --manifest-path cc-test/Cargo.toml --features test_cuda From d4ded9df82b788c185b9c5348c1c1eb92eaee2da Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Sat, 30 Apr 2022 15:47:26 +0200 Subject: [PATCH 010/138] windows-2016 was removed from Github Actions on March 15, 2022. --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62fa522e0..303a448ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - build: [stable, beta, nightly, linux32, macos, aarch64-ios, win32, win64, mingw32, mingw64, windows-2016] + build: [stable, beta, nightly, linux32, macos, aarch64-ios, win32, win64, mingw32, mingw64, windows-2019] include: - build: stable os: ubuntu-latest @@ -40,7 +40,7 @@ jobs: target: aarch64-pc-windows-msvc no_run: --no-run - build: win32 - os: windows-2016 + os: windows-2019 rust: stable-i686-msvc target: i686-pc-windows-msvc - build: win64 @@ -55,8 +55,8 @@ jobs: os: windows-latest rust: stable-x86_64-gnu target: x86_64-pc-windows-gnu - - build: windows-2016 - os: windows-2016 + - build: windows-2019 + os: windows-2019 rust: stable-x86_64 target: x86_64-pc-windows-msvc steps: From f91654ac7e229b3df4a09db0399757039c05fb90 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Sun, 10 Jul 2022 13:21:19 +0200 Subject: [PATCH 011/138] .github/workflows/main.yml: force mingw32 test to Windows 2019. Actions' windows-latest is equipped with newer mingw, which causes linking problems in mingw32 debug builds. See #677 for details. --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 303a448ca..06209366f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,11 @@ jobs: rust: stable target: x86_64-pc-windows-msvc - build: mingw32 - os: windows-latest + # windows-latest, a.k.a. windows-2022, runner is equipped with + # a newer mingw toolchain, which appears to produce unexecutable + # mixed-language binaries in debug builds. Fall back to + # windows-2019 for now and revisit it later... + os: windows-2019 rust: stable-i686-gnu target: i686-pc-windows-gnu - build: mingw64 From 11f539037af0ebe078147e42c931c6d1b7cecf5b Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 7 Mar 2022 16:23:31 +0100 Subject: [PATCH 012/138] Use SDKROOT when set. --- src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 30ebc9212..d31e5232d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2181,7 +2181,12 @@ impl Build { }; self.print(&format!("Detecting {} SDK path for {}", os, sdk)); - let sdk_path = self.apple_sdk_root(sdk.as_str())?; + let sdk_path = if let Some(sdkroot) = env::var_os("SDKROOT") { + sdkroot + } else { + self.apple_sdk_root(sdk.as_str())? + }; + cmd.args.push("-isysroot".into()); cmd.args.push(sdk_path); cmd.args.push("-fembed-bitcode".into()); From faef75008a1a802ec24f5ec2ed7731e26b02fb5e Mon Sep 17 00:00:00 2001 From: Konstantin G Date: Mon, 10 Oct 2022 16:59:52 +0100 Subject: [PATCH 013/138] Support clang >13.1 for aarch64-apple-ios-macabi and x86_64-apple-ios-macabi targets --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index d31e5232d..ac18d7c12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1581,7 +1581,7 @@ impl Build { map_darwin_target_from_rust_to_compiler_architecture(target) { cmd.args - .push(format!("--target={}-apple-ios13.0-macabi", arch).into()); + .push(format!("--target={}-apple-ios-macabi", arch).into()); } } else if target.contains("ios-sim") { if let Some(arch) = From 39fc215e123d32b964e6b81ae0d5275b86977306 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 2 Apr 2022 12:02:50 +0300 Subject: [PATCH 014/138] Support native library modifiers (RFC 2951) --- src/lib.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ac18d7c12..87e102747 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,6 +114,7 @@ pub struct Build { compiler: Option, archiver: Option, cargo_metadata: bool, + link_lib_modifiers: Vec, pic: Option, use_plt: Option, static_crt: Option, @@ -312,6 +313,7 @@ impl Build { compiler: None, archiver: None, cargo_metadata: true, + link_lib_modifiers: Vec::new(), pic: None, use_plt: None, static_crt: None, @@ -898,6 +900,16 @@ impl Build { self } + /// Adds a native library modifier that will be added to the + /// `rustc-link-lib=static:MODIFIERS=LIBRARY_NAME` metadata line + /// emitted for cargo if `cargo_metadata` is enabled. + /// See https://doc.rust-lang.org/rustc/command-line-arguments.html#-l-link-the-generated-crate-to-a-native-library + /// for the list of modifiers accepted by rustc. + pub fn link_lib_modifier(&mut self, link_lib_modifier: &str) -> &mut Build { + self.link_lib_modifiers.push(link_lib_modifier.to_string()); + self + } + /// Configures whether the compiler will emit position independent code. /// /// This option defaults to `false` for `windows-gnu` and bare metal targets and @@ -1014,7 +1026,12 @@ impl Build { } } - self.print(&format!("cargo:rustc-link-lib=static={}", lib_name)); + if self.link_lib_modifiers.is_empty() { + self.print(&format!("cargo:rustc-link-lib=static={}", lib_name)); + } else { + let m = self.link_lib_modifiers.join(","); + self.print(&format!("cargo:rustc-link-lib=static:{}={}", m, lib_name)); + } self.print(&format!("cargo:rustc-link-search=native={}", dst.display())); // Add specific C++ libraries, if enabled. From 68f1f059ae7c9fb10b4e031e321a7d0bcaf4e2b3 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 25 Oct 2022 15:44:50 -0700 Subject: [PATCH 015/138] Fix repository and homepage links in Cargo.toml --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 227e2569e..d73cae8bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,8 +3,8 @@ name = "cc" version = "1.0.73" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" -repository = "https://github.com/alexcrichton/cc-rs" -homepage = "https://github.com/alexcrichton/cc-rs" +repository = "https://github.com/rust-lang/cc-rs" +homepage = "https://github.com/rust-lang/cc-rs" documentation = "https://docs.rs/cc" description = """ A build-time dependency for Cargo build scripts to assist in invoking the native From c87216b21d6ba6b7e7977d1071c09bdb8b641107 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 25 Oct 2022 15:45:58 -0700 Subject: [PATCH 016/138] Remove support for to gh-pages --- .github/workflows/main.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 06209366f..ebd63c210 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -131,21 +131,3 @@ jobs: - name: Install Rust run: rustup update stable && rustup default stable && rustup component add rustfmt - run: cargo fmt -- --check - - publish_docs: - name: Publish Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Install Rust - run: rustup update stable && rustup default stable - - name: Build documentation - run: cargo doc --no-deps --all-features - - name: Publish documentation - run: | - cd target/doc - git init - git add . - git -c user.name='ci' -c user.email='ci' commit -m init - git push -f -q https://git:${{ secrets.github_token }}@github.com/${{ github.repository }} HEAD:gh-pages - if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' From 0f280045d8a66b98989f8e87d5ae4b6c06687511 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Sun, 5 Jun 2022 10:46:25 +0200 Subject: [PATCH 017/138] Use RUSTC_LINKER's prefix as last resort for prefix_for_target(). --- src/lib.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 87e102747..8ea910b79 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2617,11 +2617,22 @@ impl Build { } fn prefix_for_target(&self, target: &str) -> Option { + // Put aside RUSTC_LINKER's prefix to be used as last resort + let rustc_linker = self.getenv("RUSTC_LINKER").unwrap_or("".to_string()); + // let linker_prefix = rustc_linker.strip_suffix("-gcc"); // >=1.45.0 + let linker_prefix = if rustc_linker.len() > 4 { + let (prefix, suffix) = rustc_linker.split_at(rustc_linker.len() - 4); + if suffix == "-gcc" { + Some(prefix) + } else { + None + } + } else { + None + }; // CROSS_COMPILE is of the form: "arm-linux-gnueabi-" let cc_env = self.getenv("CROSS_COMPILE"); - let cross_compile = cc_env - .as_ref() - .map(|s| s.trim_right_matches('-').to_owned()); + let cross_compile = cc_env.as_ref().map(|s| s.trim_end_matches('-').to_owned()); cross_compile.or(match &target[..] { "aarch64-pc-windows-gnu" => Some("aarch64-w64-mingw32"), "aarch64-uwp-windows-gnu" => Some("aarch64-w64-mingw32"), @@ -2728,7 +2739,7 @@ impl Build { ]), // explicit None if not found, so caller knows to fall back "x86_64-unknown-linux-musl" => Some("musl"), "x86_64-unknown-netbsd" => Some("x86_64--netbsd"), - _ => None, + _ => linker_prefix, } .map(|x| x.to_owned())) } From 8daff16ce11e2753961c9b0ce3398fdeada6d941 Mon Sep 17 00:00:00 2001 From: Sean Cross Date: Thu, 9 Jun 2022 08:58:00 +0800 Subject: [PATCH 018/138] Add support for riscv32imac-unknown-xous-elf The `riscv32imac-unknown-xous-elf` target is a new Tier 3 platform. Signed-off-by: Sean Cross --- src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 8ea910b79..366d9d5e6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2694,6 +2694,11 @@ impl Build { "riscv64-unknown-elf", "riscv-none-embed", ]), + "riscv32imac-unknown-xous-elf" => self.find_working_gnu_prefix(&[ + "riscv32-unknown-elf", + "riscv64-unknown-elf", + "riscv-none-embed", + ]), "riscv32imc-unknown-none-elf" => self.find_working_gnu_prefix(&[ "riscv32-unknown-elf", "riscv64-unknown-elf", From 4796486b011f4355cf7aa8ce3ba36c83e61d223a Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Wed, 26 Oct 2022 09:59:51 -0700 Subject: [PATCH 019/138] Correctly infer ranlib/ar from cross-gcc The GCC convention is that if the toolchain is named `$target-gnu-gcc`, then ar will be available as `$target-gnu-gcc-ar`. The code as written will infer it to be `$target-gnu-ar`, which won't work. I'm not sure why the code that existed was written the way it was -- I don't know of any GCC toolchains where the `-gcc` is stripped out in the name of ar. The file listing on Debian's [GCC 6.x] and [GCC 10.x] all show the binaries using the `$target-gnu-gcc-ar` format, as does Arch Linux's [GCC 12.x]. It may seem odd that the code always adds `-gcc` for that match arm, but this matches what we do for finding `$CC` and `$CXX` where we always inject the GNU prefix when target != host. Also note that there isn't a special `ar` for C++, so even when `self.cpp == true`, we should use `-gcc-ar`. Our saving grace, and likely the reason bug reports haven't come in about this, is that we also checks if the resulting binary name is executable, and if it isn't we fall back to the default AR instead. This means the bad heuristic is likely often masked by the presence of another working default AR. See also alexcrichton/openssl-src-rs#163. [GCC 6.x]: https://packages.debian.org/stretch/gcc [GCC 10.x]: https://packages.debian.org/stable/devel/gcc [GCC 12.x]: https://archlinux.org/packages/core/x86_64/gcc/ --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 366d9d5e6..59ba6570b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2601,7 +2601,7 @@ impl Build { } else if self.get_host()? != target { match self.prefix_for_target(&target) { Some(p) => { - let target_ar = format!("{}-ar", p); + let target_ar = format!("{}-gcc-ar", p); if Command::new(&target_ar).output().is_ok() { target_ar } else { From 76c821e0223ae8920694ce36ba91923a747e25ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Miku=C5=82a?= Date: Tue, 29 Mar 2022 01:19:01 +0200 Subject: [PATCH 020/138] Add LLVM based MinGW targets --- src/lib.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 59ba6570b..885589996 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2301,7 +2301,8 @@ impl Build { if target.contains("msvc") { msvc.to_string() } else { - format!("{}.exe", gnu) + let cc = if target.contains("llvm") { clang } else { gnu }; + format!("{}.exe", cc) } } else if target.contains("apple-ios") { clang.to_string() @@ -2329,7 +2330,10 @@ impl Build { } else if self.get_host()? != target { let prefix = self.prefix_for_target(&target); match prefix { - Some(prefix) => format!("{}-{}", prefix, gnu), + Some(prefix) => { + let cc = if target.contains("llvm") { clang } else { gnu }; + format!("{}-{}", prefix, cc) + } None => default.to_string(), } } else { @@ -2634,7 +2638,8 @@ impl Build { let cc_env = self.getenv("CROSS_COMPILE"); let cross_compile = cc_env.as_ref().map(|s| s.trim_end_matches('-').to_owned()); cross_compile.or(match &target[..] { - "aarch64-pc-windows-gnu" => Some("aarch64-w64-mingw32"), + // Note: there is no `aarch64-pc-windows-gnu` target, only `-gnullvm` + "aarch64-pc-windows-gnullvm" => Some("aarch64-w64-mingw32"), "aarch64-uwp-windows-gnu" => Some("aarch64-w64-mingw32"), "aarch64-unknown-linux-gnu" => Some("aarch64-linux-gnu"), "aarch64-unknown-linux-musl" => Some("aarch64-linux-musl"), @@ -2737,6 +2742,7 @@ impl Build { "thumbv8m.main-none-eabi" => Some("arm-none-eabi"), "thumbv8m.main-none-eabihf" => Some("arm-none-eabi"), "x86_64-pc-windows-gnu" => Some("x86_64-w64-mingw32"), + "x86_64-pc-windows-gnullvm" => Some("x86_64-w64-mingw32"), "x86_64-uwp-windows-gnu" => Some("x86_64-w64-mingw32"), "x86_64-rumprun-netbsd" => Some("x86_64-rumprun-netbsd"), "x86_64-unknown-linux-gnu" => self.find_working_gnu_prefix(&[ From 00befe79c2d831018f2182b57a08bb12547a046f Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Wed, 26 Oct 2022 13:20:33 -0700 Subject: [PATCH 021/138] Continue to support binutils ar sfackler [observed] that the `binutils` package [provides] binaries named `$target-ar` (with no `-gcc`). This patch augments the change made in #735 to continue picking those up too. [observed]: https://github.com/alexcrichton/openssl-src-rs/pull/163#issuecomment-1292439189 [provides]: https://packages.debian.org/bullseye/amd64/binutils-aarch64-linux-gnu/filelist --- src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 885589996..920545494 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2605,12 +2605,17 @@ impl Build { } else if self.get_host()? != target { match self.prefix_for_target(&target) { Some(p) => { - let target_ar = format!("{}-gcc-ar", p); - if Command::new(&target_ar).output().is_ok() { - target_ar - } else { - default_ar + // GCC uses $target-gcc-ar, whereas binutils uses $target-ar -- try both. + // Prefer -gcc-ar if it exists, since that matches what we'll use for $CC. + let mut ar = default_ar; + for &infix in &["-gcc", ""] { + let target_ar = format!("{}{}-ar", p, infix); + if Command::new(&target_ar).output().is_ok() { + ar = target_ar; + break; + } } + ar } None => default_ar, } From 3eeb50b391a7543a6fed96276ee9388b5430a50d Mon Sep 17 00:00:00 2001 From: Ast-x64 Date: Tue, 26 Apr 2022 11:08:00 +0800 Subject: [PATCH 022/138] Use specified compiler in is_flag_supported Use the compiler specified instead of the default one to test whether a flag can be accepted. Fixes https://github.com/alexcrichton/cc-rs/issues/675. --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 920545494..8868a7c9f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -477,6 +477,9 @@ impl Build { .debug(false) .cpp(self.cpp) .cuda(self.cuda); + if let Some(ref c) = self.compiler { + cfg.compiler(c.clone()); + } let mut compiler = cfg.try_get_compiler()?; // Clang uses stderr for verbose output, which yields a false positive From 962af5387b8ea4551be23cf0e0236f42238f2af2 Mon Sep 17 00:00:00 2001 From: Nathaniel Daniel Date: Tue, 19 Jul 2022 12:42:51 +0800 Subject: [PATCH 023/138] Add option to emit `rerun-if-env-changed` metadata --- src/lib.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 8868a7c9f..54585576e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -125,6 +125,7 @@ pub struct Build { extra_warnings: Option, env_cache: Arc>>>, apple_sdk_root_cache: Arc>>, + emit_rerun_if_env_changed: bool, } /// Represents the types of errors that may occur while using cc-rs. @@ -322,6 +323,7 @@ impl Build { warnings_into_errors: false, env_cache: Arc::new(Mutex::new(HashMap::new())), apple_sdk_root_cache: Arc::new(Mutex::new(HashMap::new())), + emit_rerun_if_env_changed: false, } } @@ -897,6 +899,7 @@ impl Build { /// - `rustc-link-search=native=`*target folder* /// - When target is MSVC, the ATL-MFC libs are added via `rustc-link-search=native=` /// - When C++ is enabled, the C++ stdlib is added via `rustc-link-lib` + /// - If `emit_rerun_if_env_changed` is `true`, `rerun-if-env-changed=`*env* /// pub fn cargo_metadata(&mut self, cargo_metadata: bool) -> &mut Build { self.cargo_metadata = cargo_metadata; @@ -936,6 +939,17 @@ impl Build { self.use_plt = Some(use_plt); self } + + /// Define whether metadata should be emitted for cargo to detect environment + /// changes that should trigger a rebuild. + /// + /// This has no effect if the `cargo_metadata` option is `false`. + /// + /// This option defaults to `false`. + pub fn emit_rerun_if_env_changed(&mut self, emit_rerun_if_env_changed: bool) -> &mut Build { + self.emit_rerun_if_env_changed = emit_rerun_if_env_changed; + self + } /// Configures whether the /MT flag or the /MD flag will be passed to msvc build tools. /// @@ -2840,6 +2854,9 @@ impl Build { } fn getenv(&self, v: &str) -> Option { + if self.emit_rerun_if_env_changed { + self.print(&format!("cargo:rerun-if-env-changed={}", v)); + } let mut cache = self.env_cache.lock().unwrap(); if let Some(val) = cache.get(v) { return val.clone(); From 765f486e750b7b95c9568e8fb76c3b038e1bd42c Mon Sep 17 00:00:00 2001 From: Nathaniel Daniel Date: Tue, 19 Jul 2022 12:49:49 +0800 Subject: [PATCH 024/138] Run cargo fmt --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 54585576e..068f7867b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -939,8 +939,8 @@ impl Build { self.use_plt = Some(use_plt); self } - - /// Define whether metadata should be emitted for cargo to detect environment + + /// Define whether metadata should be emitted for cargo to detect environment /// changes that should trigger a rebuild. /// /// This has no effect if the `cargo_metadata` option is `false`. From 0b25dbf7f2d70b83dcd1fe005cc94a5b356157e9 Mon Sep 17 00:00:00 2001 From: Nathaniel Daniel Date: Fri, 28 Oct 2022 20:26:37 -0700 Subject: [PATCH 025/138] Change getenv to only emit metadata once per var --- src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 068f7867b..e76a757fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2854,13 +2854,13 @@ impl Build { } fn getenv(&self, v: &str) -> Option { - if self.emit_rerun_if_env_changed { - self.print(&format!("cargo:rerun-if-env-changed={}", v)); - } let mut cache = self.env_cache.lock().unwrap(); if let Some(val) = cache.get(v) { return val.clone(); } + if self.emit_rerun_if_env_changed { + self.print(&format!("cargo:rerun-if-env-changed={}", v)); + } let r = env::var(v).ok(); self.print(&format!("{} = {:?}", v, r)); cache.insert(v.to_string(), r.clone()); From 019603acacb1c4fde4cd4ca98d31deb00149cd74 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sat, 29 Oct 2022 00:25:12 -0700 Subject: [PATCH 026/138] Default `emit_rerun_if_env_changed` to `true` --- src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e76a757fb..80f0f3c62 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -323,7 +323,7 @@ impl Build { warnings_into_errors: false, env_cache: Arc::new(Mutex::new(HashMap::new())), apple_sdk_root_cache: Arc::new(Mutex::new(HashMap::new())), - emit_rerun_if_env_changed: false, + emit_rerun_if_env_changed: true, } } @@ -899,7 +899,7 @@ impl Build { /// - `rustc-link-search=native=`*target folder* /// - When target is MSVC, the ATL-MFC libs are added via `rustc-link-search=native=` /// - When C++ is enabled, the C++ stdlib is added via `rustc-link-lib` - /// - If `emit_rerun_if_env_changed` is `true`, `rerun-if-env-changed=`*env* + /// - If `emit_rerun_if_env_changed` is not `false`, `rerun-if-env-changed=`*env* /// pub fn cargo_metadata(&mut self, cargo_metadata: bool) -> &mut Build { self.cargo_metadata = cargo_metadata; @@ -945,7 +945,7 @@ impl Build { /// /// This has no effect if the `cargo_metadata` option is `false`. /// - /// This option defaults to `false`. + /// This option defaults to `true`. pub fn emit_rerun_if_env_changed(&mut self, emit_rerun_if_env_changed: bool) -> &mut Build { self.emit_rerun_if_env_changed = emit_rerun_if_env_changed; self From f4ce3f6e1a65574708fdfa48e712faf646979ee8 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sat, 29 Oct 2022 00:43:30 -0700 Subject: [PATCH 027/138] Avoid `rerun-if-env-changed` on vars set by cargo for build scripts --- src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 80f0f3c62..57f31ae40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2854,11 +2854,25 @@ impl Build { } fn getenv(&self, v: &str) -> Option { + // Returns true for environment variables cargo sets for build scripts: + // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts + // + // This handles more of the vars than we actually use (it tries to check + // complete-ish set), just to avoid needing maintenance if/when new + // calls to `getenv`/`getenv_unwrap` are added. + fn provided_by_cargo(envvar: &str) -> bool { + match envvar { + v if v.starts_with("CARGO") || v.starts_with("RUSTC") => true, + "HOST" | "TARGET" | "RUSTDOC" | "OUT_DIR" | "OPT_LEVEL" | "DEBUG" | "PROFILE" + | "NUM_JOBS" | "RUSTFLAGS" => true, + _ => false, + } + } let mut cache = self.env_cache.lock().unwrap(); if let Some(val) = cache.get(v) { return val.clone(); } - if self.emit_rerun_if_env_changed { + if self.emit_rerun_if_env_changed && !provided_by_cargo(v) { self.print(&format!("cargo:rerun-if-env-changed={}", v)); } let r = env::var(v).ok(); From 490f3f84d78e8829bfa2914c2cea19e777a76b99 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 28 Oct 2022 21:47:42 -0700 Subject: [PATCH 028/138] Prepare release 1.0.74 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d73cae8bf..abc13e295 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.73" +version = "1.0.74" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 88c2f97316d8421aaf2f3660c2f400c29aabb6d9 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sat, 29 Oct 2022 11:19:35 -0700 Subject: [PATCH 029/138] Be much more defensive about how windows RegistryKeys are handled --- src/registry.rs | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/registry.rs b/src/registry.rs index 2ac2fa63b..2c4f2ab3d 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -14,7 +14,8 @@ use std::ops::RangeFrom; use std::os::raw; use std::os::windows::prelude::*; -pub struct RegistryKey(Repr); +/// Must never be `HKEY_PERFORMANCE_DATA`. +pub(crate) struct RegistryKey(Repr); type HKEY = *mut u8; type DWORD = u32; @@ -29,7 +30,8 @@ type REGSAM = u32; const ERROR_SUCCESS: DWORD = 0; const ERROR_NO_MORE_ITEMS: DWORD = 259; -const HKEY_LOCAL_MACHINE: HKEY = 0x80000002 as HKEY; +// Sign-extend into 64 bits if needed (although it seems to work even if not). +const HKEY_LOCAL_MACHINE: HKEY = 0x80000002u32 as i32 as isize as HKEY; const REG_SZ: DWORD = 1; const KEY_READ: DWORD = 0x20019; const KEY_WOW64_32KEY: DWORD = 0x200; @@ -66,8 +68,11 @@ extern "system" { struct OwnedKey(HKEY); +/// Note: must not encode `HKEY_PERFORMANCE_DATA` or one of its subkeys. enum Repr { - Const(HKEY), + /// `HKEY_LOCAL_MACHINE`. + LocalMachine, + /// A subkey of `HKEY_LOCAL_MACHINE`. Owned(OwnedKey), } @@ -79,16 +84,17 @@ pub struct Iter<'a> { unsafe impl Sync for Repr {} unsafe impl Send for Repr {} -pub static LOCAL_MACHINE: RegistryKey = RegistryKey(Repr::Const(HKEY_LOCAL_MACHINE)); +pub(crate) const LOCAL_MACHINE: RegistryKey = RegistryKey(Repr::LocalMachine); impl RegistryKey { fn raw(&self) -> HKEY { match self.0 { - Repr::Const(val) => val, + Repr::LocalMachine => HKEY_LOCAL_MACHINE, Repr::Owned(ref val) => val.0, } } + /// Open a sub-key of `self`. pub fn open(&self, key: &OsStr) -> io::Result { let key = key.encode_wide().chain(Some(0)).collect::>(); let mut ret = 0 as *mut _; @@ -140,9 +146,13 @@ impl RegistryKey { } // The length here is the length in bytes, but we're using wide - // characters so we need to be sure to halve it for the capacity + // characters so we need to be sure to halve it for the length // passed in. - let mut v = Vec::with_capacity(len as usize / 2); + assert!(len % 2 == 0, "impossible wide string size: {} bytes", len); + let vlen = len as usize / 2; + // Defensively initialized, see comment about + // `HKEY_PERFORMANCE_DATA` below. + let mut v = vec![0u16; vlen]; let err = RegQueryValueExW( self.raw(), name.as_ptr(), @@ -151,17 +161,34 @@ impl RegistryKey { v.as_mut_ptr() as *mut _, &mut len, ); + // We don't check for `ERROR_MORE_DATA` (which would if the value + // grew between the first and second call to `RegQueryValueExW`), + // both because it's extremely unlikely, and this is a bit more + // defensive more defensive against weird types of registry keys. if err != ERROR_SUCCESS as LONG { return Err(io::Error::from_raw_os_error(err as i32)); } - v.set_len(len as usize / 2); - + // The length is allowed to change, but should still be even, as + // well as smaller. + assert!(len % 2 == 0, "impossible wide string size: {} bytes", len); + // If the length grew but returned a success code, it *probably* + // indicates we're `HKEY_PERFORMANCE_DATA` or a subkey(?). We + // consider this UB, since those keys write "undefined" or + // "unpredictable" values to len, and need to use a completely + // different loop structure. This should be impossible (and enforce + // it in the API to the best of our ability), but to mitigate the + // damage we do some smoke-checks on the len, and ensure `v` has + // been fully initialized (rather than trusting the result of + // `RegQueryValueExW`). + let actual_len = len as usize / 2; + assert!(actual_len <= v.len()); + v.truncate(actual_len); // Some registry keys may have a terminating nul character, but // we're not interested in that, so chop it off if it's there. - if v[v.len() - 1] == 0 { + if !v.is_empty() && v[v.len() - 1] == 0 { v.pop(); } - Ok(OsString::from_wide(&v)) + return Ok(OsString::from_wide(&v)); } } } From 82853c6b3a11c03adb1e92ccc45c357572146194 Mon Sep 17 00:00:00 2001 From: Alex Touchet Date: Sat, 29 Oct 2022 19:39:50 -0700 Subject: [PATCH 030/138] Update URLs --- README.md | 6 +++--- src/com.rs | 4 ++-- src/lib.rs | 4 ++-- src/registry.rs | 4 ++-- src/setup_config.rs | 4 ++-- src/winapi.rs | 4 ++-- src/windows_registry.rs | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b52e095b9..863540d2d 100644 --- a/README.md +++ b/README.md @@ -135,12 +135,12 @@ required varies per platform, but there are three broad categories: the appropriate developer tools shell. * Windows platforms targeting MinGW (e.g. your target triple ends in `-gnu`) require `cc` to be available in `PATH`. We recommend the - [MinGW-w64](http://mingw-w64.org) distribution, which is using the - [Win-builds](http://win-builds.org) installation system. + [MinGW-w64](https://www.mingw-w64.org/) distribution, which is using the + [Win-builds](http://win-builds.org/) installation system. You may also acquire it via [MSYS2](https://www.msys2.org/), as explained [here][msys2-help]. Make sure to install the appropriate architecture corresponding to your installation of - rustc. GCC from older [MinGW](http://www.mingw.org) project is compatible + rustc. GCC from older [MinGW](http://www.mingw.org/) project is compatible only with 32-bit rust compiler. [msys2-help]: https://github.com/rust-lang/rust#building-on-windows diff --git a/src/com.rs b/src/com.rs index a5f2afedf..843247e58 100644 --- a/src/com.rs +++ b/src/com.rs @@ -1,7 +1,7 @@ // Copyright © 2017 winapi-rs developers // Licensed under the Apache License, Version 2.0 -// or the MIT license -// , at your option. +// or the MIT license +// , at your option. // All files in the project carrying such notice may not be copied, modified, or distributed // except according to those terms. diff --git a/src/lib.rs b/src/lib.rs index 57f31ae40..130389205 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2227,7 +2227,7 @@ impl Build { /* * TODO we probably ultimately want the -fembed-bitcode-marker flag * but can't have it now because of an issue in LLVM: - * https://github.com/alexcrichton/cc-rs/issues/301 + * https://github.com/rust-lang/cc-rs/issues/301 * https://github.com/rust-lang/rust/pull/48896#comment-372192660 */ /* @@ -3261,7 +3261,7 @@ fn spawn(cmd: &mut Command, program: &str) -> Result<(Child, JoinHandle<()>), Er } Err(ref e) if e.kind() == io::ErrorKind::NotFound => { let extra = if cfg!(windows) { - " (see https://github.com/alexcrichton/cc-rs#compile-time-requirements \ + " (see https://github.com/rust-lang/cc-rs#compile-time-requirements \ for help)" } else { "" diff --git a/src/registry.rs b/src/registry.rs index 2ac2fa63b..3fbaaa931 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -3,8 +3,8 @@ // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license -// , at your +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// , at your // option. This file may not be copied, modified, or distributed // except according to those terms. diff --git a/src/setup_config.rs b/src/setup_config.rs index bc2b1c2d3..030051ca6 100644 --- a/src/setup_config.rs +++ b/src/setup_config.rs @@ -1,7 +1,7 @@ // Copyright © 2017 winapi-rs developers // Licensed under the Apache License, Version 2.0 -// or the MIT license -// , at your option. +// or the MIT license +// , at your option. // All files in the project carrying such notice may not be copied, modified, or distributed // except according to those terms. diff --git a/src/winapi.rs b/src/winapi.rs index c416325b5..8e04ce9cb 100644 --- a/src/winapi.rs +++ b/src/winapi.rs @@ -1,7 +1,7 @@ // Copyright © 2015-2017 winapi-rs developers // Licensed under the Apache License, Version 2.0 -// or the MIT license -// , at your option. +// or the MIT license +// , at your option. // All files in the project carrying such notice may not be copied, modified, or distributed // except according to those terms. diff --git a/src/windows_registry.rs b/src/windows_registry.rs index 549082b30..36be029bd 100644 --- a/src/windows_registry.rs +++ b/src/windows_registry.rs @@ -3,8 +3,8 @@ // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license -// , at your +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// , at your // option. This file may not be copied, modified, or distributed // except according to those terms. @@ -393,7 +393,7 @@ mod impl_ { // according to Microsoft. To help head off potential regressions though, // we keep the registry method as a fallback option. // - // [more reliable]: https://github.com/alexcrichton/cc-rs/pull/331 + // [more reliable]: https://github.com/rust-lang/cc-rs/pull/331 fn find_tool_in_vs15_path(tool: &str, target: &str) -> Option { let mut path = match vs15plus_instances(target) { Some(instances) => instances From 544fb0fe78a617d55f3fe08d0d40109e22dabe54 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sun, 30 Oct 2022 02:50:17 -0700 Subject: [PATCH 031/138] Adjust comment on HKEY_LOCAL_MACHINE Co-authored-by: Chris Denton --- src/registry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registry.rs b/src/registry.rs index 2c4f2ab3d..3e7cd0fe0 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -30,7 +30,7 @@ type REGSAM = u32; const ERROR_SUCCESS: DWORD = 0; const ERROR_NO_MORE_ITEMS: DWORD = 259; -// Sign-extend into 64 bits if needed (although it seems to work even if not). +// Sign-extend into 64 bits if needed. const HKEY_LOCAL_MACHINE: HKEY = 0x80000002u32 as i32 as isize as HKEY; const REG_SZ: DWORD = 1; const KEY_READ: DWORD = 0x20019; From c05e9d9f39cfd9680cc59ae77c0ce1f938014932 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Mon, 31 Oct 2022 23:56:21 +0200 Subject: [PATCH 032/138] Add debuginfo flags for msvc assemblers When code is being built with debuginfo, we need to instruct the assemblers accordingly. --- src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 130389205..09e76cac1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1961,8 +1961,16 @@ impl Build { cmd.arg("-I").arg(directory); } if target.contains("aarch64") || target.contains("arm") { + if self.get_debug() { + cmd.arg("-g"); + } + println!("cargo:warning=The MSVC ARM assemblers do not support -D flags"); } else { + if self.get_debug() { + cmd.arg("-Zi"); + } + for &(ref key, ref value) in self.definitions.iter() { if let Some(ref value) = *value { cmd.arg(&format!("-D{}={}", key, value)); From ad1f00de0f799c3a140a6ced13c08820606763c9 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Mon, 31 Oct 2022 01:11:21 +0200 Subject: [PATCH 033/138] Prefer `-ar` to `-gcc-ar` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As observed in https://github.com/cross-rs/cross/issues/1100, in some situations `*-gcc-ar` might actually be just broken. I believe this is most likely just a plain bug in gcc (failing to consider `--disable-lto` configuration option somewhere in their build setup,) but for the time being `*-ar` tends to avoid this problem altogether. Code added in https://github.com/rust-lang/cc-rs/pull/736 appears to be preferring `*-gcc-ar`, but no strong rationale is given to prefer one over the other. `*-gcc-ar` being outright broken sometimes seems like a rationale strong enough to continue preferring binutils’ `ar`. --- src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 130389205..21185a7fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2623,9 +2623,12 @@ impl Build { match self.prefix_for_target(&target) { Some(p) => { // GCC uses $target-gcc-ar, whereas binutils uses $target-ar -- try both. - // Prefer -gcc-ar if it exists, since that matches what we'll use for $CC. + // Prefer -ar if it exists, as builds of `-gcc-ar` have been observed to be + // outright broken (such as when targetting freebsd with `--disable-lto` + // toolchain where the archiver attempts to load the LTO plugin anyway but + // fails to find one). let mut ar = default_ar; - for &infix in &["-gcc", ""] { + for &infix in &["", "-gcc"] { let target_ar = format!("{}{}-ar", p, infix); if Command::new(&target_ar).output().is_ok() { ar = target_ar; From 25b95c6873be0c62ac625d21d949ea44cb27839f Mon Sep 17 00:00:00 2001 From: Tom Niget Date: Thu, 14 Jul 2022 16:35:10 +0200 Subject: [PATCH 034/138] Fix location of atlmfc directory on MSVC 15+ Old code was expecting {VC}\Tools\MSVC\{Version}\bin\x64\atlmfc, correct path is {VC}\Tools\MSVC\{Version}\atlmfc --- src/windows_registry.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/windows_registry.rs b/src/windows_registry.rs index 36be029bd..276688b03 100644 --- a/src/windows_registry.rs +++ b/src/windows_registry.rs @@ -431,7 +431,7 @@ mod impl_ { target: &str, instance_path: &PathBuf, ) -> Option { - let (bin_path, host_dylib_path, lib_path, include_path) = + let (root_path, bin_path, host_dylib_path, lib_path, include_path) = vs15plus_vc_paths(target, instance_path)?; let tool_path = bin_path.join(tool); if !tool_path.exists() { @@ -444,7 +444,7 @@ mod impl_ { tool.libs.push(lib_path); tool.include.push(include_path); - if let Some((atl_lib_path, atl_include_path)) = atl_paths(target, &bin_path) { + if let Some((atl_lib_path, atl_include_path)) = atl_paths(target, &root_path) { tool.libs.push(atl_lib_path); tool.include.push(atl_include_path); } @@ -457,7 +457,7 @@ mod impl_ { fn vs15plus_vc_paths( target: &str, instance_path: &PathBuf, - ) -> Option<(PathBuf, PathBuf, PathBuf, PathBuf)> { + ) -> Option<(PathBuf, PathBuf, PathBuf, PathBuf, PathBuf)> { let version_path = instance_path.join(r"VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt"); let mut version_file = File::open(version_path).ok()?; @@ -490,7 +490,7 @@ mod impl_ { .join(&host.to_lowercase()); let lib_path = path.join("lib").join(&target); let include_path = path.join("include"); - Some((bin_path, host_dylib_path, lib_path, include_path)) + Some((path, bin_path, host_dylib_path, lib_path, include_path)) } fn atl_paths(target: &str, path: &Path) -> Option<(PathBuf, PathBuf)> { From 1c8b4a03f9e9c94ee2efdafd7367c1cf2e2ebcda Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 7 Nov 2022 23:39:03 -0800 Subject: [PATCH 035/138] Prep release 1.0.75 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index abc13e295..49e74adab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.74" +version = "1.0.75" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 0e5b395501a90344958f2ada0f19a8f8a5c38cf5 Mon Sep 17 00:00:00 2001 From: roblabla Date: Tue, 8 Nov 2022 23:07:46 +0100 Subject: [PATCH 036/138] Don't separate files/opt when using msvc assembler The MSVC assemblers don't support using `--` to mark the end of the options and the start of the "verbatim" file list. When the compiler family is MSVC with clang-cl, the assembler used will be the standard MSVC assembler. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index c42698698..a89e35318 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1345,7 +1345,7 @@ impl Build { if self.cuda && self.files.len() > 1 { cmd.arg("--device-c"); } - if compiler.family == (ToolFamily::Msvc { clang_cl: true }) { + if compiler.family == (ToolFamily::Msvc { clang_cl: true }) && !is_asm { // #513: For `clang-cl`, separate flags/options from the input file. // When cross-compiling macOS -> Windows, this avoids interpreting // common `/Users/...` paths as the `/U` flag and triggering From 8d4cc27d475081060dd5b277e1464b7ed9d889d8 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 8 Nov 2022 15:28:59 -0800 Subject: [PATCH 037/138] Prep release 1.0.76 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 49e74adab..df0ff3dee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.75" +version = "1.0.76" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 0fff25ae9aebfc0635e8b2ac719fda4047b4e252 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 7 Jul 2022 13:49:21 +0900 Subject: [PATCH 038/138] Use a DWARF version consistent with rustc Rustc defaults to DWARF-2 on some targets, and DWARF-4 on others. However using -g with the C compiler yields whatever default version the C compiler prefers. One side effect is that the DWARF debug info shipped in some libraries with rustc itself (e.g. libcompiler_builtins and others) have recently switched to DWARF-5 as a side effect of upgrading the clang version used on rustc CI. (https://github.com/rust-lang/rust/issues/98746) Ideally, the preferred DWARF version would be given by the rust compiler and/or cargo, but that's not the case at the moment, so the next best thing is something that aligns with the current defaults, although work in under way to add a rustc flag that would allow to pick the preferred DWARF version (https://github.com/rust-lang/rust/pull/98350) --- src/lib.rs | 29 ++++++++++++++++++++++++++--- tests/test.rs | 25 ++++++++++++++++++++----- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a89e35318..469b0f893 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -214,13 +214,17 @@ enum ToolFamily { impl ToolFamily { /// What the flag to request debug info for this family of tools look like - fn add_debug_flags(&self, cmd: &mut Tool) { + fn add_debug_flags(&self, cmd: &mut Tool, dwarf_version: Option) { match *self { ToolFamily::Msvc { .. } => { cmd.push_cc_arg("-Z7".into()); } ToolFamily::Gnu | ToolFamily::Clang => { - cmd.push_cc_arg("-g".into()); + cmd.push_cc_arg( + dwarf_version + .map_or_else(|| "-g".into(), |v| format!("-gdwarf-{}", v)) + .into(), + ); } } } @@ -1589,7 +1593,7 @@ impl Build { cmd.args.push("-G".into()); } let family = cmd.family; - family.add_debug_flags(cmd); + family.add_debug_flags(cmd, self.get_dwarf_version()); } if self.get_force_frame_pointer() { @@ -2848,6 +2852,25 @@ impl Build { }) } + fn get_dwarf_version(&self) -> Option { + // Tentatively matches the DWARF version defaults as of rustc 1.62. + let target = self.get_target().ok()?; + if target.contains("android") + || target.contains("apple") + || target.contains("dragonfly") + || target.contains("freebsd") + || target.contains("netbsd") + || target.contains("openbsd") + || target.contains("windows-gnu") + { + Some(2) + } else if target.contains("linux") { + Some(4) + } else { + None + } + } + fn get_force_frame_pointer(&self) -> bool { self.force_frame_pointer.unwrap_or_else(|| self.get_debug()) } diff --git a/tests/test.rs b/tests/test.rs index 35ef87577..98a03e851 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -20,7 +20,7 @@ fn gnu_smoke() { test.cmd(0) .must_have("-O2") .must_have("foo.c") - .must_not_have("-g") + .must_not_have("-gdwarf-4") .must_have("-c") .must_have("-ffunction-sections") .must_have("-fdata-sections"); @@ -52,11 +52,26 @@ fn gnu_opt_level_s() { .must_not_have("-Oz"); } +#[test] +fn gnu_debug() { + let test = Test::gnu(); + test.gcc().debug(true).file("foo.c").compile("foo"); + test.cmd(0).must_have("-gdwarf-4"); + + let test = Test::gnu(); + test.gcc() + .target("x86_64-apple-darwin") + .debug(true) + .file("foo.c") + .compile("foo"); + test.cmd(0).must_have("-gdwarf-2"); +} + #[test] fn gnu_debug_fp_auto() { let test = Test::gnu(); test.gcc().debug(true).file("foo.c").compile("foo"); - test.cmd(0).must_have("-g"); + test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_have("-fno-omit-frame-pointer"); } @@ -64,7 +79,7 @@ fn gnu_debug_fp_auto() { fn gnu_debug_fp() { let test = Test::gnu(); test.gcc().debug(true).file("foo.c").compile("foo"); - test.cmd(0).must_have("-g"); + test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_have("-fno-omit-frame-pointer"); } @@ -78,7 +93,7 @@ fn gnu_debug_nofp() { .force_frame_pointer(false) .file("foo.c") .compile("foo"); - test.cmd(0).must_have("-g"); + test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_not_have("-fno-omit-frame-pointer"); let test = Test::gnu(); @@ -87,7 +102,7 @@ fn gnu_debug_nofp() { .debug(true) .file("foo.c") .compile("foo"); - test.cmd(0).must_have("-g"); + test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_not_have("-fno-omit-frame-pointer"); } From 4008959bebf5234eae6eb2bf6302156866b8bee2 Mon Sep 17 00:00:00 2001 From: Nicholas Bishop Date: Wed, 16 Nov 2022 20:20:47 -0500 Subject: [PATCH 039/138] Add `Build::asm_flag` The method allows flags to be specified that are only applied to assembly input files, not C files. This is helpful with clang, which only accepts certain assembly options if the input file is asm, not C. --- src/lib.rs | 38 +++++++++++++++++++++++++++++++++++++- tests/test.rs | 17 +++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 469b0f893..486d67e0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -97,6 +97,7 @@ pub struct Build { flags_supported: Vec, known_flag_support_status: Arc>>, ar_flags: Vec, + asm_flags: Vec, no_default_flags: bool, files: Vec, cpp: bool, @@ -299,6 +300,7 @@ impl Build { flags_supported: Vec::new(), known_flag_support_status: Arc::new(Mutex::new(HashMap::new())), ar_flags: Vec::new(), + asm_flags: Vec::new(), no_default_flags: false, files: Vec::new(), shared_flag: None, @@ -434,6 +436,25 @@ impl Build { self } + /// Add a flag that will only be used with assembly files. + /// + /// The flag will be applied to input files with either a `.s` or + /// `.asm` extension (case insensitive). + /// + /// # Example + /// + /// ```no_run + /// cc::Build::new() + /// .asm_flag("-Wa,-defsym,abc=1") + /// .file("src/foo.S") // The asm flag will be applied here + /// .file("src/bar.c") // The asm flag will not be applied here + /// .compile("foo"); + /// ``` + pub fn asm_flag(&mut self, flag: &str) -> &mut Build { + self.asm_flags.push(flag.to_string()); + self + } + fn ensure_check_file(&self) -> Result { let out_dir = self.get_out_dir()?; let src = if self.cuda { @@ -1318,7 +1339,7 @@ impl Build { } fn compile_object(&self, obj: &Object) -> Result<(), Error> { - let is_asm = obj.src.extension().and_then(|s| s.to_str()) == Some("asm"); + let is_asm = is_asm(&obj.src); let target = self.get_target()?; let msvc = target.contains("msvc"); let compiler = self.try_get_compiler()?; @@ -1349,6 +1370,9 @@ impl Build { if self.cuda && self.files.len() > 1 { cmd.arg("--device-c"); } + if is_asm { + cmd.args(&self.asm_flags); + } if compiler.family == (ToolFamily::Msvc { clang_cl: true }) && !is_asm { // #513: For `clang-cl`, separate flags/options from the input file. // When cross-compiling macOS -> Windows, this avoids interpreting @@ -3471,3 +3495,15 @@ fn which(tool: &Path) -> Option { return if check_exe(&mut exe) { Some(exe) } else { None }; }) } + +/// Check if the file's extension is either "asm" or "s", case insensitive. +fn is_asm(file: &Path) -> bool { + if let Some(ext) = file.extension() { + if let Some(ext) = ext.to_str() { + let ext = ext.to_lowercase(); + return ext == "asm" || ext == "s"; + } + } + + false +} diff --git a/tests/test.rs b/tests/test.rs index 98a03e851..161abd8ab 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -442,3 +442,20 @@ fn msvc_no_dash_dash() { test.cmd(0).must_not_have("--"); } + +// Disable this test with the parallel feature because the execution +// order is not deterministic. +#[cfg(not(feature = "parallel"))] +#[test] +fn asm_flags() { + let test = Test::gnu(); + test.gcc() + .file("foo.c") + .file("x86_64.asm") + .file("x86_64.S") + .asm_flag("--abc") + .compile("foo"); + test.cmd(0).must_not_have("--abc"); + test.cmd(1).must_have("--abc"); + test.cmd(2).must_have("--abc"); +} From daa41b9abd5c2a8af2f4f605f88b1b12c8edb172 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sat, 19 Nov 2022 19:17:54 -0800 Subject: [PATCH 040/138] Prep release 1.0.77 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index df0ff3dee..78f12e091 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.76" +version = "1.0.77" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 4ce411709170197228d353aa2bbf385efe3b958a Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 23 Nov 2022 12:14:31 -0800 Subject: [PATCH 041/138] Only pass `.asm` files to masm --- src/lib.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 486d67e0b..52e7ba648 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1339,12 +1339,14 @@ impl Build { } fn compile_object(&self, obj: &Object) -> Result<(), Error> { - let is_asm = is_asm(&obj.src); + let asm_ext = AsmFileExt::from_path(&obj.src); + let is_asm = asm_ext.is_some(); let target = self.get_target()?; let msvc = target.contains("msvc"); let compiler = self.try_get_compiler()?; let clang = compiler.family == ToolFamily::Clang; - let (mut cmd, name) = if msvc && is_asm { + + let (mut cmd, name) = if msvc && asm_ext == Some(AsmFileExt::DotAsm) { self.msvc_macro_assembler()? } else { let mut cmd = compiler.to_command(); @@ -3496,14 +3498,27 @@ fn which(tool: &Path) -> Option { }) } -/// Check if the file's extension is either "asm" or "s", case insensitive. -fn is_asm(file: &Path) -> bool { - if let Some(ext) = file.extension() { - if let Some(ext) = ext.to_str() { - let ext = ext.to_lowercase(); - return ext == "asm" || ext == "s"; +#[derive(Clone, Copy, PartialEq)] +enum AsmFileExt { + /// `.asm` files. On MSVC targets, we assume these should be passed to MASM + /// (`ml{,64}.exe`). + DotAsm, + /// `.s` or `.S` files, which do not have the special handling on MSVC targets. + DotS, +} + +impl AsmFileExt { + fn from_path(file: &Path) -> Option { + if let Some(ext) = file.extension() { + if let Some(ext) = ext.to_str() { + let ext = ext.to_lowercase(); + match &*ext { + "asm" => return Some(AsmFileExt::DotAsm), + "s" => return Some(AsmFileExt::DotS), + _ => return None, + } + } } + None } - - false } From e0df9bad074bf172cee97123e9227b5c4f6acd90 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Thu, 4 Aug 2022 20:46:12 +0200 Subject: [PATCH 042/138] Allow to use clang++ with CUDA compiler. CUDA compiler supports clang++, but it wasn't possible to use it with cc-rs, because --target option was passed to nvcc instead of clang. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 52e7ba648..90816501d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1692,7 +1692,7 @@ impl Build { cmd.args.push("--target=aarch64-unknown-windows-gnu".into()) } } else { - cmd.args.push(format!("--target={}", target).into()); + cmd.push_cc_arg(format!("--target={}", target).into()); } } } From 792c31cda686710a5f49b38ccad7c1078c46e5dd Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Thu, 4 Aug 2022 20:51:31 +0200 Subject: [PATCH 043/138] Refine CUDA support. Instead of counting all source files we count only .cu files, ones with device code that actually require the special treatment. --- src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 90816501d..2e9a4501a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1369,7 +1369,7 @@ impl Build { if !msvc || !is_asm || !is_arm { cmd.arg("-c"); } - if self.cuda && self.files.len() > 1 { + if self.cuda && self.cuda_file_count() > 1 { cmd.arg("--device-c"); } if is_asm { @@ -2037,7 +2037,7 @@ impl Build { self.assemble_progressive(dst, chunk)?; } - if self.cuda { + if self.cuda && self.cuda_file_count() > 0 { // Link the device-side code and add it to the target library, // so that non-CUDA linker can link the final binary. @@ -3012,6 +3012,13 @@ impl Build { cache.insert(sdk.into(), ret.clone()); Ok(ret) } + + fn cuda_file_count(&self) -> usize { + self.files + .iter() + .filter(|file| file.extension() == Some(OsStr::new("cu"))) + .count() + } } impl Default for Build { From 2ce2be8c85d71ff15a7b65edad9b423aa6009bd2 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Wed, 23 Nov 2022 21:58:04 +0100 Subject: [PATCH 044/138] Update CUDA toolchain. --- .github/workflows/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ebd63c210..94638d9e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,16 +96,14 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Install cuda-minimal-build-11-6 + - name: Install cuda-minimal-build-11-8 shell: bash run: | # https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=deb_network - wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin - sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub - sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb + sudo dpkg -i cuda-keyring_1.0-1_all.deb sudo apt-get update - sudo apt-get -y install cuda-minimal-build-11-6 + sudo apt-get -y install cuda-minimal-build-11-8 - name: Test 'cudart' feature shell: bash run: env PATH=/usr/local/cuda/bin:$PATH cargo test --manifest-path cc-test/Cargo.toml --features test_cuda From c4f414f449bb7cffba3bc923f277704d1d08a8ec Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Thu, 24 Nov 2022 03:25:33 +0100 Subject: [PATCH 045/138] Map source absolute paths to OUT_DIR as relative. (#684) Fixes https://github.com/rust-lang/cc-rs/issues/683 --- src/lib.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2e9a4501a..fa899df01 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,11 +56,12 @@ #![allow(deprecated)] #![deny(missing_docs)] -use std::collections::HashMap; +use std::collections::{hash_map, HashMap}; use std::env; use std::ffi::{OsStr, OsString}; use std::fmt::{self, Display, Formatter}; use std::fs; +use std::hash::Hasher; use std::io::{self, BufRead, BufReader, Read, Write}; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio}; @@ -1023,7 +1024,24 @@ impl Build { let mut objects = Vec::new(); for file in self.files.iter() { - let obj = dst.join(file).with_extension("o"); + let obj = if file.has_root() { + // If `file` is an absolute path, prefix the `basename` + // with the `dirname`'s hash to ensure name uniqueness. + let basename = file + .file_name() + .ok_or_else(|| Error::new(ErrorKind::InvalidArgument, "file_name() failure"))? + .to_string_lossy(); + let dirname = file + .parent() + .ok_or_else(|| Error::new(ErrorKind::InvalidArgument, "parent() failure"))? + .to_string_lossy(); + let mut hasher = hash_map::DefaultHasher::new(); + hasher.write(dirname.to_string().as_bytes()); + dst.join(format!("{:016x}-{}", hasher.finish(), basename)) + .with_extension("o") + } else { + dst.join(file).with_extension("o") + }; let obj = if !obj.starts_with(&dst) { dst.join(obj.file_name().ok_or_else(|| { Error::new(ErrorKind::IOError, "Getting object file details failed.") From 0e51f6d8a05548bd9991bfd8184b45b77b261a17 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Wed, 30 Nov 2022 22:32:00 +0100 Subject: [PATCH 046/138] Default to llvm-lib when using clang-cl in msvc environment. The problem is that the vendor librarian can't handle object modules compiled with clang-cl -flto. llvm-lib on the other hand can handle any object modules, which makes it a preferred option. --- src/lib.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fa899df01..1ebd2cc7a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2665,10 +2665,29 @@ impl Build { "emar".to_string() } else if target.contains("msvc") { - match windows_registry::find(&target, "lib.exe") { - Some(t) => return Ok((t, "lib.exe".to_string())), - None => "lib.exe".to_string(), + let compiler = self.get_base_compiler()?; + let mut lib = String::new(); + if compiler.family == (ToolFamily::Msvc { clang_cl: true }) { + // See if there is 'llvm-lib' next to 'clang-cl' + // Another possibility could be to see if there is 'clang' + // next to 'clang-cl' and use 'search_programs()' to locate + // 'llvm-lib'. This is because 'clang-cl' doesn't support + // the -print-search-dirs option. + if let Some(mut cmd) = which(&compiler.path) { + cmd.pop(); + cmd.push("llvm-lib.exe"); + if let Some(llvm_lib) = which(&cmd) { + lib = llvm_lib.to_str().unwrap().to_owned(); + } + } + } + if lib.is_empty() { + lib = match windows_registry::find(&target, "lib.exe") { + Some(t) => return Ok((t, "lib.exe".to_string())), + None => "lib.exe".to_string(), + } } + lib } else if target.contains("illumos") { // The default 'ar' on illumos uses a non-standard flags, // but the OS comes bundled with a GNU-compatible variant. From 7d168c4a9b07fd422fb37e2de1f19f1635c8d419 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 14 Dec 2022 16:31:53 -0800 Subject: [PATCH 047/138] Prep release 1.0.78 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 78f12e091..c5b9003ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.77" +version = "1.0.78" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 03e5c7cdef128d04fe6fb93790638fa64306fb17 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 5 Jan 2023 09:26:09 -0800 Subject: [PATCH 048/138] Update bitcode TODO As of Xcode 14 Bitcode is deprecated but Apple still accepts apps built with bitcode if you're still using Xcode 13. Once Apple stops allowing you to upload apps built with Xcode 13 this argument should be removed. https://github.com/rust-lang/cc-rs/pull/768 --- src/lib.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1ebd2cc7a..8557e853c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2279,18 +2279,8 @@ impl Build { cmd.args.push("-isysroot".into()); cmd.args.push(sdk_path); + // TODO: Remove this once Apple stops accepting apps built with Xcode 13 cmd.args.push("-fembed-bitcode".into()); - /* - * TODO we probably ultimately want the -fembed-bitcode-marker flag - * but can't have it now because of an issue in LLVM: - * https://github.com/rust-lang/cc-rs/issues/301 - * https://github.com/rust-lang/rust/pull/48896#comment-372192660 - */ - /* - if self.get_opt_level()? == "0" { - cmd.args.push("-fembed-bitcode-marker".into()); - } - */ Ok(()) } From cf0a78b8ba8d15301c6596b35842da38f62b2233 Mon Sep 17 00:00:00 2001 From: Gavin Li Date: Wed, 18 Jan 2023 23:02:03 -0800 Subject: [PATCH 049/138] Fix "-arch arm64" flag for aarch64-ios-sim (#759) --- src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8557e853c..508c90dd1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2212,7 +2212,7 @@ impl Build { } } else if is_sim { match arch { - "arm64" | "aarch64" => ArchSpec::Simulator("-arch arm64"), + "arm64" | "aarch64" => ArchSpec::Simulator("arm64"), "x86_64" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( @@ -2262,7 +2262,13 @@ impl Build { format!("{}os", sdk_prefix) } ArchSpec::Simulator(arch) => { - cmd.args.push(arch.into()); + if arch.starts_with('-') { + // -m32 or -m64 + cmd.args.push(arch.into()); + } else { + cmd.args.push("-arch".into()); + cmd.args.push(arch.into()); + } cmd.args .push(format!("-m{}simulator-version-min={}", sim_prefix, min_version).into()); format!("{}simulator", sdk_prefix) From 3f773227134482bb74e972e243a42326925d5b9e Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Mon, 23 Jan 2023 15:47:57 -0800 Subject: [PATCH 050/138] Expose get_archiver and get_ranlib (#763) --- src/lib.rs | 295 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 221 insertions(+), 74 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 508c90dd1..58dbcebe8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,6 +115,7 @@ pub struct Build { env: Vec<(OsString, OsString)>, compiler: Option, archiver: Option, + ranlib: Option, cargo_metadata: bool, link_lib_modifiers: Vec, pic: Option, @@ -320,6 +321,7 @@ impl Build { env: Vec::new(), compiler: None, archiver: None, + ranlib: None, cargo_metadata: true, link_lib_modifiers: Vec::new(), pic: None, @@ -916,6 +918,17 @@ impl Build { self.archiver = Some(archiver.as_ref().to_owned()); self } + + /// Configures the tool used to index archives. + /// + /// This option is automatically determined from the target platform or a + /// number of environment variables, so it's not required to call this + /// function. + pub fn ranlib>(&mut self, ranlib: P) -> &mut Build { + self.ranlib = Some(ranlib.as_ref().to_owned()); + self + } + /// Define whether metadata should be emitted for cargo allowing it to /// automatically link the binary. Defaults to `true`. /// @@ -2094,7 +2107,11 @@ impl Build { // Non-msvc targets (those using `ar`) need a separate step to add // the symbol table to archives since our construction command of // `cq` doesn't add it for us. - let (mut ar, cmd) = self.get_ar()?; + let (mut ar, cmd, _any_flags) = self.get_ar()?; + + // NOTE: We add `s` even if flags were passed using $ARFLAGS/ar_flag, because `s` + // here represents a _mode_, not an arbitrary flag. Further discussion of this choice + // can be seen in https://github.com/rust-lang/cc-rs/pull/763. run(ar.arg("s").arg(dst), &cmd)?; } @@ -2105,12 +2122,16 @@ impl Build { let target = self.get_target()?; if target.contains("msvc") { - let (mut cmd, program) = self.get_ar()?; + let (mut cmd, program, any_flags) = self.get_ar()?; + // NOTE: -out: here is an I/O flag, and so must be included even if $ARFLAGS/ar_flag is + // in use. -nologo on the other hand is just a regular flag, and one that we'll skip if + // the caller has explicitly dictated the flags they want. See + // https://github.com/rust-lang/cc-rs/pull/763 for further discussion. let mut out = OsString::from("-out:"); out.push(dst); - cmd.arg(out).arg("-nologo"); - for flag in self.ar_flags.iter() { - cmd.arg(flag); + cmd.arg(out); + if !any_flags { + cmd.arg("-nologo"); } // If the library file already exists, add the library name // as an argument to let lib.exe know we are appending the objs. @@ -2120,7 +2141,7 @@ impl Build { cmd.args(objs); run(&mut cmd, &program)?; } else { - let (mut ar, cmd) = self.get_ar()?; + let (mut ar, cmd, _any_flags) = self.get_ar()?; // Set an environment variable to tell the OSX archiver to ensure // that all dates listed in the archive are zero, improving @@ -2145,9 +2166,10 @@ impl Build { // In any case if this doesn't end up getting read, it shouldn't // cause that many issues! ar.env("ZERO_AR_DATE", "1"); - for flag in self.ar_flags.iter() { - ar.arg(flag); - } + + // NOTE: We add cq here regardless of whether $ARFLAGS/ar_flag have been used because + // it dictates the _mode_ ar runs in, which the setter of $ARFLAGS/ar_flag can't + // dictate. See https://github.com/rust-lang/cc-rs/pull/763 for further discussion. run(ar.arg("cq").arg(dst).args(objs), &cmd)?; } @@ -2639,81 +2661,206 @@ impl Build { } } - fn get_ar(&self) -> Result<(Command, String), Error> { - if let Some(ref p) = self.archiver { - let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("ar"); - return Ok((self.cmd(p), name.to_string())); + fn get_ar(&self) -> Result<(Command, String, bool), Error> { + self.try_get_archiver_and_flags() + } + + /// Get the archiver (ar) that's in use for this configuration. + /// + /// You can use [`Command::get_program`] to get just the path to the command. + /// + /// This method will take into account all configuration such as debug + /// information, optimization level, include directories, defines, etc. + /// Additionally, the compiler binary in use follows the standard + /// conventions for this path, e.g. looking at the explicitly set compiler, + /// environment variables (a number of which are inspected here), and then + /// falling back to the default configuration. + /// + /// # Panics + /// + /// Panics if an error occurred while determining the architecture. + pub fn get_archiver(&self) -> Command { + match self.try_get_archiver() { + Ok(tool) => tool, + Err(e) => fail(&e.message), + } + } + + /// Get the archiver that's in use for this configuration. + /// + /// This will return a result instead of panicing; + /// see [`get_archiver()`] for the complete description. + pub fn try_get_archiver(&self) -> Result { + Ok(self.try_get_archiver_and_flags()?.0) + } + + fn try_get_archiver_and_flags(&self) -> Result<(Command, String, bool), Error> { + let (mut cmd, name) = self.get_base_archiver()?; + let flags = self.envflags("ARFLAGS"); + let mut any_flags = !flags.is_empty(); + cmd.args(flags); + for flag in &self.ar_flags { + any_flags = true; + cmd.arg(flag); } - if let Ok(p) = self.get_var("AR") { - return Ok((self.cmd(&p), p)); + Ok((cmd, name, any_flags)) + } + + fn get_base_archiver(&self) -> Result<(Command, String), Error> { + if let Some(ref a) = self.archiver { + return Ok((self.cmd(a), a.to_string_lossy().into_owned())); } - let target = self.get_target()?; - let default_ar = "ar".to_string(); - let program = if target.contains("android") { - format!("{}-ar", target.replace("armv7", "arm")) - } else if target.contains("emscripten") { - // Windows use bat files so we have to be a bit more specific - if cfg!(windows) { - let mut cmd = self.cmd("cmd"); - cmd.arg("/c").arg("emar.bat"); - return Ok((cmd, "emar.bat".to_string())); - } - "emar".to_string() - } else if target.contains("msvc") { - let compiler = self.get_base_compiler()?; - let mut lib = String::new(); - if compiler.family == (ToolFamily::Msvc { clang_cl: true }) { - // See if there is 'llvm-lib' next to 'clang-cl' - // Another possibility could be to see if there is 'clang' - // next to 'clang-cl' and use 'search_programs()' to locate - // 'llvm-lib'. This is because 'clang-cl' doesn't support - // the -print-search-dirs option. - if let Some(mut cmd) = which(&compiler.path) { - cmd.pop(); - cmd.push("llvm-lib.exe"); - if let Some(llvm_lib) = which(&cmd) { - lib = llvm_lib.to_str().unwrap().to_owned(); + self.get_base_archiver_variant("AR", "ar") + } + + /// Get the ranlib that's in use for this configuration. + /// + /// You can use [`Command::get_program`] to get just the path to the command. + /// + /// This method will take into account all configuration such as debug + /// information, optimization level, include directories, defines, etc. + /// Additionally, the compiler binary in use follows the standard + /// conventions for this path, e.g. looking at the explicitly set compiler, + /// environment variables (a number of which are inspected here), and then + /// falling back to the default configuration. + /// + /// # Panics + /// + /// Panics if an error occurred while determining the architecture. + pub fn get_ranlib(&self) -> Command { + match self.try_get_ranlib() { + Ok(tool) => tool, + Err(e) => fail(&e.message), + } + } + + /// Get the ranlib that's in use for this configuration. + /// + /// This will return a result instead of panicing; + /// see [`get_ranlib()`] for the complete description. + pub fn try_get_ranlib(&self) -> Result { + let mut cmd = self.get_base_ranlib()?; + cmd.args(self.envflags("RANLIBFLAGS")); + Ok(cmd) + } + + fn get_base_ranlib(&self) -> Result { + if let Some(ref r) = self.ranlib { + return Ok(self.cmd(r)); + } + + Ok(self.get_base_archiver_variant("RANLIB", "ranlib")?.0) + } + + fn get_base_archiver_variant(&self, env: &str, tool: &str) -> Result<(Command, String), Error> { + let target = self.get_target()?; + let mut name = String::new(); + let tool_opt: Option = self + .env_tool(env) + .map(|(tool, _wrapper, args)| { + let mut cmd = self.cmd(tool); + cmd.args(args); + cmd + }) + .or_else(|| { + if target.contains("emscripten") { + // Windows use bat files so we have to be a bit more specific + if cfg!(windows) { + let mut cmd = self.cmd("cmd"); + name = format!("em{}.bat", tool); + cmd.arg("/c").arg(&name); + Some(cmd) + } else { + name = format!("em{}", tool); + Some(self.cmd(&name)) } + } else { + None } - } - if lib.is_empty() { - lib = match windows_registry::find(&target, "lib.exe") { - Some(t) => return Ok((t, "lib.exe".to_string())), - None => "lib.exe".to_string(), - } - } - lib - } else if target.contains("illumos") { - // The default 'ar' on illumos uses a non-standard flags, - // but the OS comes bundled with a GNU-compatible variant. - // - // Use the GNU-variant to match other Unix systems. - "gar".to_string() - } else if self.get_host()? != target { - match self.prefix_for_target(&target) { - Some(p) => { - // GCC uses $target-gcc-ar, whereas binutils uses $target-ar -- try both. - // Prefer -ar if it exists, as builds of `-gcc-ar` have been observed to be - // outright broken (such as when targetting freebsd with `--disable-lto` - // toolchain where the archiver attempts to load the LTO plugin anyway but - // fails to find one). - let mut ar = default_ar; - for &infix in &["", "-gcc"] { - let target_ar = format!("{}{}-ar", p, infix); - if Command::new(&target_ar).output().is_ok() { - ar = target_ar; - break; + }); + + let default = tool.to_string(); + let tool = match tool_opt { + Some(t) => t, + None => { + if target.contains("android") { + name = format!("{}-{}", target.replace("armv7", "arm"), tool); + self.cmd(&name) + } else if target.contains("msvc") { + // NOTE: There isn't really a ranlib on msvc, so arguably we should return + // `None` somehow here. But in general, callers will already have to be aware + // of not running ranlib on Windows anyway, so it feels okay to return lib.exe + // here. + + let compiler = self.get_base_compiler()?; + let mut lib = String::new(); + if compiler.family == (ToolFamily::Msvc { clang_cl: true }) { + // See if there is 'llvm-lib' next to 'clang-cl' + // Another possibility could be to see if there is 'clang' + // next to 'clang-cl' and use 'search_programs()' to locate + // 'llvm-lib'. This is because 'clang-cl' doesn't support + // the -print-search-dirs option. + if let Some(mut cmd) = which(&compiler.path) { + cmd.pop(); + cmd.push("llvm-lib.exe"); + if let Some(llvm_lib) = which(&cmd) { + lib = llvm_lib.to_str().unwrap().to_owned(); + } + } + } + + if lib.is_empty() { + name = String::from("lib.exe"); + match windows_registry::find(&target, "lib.exe") { + Some(t) => t, + None => self.cmd("lib.exe"), + } + } else { + name = lib; + self.cmd(&name) + } + } else if target.contains("illumos") { + // The default 'ar' on illumos uses a non-standard flags, + // but the OS comes bundled with a GNU-compatible variant. + // + // Use the GNU-variant to match other Unix systems. + name = format!("g{}", tool); + self.cmd(&name) + } else if self.get_host()? != target { + match self.prefix_for_target(&target) { + Some(p) => { + // GCC uses $target-gcc-ar, whereas binutils uses $target-ar -- try both. + // Prefer -ar if it exists, as builds of `-gcc-ar` have been observed to be + // outright broken (such as when targetting freebsd with `--disable-lto` + // toolchain where the archiver attempts to load the LTO plugin anyway but + // fails to find one). + // + // The same applies to ranlib. + let mut chosen = default; + for &infix in &["", "-gcc"] { + let target_p = format!("{}{}-{}", p, infix, tool); + if Command::new(&target_p).output().is_ok() { + chosen = target_p; + break; + } + } + name = chosen; + self.cmd(&name) + } + None => { + name = default; + self.cmd(&name) } } - ar + } else { + name = default; + self.cmd(&name) } - None => default_ar, } - } else { - default_ar }; - Ok((self.cmd(&program), program)) + + Ok((tool, name)) } fn prefix_for_target(&self, target: &str) -> Option { From d69bc7db4493685508a1fecab8b98c5de811bd45 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 24 Jan 2023 13:02:50 +1100 Subject: [PATCH 051/138] Fix `Build::is_flag_supported`: Check exit status of the compiler (#757) --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 58dbcebe8..2676b1f0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -540,7 +540,7 @@ impl Build { cmd.arg(&src); let output = cmd.output()?; - let is_supported = output.stderr.is_empty(); + let is_supported = output.status.success() && output.stderr.is_empty(); known_status.insert(flag.to_owned(), is_supported); Ok(is_supported) From c4f7a47ee3d73990c43943029c12ca776c61d0b5 Mon Sep 17 00:00:00 2001 From: Josh Guilfoyle Date: Wed, 25 Jan 2023 13:45:06 -0800 Subject: [PATCH 052/138] Add support for riscv32imc-esp-espidf (#776) --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 2676b1f0a..abc5d7a95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2947,6 +2947,7 @@ impl Build { "riscv64-unknown-elf", "riscv-none-embed", ]), + "riscv32imc-esp-espidf" => Some("riscv32-esp-elf"), "riscv32imc-unknown-none-elf" => self.find_working_gnu_prefix(&[ "riscv32-unknown-elf", "riscv64-unknown-elf", From f507481317232a8c0f7083afd0044bb072bcda09 Mon Sep 17 00:00:00 2001 From: Dennis Duda Date: Tue, 24 Jan 2023 02:18:14 +0100 Subject: [PATCH 053/138] fix msbuild from vs2022 not being found --- src/windows_registry.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/windows_registry.rs b/src/windows_registry.rs index 276688b03..b05498c29 100644 --- a/src/windows_registry.rs +++ b/src/windows_registry.rs @@ -866,7 +866,9 @@ mod impl_ { // see http://stackoverflow.com/questions/328017/path-to-msbuild pub fn find_msbuild(target: &str) -> Option { // VS 15 (2017) changed how to locate msbuild - if let Some(r) = find_msbuild_vs16(target) { + if let Some(r) = find_msbuild_vs17(target) { + return Some(r); + } else if let Some(r) = find_msbuild_vs16(target) { return Some(r); } else if let Some(r) = find_msbuild_vs15(target) { return Some(r); From 7575f2f8b776fc39cc6d89a4afa23b724368b540 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sat, 28 Jan 2023 03:29:21 +1100 Subject: [PATCH 054/138] Fix `gcc-shim`, apply clippy warning & optimizations to it (#777) --- src/bin/gcc-shim.rs | 68 ++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/src/bin/gcc-shim.rs b/src/bin/gcc-shim.rs index 1731df82e..e5b537258 100644 --- a/src/bin/gcc-shim.rs +++ b/src/bin/gcc-shim.rs @@ -2,7 +2,7 @@ use std::env; use std::fs::File; -use std::io::prelude::*; +use std::io::{self, prelude::*}; use std::path::PathBuf; fn main() { @@ -10,39 +10,61 @@ fn main() { let program = args.next().expect("Unexpected empty args"); let out_dir = PathBuf::from( - env::var_os("GCCTEST_OUT_DIR").expect(&format!("{}: GCCTEST_OUT_DIR not found", program)), + env::var_os("GCCTEST_OUT_DIR") + .unwrap_or_else(|| panic!("{}: GCCTEST_OUT_DIR not found", program)), ); // Find the first nonexistent candidate file to which the program's args can be written. - for i in 0.. { - let candidate = &out_dir.join(format!("out{}", i)); + let candidate = (0..).find_map(|i| { + let candidate = out_dir.join(format!("out{}", i)); - // If the file exists, commands have already run. Try again. if candidate.exists() { - continue; + // If the file exists, commands have already run. Try again. + None + } else { + Some(candidate) } + }).unwrap_or_else(|| panic!("Cannot find the first nonexistent candidate file to which the program's args can be written under out_dir '{}'", out_dir.display())); - // Create a file and record the args passed to the command. - let mut f = File::create(candidate).expect(&format!( - "{}: can't create candidate: {}", + // Create a file and record the args passed to the command. + let f = File::create(&candidate).unwrap_or_else(|e| { + panic!( + "{}: can't create candidate: {}, error: {}", program, - candidate.to_string_lossy() - )); + candidate.display(), + e + ) + }); + let mut f = io::BufWriter::new(f); + + (|| { for arg in args { - writeln!(f, "{}", arg).expect(&format!( - "{}: can't write to candidate: {}", - program, - candidate.to_string_lossy() - )); + writeln!(f, "{}", arg)?; } - break; - } + + f.flush()?; + + let mut f = f.into_inner()?; + f.flush()?; + f.sync_all() + })() + .unwrap_or_else(|e| { + panic!( + "{}: can't write to candidate: {}, error: {}", + program, + candidate.display(), + e + ) + }); // Create a file used by some tests. let path = &out_dir.join("libfoo.a"); - File::create(path).expect(&format!( - "{}: can't create libfoo.a: {}", - program, - path.to_string_lossy() - )); + File::create(path).unwrap_or_else(|e| { + panic!( + "{}: can't create libfoo.a: {}, error: {}", + program, + path.display(), + e + ) + }); } From 60e8e5a55f253d6a1e382ed0ac63582e8dafa08b Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 27 Jan 2023 17:17:13 -0800 Subject: [PATCH 055/138] Prep release 1.0.79 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c5b9003ae..3128f0367 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.78" +version = "1.0.79" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 00b57f2b7a99e87f39702b9d41664fee8f79ba7a Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 31 Jan 2023 00:28:48 -0800 Subject: [PATCH 056/138] Bump MSRV to 1.46.0 (#781) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 94638d9e9..ceb9e539a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,7 +117,7 @@ jobs: steps: - uses: actions/checkout@master - name: Install Rust - run: rustup update 1.34.0 --no-self-update && rustup default 1.34.0 + run: rustup update 1.46.0 --no-self-update && rustup default 1.46.0 shell: bash - run: cargo build From 6008cbf1ce187c1c65ba2c0684a5d498495f15d5 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 1 Feb 2023 09:45:00 +1100 Subject: [PATCH 057/138] Make `windows_registry::VsVers` non_exhaustive (#783) --- src/windows_registry.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/windows_registry.rs b/src/windows_registry.rs index b05498c29..07a9c4db6 100644 --- a/src/windows_registry.rs +++ b/src/windows_registry.rs @@ -80,6 +80,7 @@ pub fn find_tool(target: &str, tool: &str) -> Option { /// A version of Visual Studio #[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[non_exhaustive] pub enum VsVers { /// Visual Studio 12 (2013) Vs12, @@ -91,13 +92,6 @@ pub enum VsVers { Vs16, /// Visual Studio 17 (2022) Vs17, - - /// Hidden variant that should not be matched on. Callers that want to - /// handle an enumeration of `VsVers` instances should always have a default - /// case meaning that it's a VS version they don't understand. - #[doc(hidden)] - #[allow(bad_style)] - __Nonexhaustive_do_not_match_this_or_your_code_will_break, } /// Find the most recent installed version of Visual Studio @@ -106,7 +100,7 @@ pub enum VsVers { /// generator. #[cfg(not(windows))] pub fn find_vs_version() -> Result { - Err(format!("not windows")) + Err("not windows".to_string()) } /// Documented above From 9e50fb819a3593ea35dee74cdcf6b94d66a5d62b Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:45:44 +0100 Subject: [PATCH 058/138] Prioritize RUSTC_LINKER over target->cross-compile-prefix table. (#767) fix https://github.com/rust-lang/cc-rs/pull/654 --- src/lib.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index abc5d7a95..66f917b86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2864,23 +2864,14 @@ impl Build { } fn prefix_for_target(&self, target: &str) -> Option { - // Put aside RUSTC_LINKER's prefix to be used as last resort - let rustc_linker = self.getenv("RUSTC_LINKER").unwrap_or("".to_string()); - // let linker_prefix = rustc_linker.strip_suffix("-gcc"); // >=1.45.0 - let linker_prefix = if rustc_linker.len() > 4 { - let (prefix, suffix) = rustc_linker.split_at(rustc_linker.len() - 4); - if suffix == "-gcc" { - Some(prefix) - } else { - None - } - } else { - None - }; + // Put aside RUSTC_LINKER's prefix to be used as second choice, after CROSS_COMPILE + let linker_prefix = self + .getenv("RUSTC_LINKER") + .and_then(|var| var.strip_suffix("-gcc").map(str::to_string)); // CROSS_COMPILE is of the form: "arm-linux-gnueabi-" let cc_env = self.getenv("CROSS_COMPILE"); let cross_compile = cc_env.as_ref().map(|s| s.trim_end_matches('-').to_owned()); - cross_compile.or(match &target[..] { + cross_compile.or(linker_prefix).or(match &target[..] { // Note: there is no `aarch64-pc-windows-gnu` target, only `-gnullvm` "aarch64-pc-windows-gnullvm" => Some("aarch64-w64-mingw32"), "aarch64-uwp-windows-gnu" => Some("aarch64-w64-mingw32"), @@ -2994,7 +2985,7 @@ impl Build { ]), // explicit None if not found, so caller knows to fall back "x86_64-unknown-linux-musl" => Some("musl"), "x86_64-unknown-netbsd" => Some("x86_64--netbsd"), - _ => linker_prefix, + _ => None, } .map(|x| x.to_owned())) } From cadbb52b5fa3ca630b3fc9a3025588f91ed662aa Mon Sep 17 00:00:00 2001 From: Cody Schafer Date: Wed, 1 Feb 2023 21:26:36 -0500 Subject: [PATCH 059/138] If a file path references a parent directory, transform it to ensure we keep inside the OUT_DIR (#786) --- src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 66f917b86..499921767 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1037,9 +1037,10 @@ impl Build { let mut objects = Vec::new(); for file in self.files.iter() { - let obj = if file.has_root() { - // If `file` is an absolute path, prefix the `basename` - // with the `dirname`'s hash to ensure name uniqueness. + let obj = if file.has_root() || file.components().any(|x| x == Component::ParentDir) { + // If `file` is an absolute path or might not be usable directly as a suffix due to + // using "..", use the `basename` prefixed with the `dirname`'s hash to ensure name + // uniqueness. let basename = file .file_name() .ok_or_else(|| Error::new(ErrorKind::InvalidArgument, "file_name() failure"))? From d2479572a1165b775734980564b0fdf9236dd4b7 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 2 Feb 2023 19:34:24 +1100 Subject: [PATCH 060/138] Speedup `Build::clone`: Use `Arc` instead of `String`/`PathBuf`/`OsString` (#782) --- src/lib.rs | 131 ++++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 67 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 499921767..d31f66c24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,33 +91,33 @@ pub mod windows_registry; /// documentation on each method itself. #[derive(Clone, Debug)] pub struct Build { - include_directories: Vec, - definitions: Vec<(String, Option)>, - objects: Vec, - flags: Vec, - flags_supported: Vec, + include_directories: Vec>, + definitions: Vec<(Arc, Option>)>, + objects: Vec>, + flags: Vec>, + flags_supported: Vec>, known_flag_support_status: Arc>>, - ar_flags: Vec, - asm_flags: Vec, + ar_flags: Vec>, + asm_flags: Vec>, no_default_flags: bool, - files: Vec, + files: Vec>, cpp: bool, - cpp_link_stdlib: Option>, - cpp_set_stdlib: Option, + cpp_link_stdlib: Option>>, + cpp_set_stdlib: Option>, cuda: bool, - cudart: Option, - target: Option, - host: Option, - out_dir: Option, - opt_level: Option, + cudart: Option>, + target: Option>, + host: Option>, + out_dir: Option>, + opt_level: Option>, debug: Option, force_frame_pointer: Option, - env: Vec<(OsString, OsString)>, - compiler: Option, - archiver: Option, - ranlib: Option, + env: Vec<(Arc, Arc)>, + compiler: Option>, + archiver: Option>, + ranlib: Option>, cargo_metadata: bool, - link_lib_modifiers: Vec, + link_lib_modifiers: Vec>, pic: Option, use_plt: Option, static_crt: Option, @@ -352,7 +352,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn include>(&mut self, dir: P) -> &mut Build { - self.include_directories.push(dir.as_ref().to_path_buf()); + self.include_directories.push(dir.as_ref().into()); self } @@ -398,13 +398,13 @@ impl Build { /// ``` pub fn define<'a, V: Into>>(&mut self, var: &str, val: V) -> &mut Build { self.definitions - .push((var.to_string(), val.into().map(|s| s.to_string()))); + .push((var.into(), val.into().map(Into::into))); self } /// Add an arbitrary object file to link in pub fn object>(&mut self, obj: P) -> &mut Build { - self.objects.push(obj.as_ref().to_path_buf()); + self.objects.push(obj.as_ref().into()); self } @@ -419,7 +419,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn flag(&mut self, flag: &str) -> &mut Build { - self.flags.push(flag.to_string()); + self.flags.push(flag.into()); self } @@ -435,7 +435,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn ar_flag(&mut self, flag: &str) -> &mut Build { - self.ar_flags.push(flag.to_string()); + self.ar_flags.push(flag.into()); self } @@ -454,7 +454,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn asm_flag(&mut self, flag: &str) -> &mut Build { - self.asm_flags.push(flag.to_string()); + self.asm_flags.push(flag.into()); self } @@ -558,7 +558,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn flag_if_supported(&mut self, flag: &str) -> &mut Build { - self.flags_supported.push(flag.to_string()); + self.flags_supported.push(flag.into()); self } @@ -612,7 +612,7 @@ impl Build { /// Add a file which will be compiled pub fn file>(&mut self, p: P) -> &mut Build { - self.files.push(p.as_ref().to_path_buf()); + self.files.push(p.as_ref().into()); self } @@ -649,7 +649,7 @@ impl Build { self.cuda = cuda; if cuda { self.cpp = true; - self.cudart = Some("static".to_string()); + self.cudart = Some("static".into()); } self } @@ -662,7 +662,7 @@ impl Build { /// at all, if the default is right for the project. pub fn cudart(&mut self, cudart: &str) -> &mut Build { if self.cuda { - self.cudart = Some(cudart.to_string()); + self.cudart = Some(cudart.into()); } self } @@ -828,7 +828,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn target(&mut self, target: &str) -> &mut Build { - self.target = Some(target.to_string()); + self.target = Some(target.into()); self } @@ -846,7 +846,7 @@ impl Build { /// .compile("foo"); /// ``` pub fn host(&mut self, host: &str) -> &mut Build { - self.host = Some(host.to_string()); + self.host = Some(host.into()); self } @@ -855,7 +855,7 @@ impl Build { /// This option is automatically scraped from the `OPT_LEVEL` environment /// variable by build scripts, so it's not required to call this function. pub fn opt_level(&mut self, opt_level: u32) -> &mut Build { - self.opt_level = Some(opt_level.to_string()); + self.opt_level = Some(opt_level.to_string().into()); self } @@ -864,7 +864,7 @@ impl Build { /// This option is automatically scraped from the `OPT_LEVEL` environment /// variable by build scripts, so it's not required to call this function. pub fn opt_level_str(&mut self, opt_level: &str) -> &mut Build { - self.opt_level = Some(opt_level.to_string()); + self.opt_level = Some(opt_level.into()); self } @@ -895,7 +895,7 @@ impl Build { /// This option is automatically scraped from the `OUT_DIR` environment /// variable by build scripts, so it's not required to call this function. pub fn out_dir>(&mut self, out_dir: P) -> &mut Build { - self.out_dir = Some(out_dir.as_ref().to_owned()); + self.out_dir = Some(out_dir.as_ref().into()); self } @@ -905,7 +905,7 @@ impl Build { /// number of environment variables, so it's not required to call this /// function. pub fn compiler>(&mut self, compiler: P) -> &mut Build { - self.compiler = Some(compiler.as_ref().to_owned()); + self.compiler = Some(compiler.as_ref().into()); self } @@ -915,7 +915,7 @@ impl Build { /// number of environment variables, so it's not required to call this /// function. pub fn archiver>(&mut self, archiver: P) -> &mut Build { - self.archiver = Some(archiver.as_ref().to_owned()); + self.archiver = Some(archiver.as_ref().into()); self } @@ -925,7 +925,7 @@ impl Build { /// number of environment variables, so it's not required to call this /// function. pub fn ranlib>(&mut self, ranlib: P) -> &mut Build { - self.ranlib = Some(ranlib.as_ref().to_owned()); + self.ranlib = Some(ranlib.as_ref().into()); self } @@ -951,7 +951,7 @@ impl Build { /// See https://doc.rust-lang.org/rustc/command-line-arguments.html#-l-link-the-generated-crate-to-a-native-library /// for the list of modifiers accepted by rustc. pub fn link_lib_modifier(&mut self, link_lib_modifier: &str) -> &mut Build { - self.link_lib_modifiers.push(link_lib_modifier.to_string()); + self.link_lib_modifiers.push(link_lib_modifier.into()); self } @@ -1004,8 +1004,7 @@ impl Build { A: AsRef, B: AsRef, { - self.env - .push((a.as_ref().to_owned(), b.as_ref().to_owned())); + self.env.push((a.as_ref().into(), b.as_ref().into())); self } @@ -1116,7 +1115,7 @@ impl Build { } let cudart = match &self.cudart { - Some(opt) => opt.as_str(), // {none|shared|static} + Some(opt) => &*opt, // {none|shared|static} None => "none", }; if cudart != "none" { @@ -1405,7 +1404,7 @@ impl Build { cmd.arg("--device-c"); } if is_asm { - cmd.args(&self.asm_flags); + cmd.args(self.asm_flags.iter().map(std::ops::Deref::deref)); } if compiler.family == (ToolFamily::Msvc { clang_cl: true }) && !is_asm { // #513: For `clang-cl`, separate flags/options from the input file. @@ -1437,9 +1436,7 @@ impl Build { "Expand may only be called for a single file" ); - for file in self.files.iter() { - cmd.arg(file); - } + cmd.args(self.files.iter().map(std::ops::Deref::deref)); let name = compiler .path @@ -1520,7 +1517,7 @@ impl Build { for directory in self.include_directories.iter() { cmd.args.push("-I".into()); - cmd.args.push(directory.into()); + cmd.args.push((**directory).into()); } // If warnings and/or extra_warnings haven't been explicitly set, @@ -1546,12 +1543,12 @@ impl Build { } for flag in self.flags.iter() { - cmd.args.push(flag.into()); + cmd.args.push((**flag).into()); } for flag in self.flags_supported.iter() { if self.is_flag_supported(flag).unwrap_or(false) { - cmd.push_cc_arg(flag.into()); + cmd.push_cc_arg((**flag).into()); } } @@ -2020,7 +2017,7 @@ impl Build { let mut cmd = windows_registry::find(&target, tool).unwrap_or_else(|| self.cmd(tool)); cmd.arg("-nologo"); // undocumented, yet working with armasm[64] for directory in self.include_directories.iter() { - cmd.arg("-I").arg(directory); + cmd.arg("-I").arg(&**directory); } if target.contains("aarch64") || target.contains("arm") { if self.get_debug() { @@ -2046,7 +2043,7 @@ impl Build { cmd.arg("-safeseh"); } for flag in self.flags.iter() { - cmd.arg(flag); + cmd.arg(&**flag); } Ok((cmd, tool.to_string())) @@ -2063,7 +2060,7 @@ impl Build { let objs: Vec<_> = objs .iter() .map(|o| o.dst.clone()) - .chain(self.objects.clone()) + .chain(self.objects.iter().map(|path| (**path).to_owned())) .collect(); for chunk in objs.chunks(100) { self.assemble_progressive(dst, chunk)?; @@ -2323,8 +2320,8 @@ impl Build { } fn get_base_compiler(&self) -> Result { - if let Some(ref c) = self.compiler { - return Ok(Tool::new(c.clone())); + if let Some(c) = &self.compiler { + return Ok(Tool::new((**c).to_owned())); } let host = self.get_host()?; let target = self.get_target()?; @@ -2633,8 +2630,8 @@ impl Build { /// 3. Else the default is `libc++` for OS X and BSDs, `libc++_shared` for Android, /// `None` for MSVC and `libstdc++` for anything else. fn get_cpp_link_stdlib(&self) -> Result, Error> { - match self.cpp_link_stdlib.clone() { - Some(s) => Ok(s), + match &self.cpp_link_stdlib { + Some(s) => Ok(s.as_ref().map(|s| (*s).to_string())), None => { if let Ok(stdlib) = self.get_var("CXXSTDLIB") { if stdlib.is_empty() { @@ -2702,14 +2699,14 @@ impl Build { cmd.args(flags); for flag in &self.ar_flags { any_flags = true; - cmd.arg(flag); + cmd.arg(&**flag); } Ok((cmd, name, any_flags)) } fn get_base_archiver(&self) -> Result<(Command, String), Error> { if let Some(ref a) = self.archiver { - return Ok((self.cmd(a), a.to_string_lossy().into_owned())); + return Ok((self.cmd(&**a), a.to_string_lossy().into_owned())); } self.get_base_archiver_variant("AR", "ar") @@ -2748,7 +2745,7 @@ impl Build { fn get_base_ranlib(&self) -> Result { if let Some(ref r) = self.ranlib { - return Ok(self.cmd(r)); + return Ok(self.cmd(&**r)); } Ok(self.get_base_archiver_variant("RANLIB", "ranlib")?.0) @@ -3024,22 +3021,22 @@ impl Build { } fn get_target(&self) -> Result { - match self.target.clone() { - Some(t) => Ok(t), + match self.target { + Some(ref t) => Ok((*t).to_string()), None => Ok(self.getenv_unwrap("TARGET")?), } } fn get_host(&self) -> Result { - match self.host.clone() { - Some(h) => Ok(h), + match &self.host { + Some(h) => Ok((**h).to_string()), None => Ok(self.getenv_unwrap("HOST")?), } } fn get_opt_level(&self) -> Result { - match self.opt_level.as_ref().cloned() { - Some(ol) => Ok(ol), + match &self.opt_level { + Some(ol) => Ok((**ol).to_string()), None => Ok(self.getenv_unwrap("OPT_LEVEL")?), } } @@ -3075,8 +3072,8 @@ impl Build { } fn get_out_dir(&self) -> Result { - match self.out_dir.clone() { - Some(p) => Ok(p), + match &self.out_dir { + Some(p) => Ok((**p).into()), None => Ok(env::var_os("OUT_DIR").map(PathBuf::from).ok_or_else(|| { Error::new( ErrorKind::EnvVarNotFound, From 1997951dcc87d2bdbb7ce5870ab5e8650428081d Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Thu, 2 Feb 2023 14:29:22 -0600 Subject: [PATCH 061/138] Fix --target cflag on FreeBSD when compiling against clang (#785) Closes https://github.com/rust-lang/cc-rs/issues/463 --- src/lib.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index d31f66c24..a998f4951 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1720,6 +1720,23 @@ impl Build { } else if target.contains("aarch64") { cmd.args.push("--target=aarch64-unknown-windows-gnu".into()) } + } else if target.ends_with("-freebsd") && self.get_host()?.eq(target) { + // clang <= 13 on FreeBSD doesn't support a target triple without at least + // the major os version number appended; e.g. use x86_64-unknown-freebsd13 + // or x86_64-unknown-freebsd13.0 instead of x86_64-unknown-freebsd. + let stdout = std::process::Command::new("freebsd-version") + .output() + .map_err(|e| { + Error::new( + ErrorKind::ToolNotFound, + &format!("Error executing freebsd-version: {}", e), + ) + })? + .stdout; + let stdout = String::from_utf8_lossy(&stdout); + let os_ver = stdout.split('-').next().unwrap(); + + cmd.push_cc_arg(format!("--target={}{}", target, os_ver).into()); } else { cmd.push_cc_arg(format!("--target={}", target).into()); } From 0d9a0f84e43fdb3eef34ff9726f7cb915a2b619a Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sun, 5 Feb 2023 23:58:01 +0100 Subject: [PATCH 062/138] Append `freebsd-version` to all --target=*-freebsd if executed on FreeBSD. (#788) --- src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index a998f4951..d26ba67bd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1720,10 +1720,13 @@ impl Build { } else if target.contains("aarch64") { cmd.args.push("--target=aarch64-unknown-windows-gnu".into()) } - } else if target.ends_with("-freebsd") && self.get_host()?.eq(target) { + } else if target.ends_with("-freebsd") && self.get_host()?.ends_with("-freebsd") + { // clang <= 13 on FreeBSD doesn't support a target triple without at least // the major os version number appended; e.g. use x86_64-unknown-freebsd13 // or x86_64-unknown-freebsd13.0 instead of x86_64-unknown-freebsd. + // The current version is appended. If it doesn't align with your goals, pass + // .flag("--target=...") in the build script or adjust CXXFLAGS accordingly. let stdout = std::process::Command::new("freebsd-version") .output() .map_err(|e| { From 3e3f827a96aeb613956a3ba163324fcca74321c2 Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Thu, 9 Feb 2023 11:56:21 -0600 Subject: [PATCH 063/138] Reword `push_cc_arg()` doc comments. * Clarify that the flag is added after other arguments (although obvious from the function name), * Push extraneous info to the second paragraph to display nicer in IDE completions, * Reword the bits about nvcc and the `-Xcompiler` flag. --- src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d26ba67bd..0806a13c7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3277,11 +3277,12 @@ impl Tool { self.removed_args.push(flag); } - /// Add a flag, and optionally prepend the NVCC wrapper flag "-Xcompiler". + /// Push a flag to the end of the compiler's arguments list. /// - /// Currently this is only used for compiling CUDA sources, since NVCC only - /// accepts a limited set of GNU-like flags, and the rest must be prefixed - /// with a "-Xcompiler" flag to get passed to the underlying C++ compiler. + /// Currently `-Xcompiler` is inserted before the passed flag when compiling + /// CUDA sources since NVCC only accepts a limited set of GNU-like flags, + /// while the rest must be prefixed with the `-Xcompiler` flag to get passed + /// to the underlying C++ compiler. fn push_cc_arg(&mut self, flag: OsString) { if self.cuda { self.args.push("-Xcompiler".into()); From 0b68d59b464b7115d2dac2ed70fd0f093b5ebd8e Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Thu, 9 Feb 2023 15:01:24 -0600 Subject: [PATCH 064/138] Clarify implementation limitations of funcs in doc comment --- src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0806a13c7..688f476fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3290,6 +3290,9 @@ impl Tool { self.args.push(flag); } + /// Checks if an argument or flag has already been specified or conflicts. + /// + /// Currently only checks optimization flags. fn is_duplicate_opt_arg(&self, flag: &OsString) -> bool { let flag = flag.to_str().unwrap(); let mut chars = flag.chars(); @@ -3317,7 +3320,7 @@ impl Tool { return false; } - /// Don't push optimization arg if it conflicts with existing args + /// Don't push optimization arg if it conflicts with existing args. fn push_opt_unless_duplicate(&mut self, flag: OsString) { if self.is_duplicate_opt_arg(&flag) { println!("Info: Ignoring duplicate arg {:?}", &flag); From 3eb17abcf86f34746483fb4b6fe932672692dc02 Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Tue, 21 Mar 2023 17:57:31 -0500 Subject: [PATCH 065/138] Prevent cloning on every `get_xxx()` call (#793) --- src/lib.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 688f476fe..89f779e05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ #![allow(deprecated)] #![deny(missing_docs)] +use std::borrow::Cow; use std::collections::{hash_map, HashMap}; use std::env; use std::ffi::{OsStr, OsString}; @@ -3040,24 +3041,24 @@ impl Build { prefixes.first().map(|prefix| *prefix)) } - fn get_target(&self) -> Result { - match self.target { - Some(ref t) => Ok((*t).to_string()), - None => Ok(self.getenv_unwrap("TARGET")?), + fn get_target(&self) -> Result, Error> { + match &self.target { + Some(t) => Ok(Cow::Borrowed(&t)), + None => Ok(Cow::Owned(self.getenv_unwrap("TARGET")?)), } } - fn get_host(&self) -> Result { + fn get_host(&self) -> Result, Error> { match &self.host { - Some(h) => Ok((**h).to_string()), - None => Ok(self.getenv_unwrap("HOST")?), + Some(h) => Ok(Cow::Borrowed(&h)), + None => Ok(Cow::Owned(self.getenv_unwrap("HOST")?)), } } - fn get_opt_level(&self) -> Result { + fn get_opt_level(&self) -> Result, Error> { match &self.opt_level { - Some(ol) => Ok((**ol).to_string()), - None => Ok(self.getenv_unwrap("OPT_LEVEL")?), + Some(ol) => Ok(Cow::Borrowed(&ol)), + None => Ok(Cow::Owned(self.getenv_unwrap("OPT_LEVEL")?)), } } From c60f69a084269a5b2f96781449db01b2c23f35ed Mon Sep 17 00:00:00 2001 From: Mahmoud Al-Qudsi Date: Tue, 21 Mar 2023 17:57:43 -0500 Subject: [PATCH 066/138] Change approach for native and cross-compilation with clang++ on FreeBSD (#794) --- src/lib.rs | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 89f779e05..0a4427aa4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1721,26 +1721,30 @@ impl Build { } else if target.contains("aarch64") { cmd.args.push("--target=aarch64-unknown-windows-gnu".into()) } - } else if target.ends_with("-freebsd") && self.get_host()?.ends_with("-freebsd") - { - // clang <= 13 on FreeBSD doesn't support a target triple without at least - // the major os version number appended; e.g. use x86_64-unknown-freebsd13 - // or x86_64-unknown-freebsd13.0 instead of x86_64-unknown-freebsd. - // The current version is appended. If it doesn't align with your goals, pass - // .flag("--target=...") in the build script or adjust CXXFLAGS accordingly. - let stdout = std::process::Command::new("freebsd-version") - .output() - .map_err(|e| { - Error::new( - ErrorKind::ToolNotFound, - &format!("Error executing freebsd-version: {}", e), - ) - })? - .stdout; - let stdout = String::from_utf8_lossy(&stdout); - let os_ver = stdout.split('-').next().unwrap(); + } else if target.ends_with("-freebsd") { + // FreeBSD only supports C++11 and above when compiling against libc++ + // (available from FreeBSD 10 onwards). Under FreeBSD, clang uses libc++ by + // default on FreeBSD 10 and newer unless `--target` is manually passed to + // the compiler, in which case its default behavior differs: + // * If --target=xxx-unknown-freebsdX(.Y) is specified and X is greater than + // or equal to 10, clang++ uses libc++ + // * If --target=xxx-unknown-freebsd is specified (without a version), + // clang++ cannot assume libc++ is available and reverts to a default of + // libstdc++ (this behavior was changed in llvm 14). + // + // This breaks C++11 (or greater) builds if targeting FreeBSD with the + // generic xxx-unknown-freebsd triple on clang 13 or below *without* + // explicitly specifying that libc++ should be used. + // When cross-compiling, we can't infer from the rust/cargo target triple + // which major version of FreeBSD we are targeting, so we need to make sure + // that libc++ is used (unless the user has explicitly specified otherwise). + // There's no compelling reason to use a different approach when compiling + // natively. + if self.cpp && self.cpp_set_stdlib.is_none() { + cmd.push_cc_arg("-stdlib=libc++".into()); + } - cmd.push_cc_arg(format!("--target={}{}", target, os_ver).into()); + cmd.push_cc_arg(format!("--target={}", target).into()); } else { cmd.push_cc_arg(format!("--target={}", target).into()); } From 58e66f5beca3440c04a6a840bfe9e5bf60cf0dc5 Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Tue, 21 Mar 2023 23:58:57 +0100 Subject: [PATCH 067/138] Default to llvm-ar when compiling for wasm. (#657) --- src/lib.rs | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0a4427aa4..598bcb83f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1120,7 +1120,7 @@ impl Build { None => "none", }; if cudart != "none" { - if let Some(nvcc) = which(&self.get_compiler().path) { + if let Some(nvcc) = which(&self.get_compiler().path, None) { // Try to figure out the -L search path. If it fails, // it's on user to specify one by passing it through // RUSTFLAGS environment variable. @@ -2798,6 +2798,20 @@ impl Build { name = format!("em{}", tool); Some(self.cmd(&name)) } + } else if target.starts_with("wasm32") { + // Formally speaking one should be able to use this approach, + // parsing -print-search-dirs output, to cover all clang targets, + // including Android SDKs and other cross-compilation scenarios... + // And even extend it to gcc targets by seaching for "ar" instead + // of "llvm-ar"... + let compiler = self.get_base_compiler().ok()?; + if compiler.family == ToolFamily::Clang { + name = format!("llvm-{}", tool); + search_programs(&mut self.cmd(&compiler.path), &name) + .map(|name| self.cmd(&name)) + } else { + None + } } else { None } @@ -2824,10 +2838,10 @@ impl Build { // next to 'clang-cl' and use 'search_programs()' to locate // 'llvm-lib'. This is because 'clang-cl' doesn't support // the -print-search-dirs option. - if let Some(mut cmd) = which(&compiler.path) { + if let Some(mut cmd) = which(&compiler.path, None) { cmd.pop(); cmd.push("llvm-lib.exe"); - if let Some(llvm_lib) = which(&cmd) { + if let Some(llvm_lib) = which(&cmd, None) { lib = llvm_lib.to_str().unwrap().to_owned(); } } @@ -3684,7 +3698,7 @@ fn map_darwin_target_from_rust_to_compiler_architecture(target: &str) -> Option< } } -fn which(tool: &Path) -> Option { +fn which(tool: &Path, path_entries: Option) -> Option { fn check_exe(exe: &mut PathBuf) -> bool { let exe_ext = std::env::consts::EXE_EXTENSION; exe.exists() || (!exe_ext.is_empty() && exe.set_extension(exe_ext) && exe.exists()) @@ -3697,13 +3711,27 @@ fn which(tool: &Path) -> Option { } // Loop through PATH entries searching for the |tool|. - let path_entries = env::var_os("PATH")?; + let path_entries = path_entries.or(env::var_os("PATH"))?; env::split_paths(&path_entries).find_map(|path_entry| { let mut exe = path_entry.join(tool); return if check_exe(&mut exe) { Some(exe) } else { None }; }) } +// search for |prog| on 'programs' path in '|cc| -print-search-dirs' output +fn search_programs(cc: &mut Command, prog: &str) -> Option { + let search_dirs = run_output(cc.arg("-print-search-dirs"), "cc").ok()?; + // clang driver appears to be forcing UTF-8 output even on Windows, + // hence from_utf8 is assumed to be usable in all cases. + let search_dirs = std::str::from_utf8(&search_dirs).ok()?; + for dirs in search_dirs.split(|c| c == '\r' || c == '\n') { + if let Some(path) = dirs.strip_prefix("programs: =") { + return which(Path::new(prog), Some(OsString::from(path))); + } + } + None +} + #[derive(Clone, Copy, PartialEq)] enum AsmFileExt { /// `.asm` files. On MSVC targets, we assume these should be passed to MASM From 18a62b6894f426bddaef3b8165b74bf18c95bf0e Mon Sep 17 00:00:00 2001 From: zhaixiaojuan Date: Thu, 18 Nov 2021 17:44:13 +0800 Subject: [PATCH 068/138] Add loongarch64 support --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 598bcb83f..164ae1d27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2945,6 +2945,7 @@ impl Build { ]), // explicit None if not found, so caller knows to fall back "i686-unknown-linux-musl" => Some("musl"), "i686-unknown-netbsd" => Some("i486--netbsdelf"), + "loongarch64-unknown-linux-gnu" => Some("loongarch64-linux-gnu"), "mips-unknown-linux-gnu" => Some("mips-linux-gnu"), "mips-unknown-linux-musl" => Some("mips-linux-musl"), "mipsel-unknown-linux-gnu" => Some("mipsel-linux-gnu"), From ecc60ba25a5d1b2e80afd1b2211e2d7a8ef55762 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Wed, 22 Mar 2023 18:55:15 +0100 Subject: [PATCH 069/138] Refine CUDA documentation. Original wording gives an impression that user-provided flags are prefixed with -Xcompiler, which is not the case. -Xcompiler is added only to internally generated flags. --- src/lib.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 164ae1d27..c17fdbb94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -640,10 +640,12 @@ impl Build { /// Set CUDA C++ support. /// - /// Enabling CUDA will pass the detected C/C++ toolchain as an argument to - /// the CUDA compiler, NVCC. NVCC itself accepts some limited GNU-like args; - /// any other arguments for the C/C++ toolchain will be redirected using - /// "-Xcompiler" flags. + /// Enabling CUDA will invoke the CUDA compiler, NVCC. While NVCC accepts + /// the most common compiler flags, e.g. `-std=c++17`, some project-specific + /// flags might have to be prefixed with "-Xcompiler" flag, for example as + /// `.flag("-Xcompiler").flag("-fpermissive")`. See the documentation for + /// `nvcc`, the CUDA compiler driver, at https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/ + /// for more information. /// /// If enabled, this also implicitly enables C++ support. pub fn cuda(&mut self, cuda: bool) -> &mut Build { @@ -3297,12 +3299,14 @@ impl Tool { self.removed_args.push(flag); } - /// Push a flag to the end of the compiler's arguments list. + /// Push an "exotic" flag to the end of the compiler's arguments list. /// - /// Currently `-Xcompiler` is inserted before the passed flag when compiling - /// CUDA sources since NVCC only accepts a limited set of GNU-like flags, - /// while the rest must be prefixed with the `-Xcompiler` flag to get passed - /// to the underlying C++ compiler. + /// Nvidia compiler accepts only the most common compiler flags like `-D`, + /// `-I`, `-c`, etc. Options meant specifically for the underlying + /// host C++ compiler have to be prefixed with '-Xcompiler`. + /// [Another possible future application for this function is passing + /// clang-specific flags to clang-cl, which otherwise accepts only + /// MSVC-specific options.] fn push_cc_arg(&mut self, flag: OsString) { if self.cuda { self.args.push("-Xcompiler".into()); From c1bf1c8ee4f823a607837c68cca7988617ca4951 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Mon, 8 May 2023 10:09:11 +0800 Subject: [PATCH 070/138] Link against libc++ on AIX --- src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index d1bd3c667..4bb110162 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2680,6 +2680,8 @@ impl Build { Ok(Some("c++".to_string())) } else if target.contains("openbsd") { Ok(Some("c++".to_string())) + } else if target.contains("aix") { + Ok(Some("c++".to_string())) } else if target.contains("android") { Ok(Some("c++_shared".to_string())) } else { From ff45d42c0de626273202e2783cafd6305b555a2f Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 30 Jan 2023 19:32:52 +1100 Subject: [PATCH 071/138] Optimize `Build::compile_objects`: Only spawns one thread - Refactor: Extract new fn `Build::compile_object_inner` - Optimize away thread spawning in `Build::compile_objects` Instead of spawning a new thread just to manage the compilation task, we instead spawn the child and put it into a Vec and manage it in the current thread. - Add new optional dep os_pipe v1 - Optimize `Build::compile_objects`: Only spawn one `print` thread instead of one per `Child`. - Rm unused explicit lifetime in `Build::compile_objects` - Optimize `Build::compile_objects`: Do not wait on `child` in `KillOnDrop::drop` - Refactor: Create newtype `PrintThread` - Refactor `spawn`/`run`/`run_output` to use `PrintThread` and extract new fn `run_inner`. - Refactor `Build::compile_objects` to use `spawn` when feat `parallel` is enabled - Fix `spawn`: Ensure stderr is reset on panic - Refactor: Extract new fn `wait_on_child` - Rename `Build::compile_object_inner` => `create_compile_object_cmd` Signed-off-by: Jiahao XU --- Cargo.toml | 1 + src/lib.rs | 252 ++++++++++++++++++++++++++++------------------------- 2 files changed, 135 insertions(+), 118 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3128f0367..3117b8e21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ edition = "2018" [dependencies] jobserver = { version = "0.1.16", optional = true } +os_pipe = "1" [features] parallel = ["jobserver"] diff --git a/src/lib.rs b/src/lib.rs index 4bb110162..688adbd8e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1218,8 +1218,7 @@ impl Build { } #[cfg(feature = "parallel")] - fn compile_objects<'me>(&'me self, objs: &[Object]) -> Result<(), Error> { - use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; + fn compile_objects(&self, objs: &[Object]) -> Result<(), Error> { use std::sync::Once; // Limit our parallelism globally with a jobserver. Start off by @@ -1242,56 +1241,28 @@ impl Build { // Note that this jobserver is cached globally so we only used one per // process and only worry about creating it once. // - // * Next we use a raw `thread::spawn` per thread to actually compile - // objects in parallel. We only actually spawn a thread after we've - // acquired a token to perform some work - // - // * Finally though we want to keep the dependencies of this crate - // pretty light, so we avoid using a safe abstraction like `rayon` and - // instead rely on some bits of `unsafe` code. We know that this stack - // frame persists while everything is compiling so we use all the - // stack-allocated objects without cloning/reallocating. We use a - // transmute to `State` with a `'static` lifetime to persist - // everything we need across the boundary, and the join-on-drop - // semantics of `JoinOnDrop` should ensure that our stack frame is - // alive while threads are alive. + // * Next we use spawn the process to actually compile objects in + // parallel after we've acquired a token to perform some work // // With all that in mind we compile all objects in a loop here, after we // acquire the appropriate tokens, Once all objects have been compiled - // we join on all the threads and propagate the results of compilation. - // - // Note that as a slight optimization we try to break out as soon as - // possible as soon as any compilation fails to ensure that errors get - // out to the user as fast as possible. - let error = AtomicBool::new(false); - let mut threads = Vec::new(); - for obj in objs { - if error.load(SeqCst) { - break; - } - let token = server.acquire()?; - let state = State { - build: self, - obj, - error: &error, - }; - let state = unsafe { std::mem::transmute::>(state) }; - let thread = thread::spawn(|| { - let state: State<'me> = state; // erase the `'static` lifetime - let result = state.build.compile_object(state.obj); - if result.is_err() { - state.error.store(true, SeqCst); - } - drop(token); // make sure our jobserver token is released after the compile - return result; - }); - threads.push(JoinOnDrop(Some(thread))); - } + // we wait on all the processes and propagate the results of compilation. + let print = PrintThread::new()?; - for mut thread in threads { - if let Some(thread) = thread.0.take() { - thread.join().expect("thread should not panic")?; - } + let children = objs + .iter() + .map(|obj| { + let (mut cmd, program) = self.create_compile_object_cmd(obj)?; + let token = server.acquire()?; + + let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; + + Ok((cmd, program, KillOnDrop(child), token)) + }) + .collect::, Error>>()?; + + for (cmd, program, mut child, _token) in children { + wait_on_child(&cmd, &program, &mut child.0)?; } // Reacquire our process's token before we proceed, which we released @@ -1302,16 +1273,6 @@ impl Build { return Ok(()); - /// Shared state from the parent thread to the child thread. This - /// package of pointers is temporarily transmuted to a `'static` - /// lifetime to cross the thread boundary and then once the thread is - /// running we erase the `'static` to go back to an anonymous lifetime. - struct State<'a> { - build: &'a Build, - obj: &'a Object, - error: &'a AtomicBool, - } - /// Returns a suitable `jobserver::Client` used to coordinate /// parallelism between build scripts. fn jobserver() -> &'static jobserver::Client { @@ -1357,26 +1318,30 @@ impl Build { return client; } - struct JoinOnDrop(Option>>); + struct KillOnDrop(Child); - impl Drop for JoinOnDrop { + impl Drop for KillOnDrop { fn drop(&mut self) { - if let Some(thread) = self.0.take() { - drop(thread.join()); - } + let child = &mut self.0; + + child.kill().ok(); } } } #[cfg(not(feature = "parallel"))] fn compile_objects(&self, objs: &[Object]) -> Result<(), Error> { + let print = PrintThread::new()?; + for obj in objs { - self.compile_object(obj)?; + let (mut cmd, name) = self.create_compile_object_cmd(obj)?; + run_inner(&mut cmd, &name, print.pipe_writer_cloned()?.unwrap())?; } + Ok(()) } - fn compile_object(&self, obj: &Object) -> Result<(), Error> { + fn create_compile_object_cmd(&self, obj: &Object) -> Result<(Command, String), Error> { let asm_ext = AsmFileExt::from_path(&obj.src); let is_asm = asm_ext.is_some(); let target = self.get_target()?; @@ -1425,8 +1390,7 @@ impl Build { self.fix_env_for_apple_os(&mut cmd)?; } - run(&mut cmd, &name)?; - Ok(()) + Ok((cmd, name)) } /// This will return a result instead of panicing; see expand() for the complete description. @@ -3463,21 +3427,19 @@ impl Tool { } } -fn run(cmd: &mut Command, program: &str) -> Result<(), Error> { - let (mut child, print) = spawn(cmd, program)?; +fn wait_on_child(cmd: &Command, program: &str, child: &mut Child) -> Result<(), Error> { let status = match child.wait() { Ok(s) => s, - Err(_) => { + Err(e) => { return Err(Error::new( ErrorKind::ToolExecError, &format!( - "Failed to wait on spawned child process, command {:?} with args {:?}.", - cmd, program + "Failed to wait on spawned child process, command {:?} with args {:?}: {}.", + cmd, program, e ), )); } }; - print.join().unwrap(); println!("{}", status); if status.success() { @@ -3493,9 +3455,28 @@ fn run(cmd: &mut Command, program: &str) -> Result<(), Error> { } } +fn run_inner( + cmd: &mut Command, + program: &str, + pipe_writer: os_pipe::PipeWriter, +) -> Result<(), Error> { + let mut child = spawn(cmd, program, pipe_writer)?; + wait_on_child(cmd, program, &mut child) +} + +fn run(cmd: &mut Command, program: &str) -> Result<(), Error> { + let mut print = PrintThread::new()?; + run_inner(cmd, program, print.pipe_writer().take().unwrap())?; + + Ok(()) +} + fn run_output(cmd: &mut Command, program: &str) -> Result, Error> { cmd.stdout(Stdio::piped()); - let (mut child, print) = spawn(cmd, program)?; + + let mut print = PrintThread::new()?; + let mut child = spawn(cmd, program, print.pipe_writer().take().unwrap())?; + let mut stdout = vec![]; child .stdout @@ -3503,53 +3484,33 @@ fn run_output(cmd: &mut Command, program: &str) -> Result, Error> { .unwrap() .read_to_end(&mut stdout) .unwrap(); - let status = match child.wait() { - Ok(s) => s, - Err(_) => { - return Err(Error::new( - ErrorKind::ToolExecError, - &format!( - "Failed to wait on spawned child process, command {:?} with args {:?}.", - cmd, program - ), - )); - } - }; - print.join().unwrap(); - println!("{}", status); - if status.success() { - Ok(stdout) - } else { - Err(Error::new( - ErrorKind::ToolExecError, - &format!( - "Command {:?} with args {:?} did not execute successfully (status code {}).", - cmd, program, status - ), - )) - } + wait_on_child(cmd, program, &mut child)?; + + Ok(stdout) } -fn spawn(cmd: &mut Command, program: &str) -> Result<(Child, JoinHandle<()>), Error> { - println!("running: {:?}", cmd); +fn spawn( + cmd: &mut Command, + program: &str, + pipe_writer: os_pipe::PipeWriter, +) -> Result { + struct ResetStderr<'cmd>(&'cmd mut Command); - // Capture the standard error coming from these programs, and write it out - // with cargo:warning= prefixes. Note that this is a bit wonky to avoid - // requiring the output to be UTF-8, we instead just ship bytes from one - // location to another. - match cmd.stderr(Stdio::piped()).spawn() { - Ok(mut child) => { - let stderr = BufReader::new(child.stderr.take().unwrap()); - let print = thread::spawn(move || { - for line in stderr.split(b'\n').filter_map(|l| l.ok()) { - print!("cargo:warning="); - std::io::stdout().write_all(&line).unwrap(); - println!(""); - } - }); - Ok((child, print)) + impl Drop for ResetStderr<'_> { + fn drop(&mut self) { + // Reset stderr to default to release pipe_writer so that print thread will + // not block forever. + self.0.stderr(Stdio::inherit()); } + } + + println!("running: {:?}", cmd); + + let cmd = ResetStderr(cmd); + + match cmd.0.stderr(pipe_writer).spawn() { + Ok(child) => Ok(child), Err(ref e) if e.kind() == io::ErrorKind::NotFound => { let extra = if cfg!(windows) { " (see https://github.com/rust-lang/cc-rs#compile-time-requirements \ @@ -3562,11 +3523,11 @@ fn spawn(cmd: &mut Command, program: &str) -> Result<(Child, JoinHandle<()>), Er &format!("Failed to find tool. Is `{}` installed?{}", program, extra), )) } - Err(ref e) => Err(Error::new( + Err(e) => Err(Error::new( ErrorKind::ToolExecError, &format!( "Command {:?} with args {:?} failed to start: {:?}", - cmd, program, e + cmd.0, program, e ), )), } @@ -3767,3 +3728,58 @@ impl AsmFileExt { None } } + +struct PrintThread { + handle: Option>, + pipe_writer: Option, +} + +impl PrintThread { + fn new() -> Result { + let (pipe_reader, pipe_writer) = os_pipe::pipe()?; + + // Capture the standard error coming from compilation, and write it out + // with cargo:warning= prefixes. Note that this is a bit wonky to avoid + // requiring the output to be UTF-8, we instead just ship bytes from one + // location to another. + let print = thread::spawn(move || { + let mut stderr = BufReader::with_capacity(4096, pipe_reader); + let mut line = String::with_capacity(20); + let mut stdout = io::stdout(); + + // read_line returns 0 on Eof + while stderr.read_line(&mut line).unwrap() != 0 { + writeln!(&mut stdout, "cargo:warning={}", line).ok(); + + // read_line does not clear the buffer + line.clear(); + } + }); + + Ok(Self { + handle: Some(print), + pipe_writer: Some(pipe_writer), + }) + } + + fn pipe_writer(&mut self) -> &mut Option { + &mut self.pipe_writer + } + + fn pipe_writer_cloned(&self) -> Result, Error> { + self.pipe_writer + .as_ref() + .map(os_pipe::PipeWriter::try_clone) + .transpose() + .map_err(From::from) + } +} + +impl Drop for PrintThread { + fn drop(&mut self) { + // Drop pipe_writer first to avoid deadlock + self.pipe_writer.take(); + + self.handle.take().unwrap().join().unwrap(); + } +} From ced2360753fa0c0023f32cfda8afe8fd4001923f Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 14 May 2023 21:34:46 +1000 Subject: [PATCH 072/138] Speedup CI - Disable incremental build since it's always build from scratch - Use crates.io sparse registry - Use `actions/checkout@v3` instead of the master branch - Use `rustup toolchain install` instead of `rustup-update`, plus also pass `--profile minimal` for minimum installation. - Rm unnecessary `cargo-build` step in job test - Run `cargo-check` instead of `cargo-b` in job msrv Signed-off-by: Jiahao XU --- .github/workflows/main.yml | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ceb9e539a..f85288d4d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,10 @@ name: CI on: [push, pull_request] +env: + CARGO_INCREMENTAL: 0 + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + jobs: test: name: Test @@ -64,15 +68,13 @@ jobs: rust: stable-x86_64 target: x86_64-pc-windows-msvc steps: - - uses: actions/checkout@master - - name: Update Rustup (temporary workaround) - run: rustup self update - shell: bash - if: startsWith(matrix.os, 'windows') + - uses: actions/checkout@v3 - name: Install Rust (rustup) - run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }} + run: | + set -euxo pipefail + rustup toolchain install ${{ matrix.rust }} --no-self-update --profile minimal --target ${{ matrix.target }} + rustup default ${{ matrix.rust }} shell: bash - - run: rustup target add ${{ matrix.target }} - name: Install g++-multilib run: | set -e @@ -84,7 +86,6 @@ jobs: sudo apt-get update sudo apt-get install g++-multilib if: matrix.build == 'linux32' - - run: cargo build - run: cargo test ${{ matrix.no_run }} - run: cargo test ${{ matrix.no_run }} --features parallel - run: cargo test ${{ matrix.no_run }} --manifest-path cc-test/Cargo.toml --target ${{ matrix.target }} @@ -95,7 +96,7 @@ jobs: name: Test CUDA support runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Install cuda-minimal-build-11-8 shell: bash run: | @@ -106,7 +107,8 @@ jobs: sudo apt-get -y install cuda-minimal-build-11-8 - name: Test 'cudart' feature shell: bash - run: env PATH=/usr/local/cuda/bin:$PATH cargo test --manifest-path cc-test/Cargo.toml --features test_cuda + run: | + PATH="/usr/local/cuda/bin:$PATH" cargo test --manifest-path cc-test/Cargo.toml --features test_cuda msrv: name: MSRV @@ -115,17 +117,22 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Install Rust - run: rustup update 1.46.0 --no-self-update && rustup default 1.46.0 + run: | + rustup toolchain install 1.46.0 --no-self-update --profile minimal + rustup default 1.46.0 shell: bash - - run: cargo build + - run: cargo check --lib rustfmt: name: Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Install Rust - run: rustup update stable && rustup default stable && rustup component add rustfmt + run: | + rustup toolchain install stable --no-self-update --profile minimal --component rustfmt + rustup default stable + shell: bash - run: cargo fmt -- --check From d2918f19220a5c9032444b5c8c033a2c80109277 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 14 May 2023 22:06:08 +1000 Subject: [PATCH 073/138] Enable dependabot for Github Action Workflows Signed-off-by: Jiahao XU --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7377d3759..e211b1888 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,11 @@ version: 2 updates: +- package-ecosystem: "github-actions" + # Workflow files stored in the + # default location of `.github/workflows` + directory: "/" + schedule: + interval: "daily" - package-ecosystem: cargo directory: "/" schedule: From 57853c4bf8a89a0f4c9137eb367ac580305c6919 Mon Sep 17 00:00:00 2001 From: James Farrell Date: Wed, 17 May 2023 19:14:29 +0000 Subject: [PATCH 074/138] Be more stringent about handling Android NDK. Wrapper scripts for the NDK have the format -, so when checking to see if the filename contains "android" (which will be part of the target), split off the target part of the filename and only check that, instead of checking the entire filename. This is a very minor improvement, but it is more correct and does have a practical benefit for us on the Android toolchain team: Our build process for rustc uses wrapper scripts that include the target in their name, and the cc crate is misidentifying them as being from the NDK and causing our Windows build to fail without a workaround. --- src/lib.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4bb110162..428b6f9c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3616,13 +3616,12 @@ static NEW_STANDALONE_ANDROID_COMPILERS: [&str; 4] = [ fn android_clang_compiler_uses_target_arg_internally(clang_path: &Path) -> bool { if let Some(filename) = clang_path.file_name() { if let Some(filename_str) = filename.to_str() { - filename_str.contains("android") - } else { - false + if let Some(idx) = filename_str.rfind("-") { + return filename_str.split_at(idx).0.contains("android"); + } } - } else { - false } + false } #[test] @@ -3635,6 +3634,9 @@ fn test_android_clang_compiler_uses_target_arg_internally() { &PathBuf::from(format!("armv7a-linux-androideabi{}-clang++", version)) )); } + assert!(!android_clang_compiler_uses_target_arg_internally( + &PathBuf::from("clang-i686-linux-android") + )); assert!(!android_clang_compiler_uses_target_arg_internally( &PathBuf::from("clang") )); From d2a6839d06b6db6bae7bdb103da92ce06500bb98 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Mon, 17 Jul 2023 11:24:40 +0300 Subject: [PATCH 075/138] add a function to populate flags from an arbitrary environment variable (#818) --- README.md | 12 +++++- src/lib.rs | 112 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 83 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 863540d2d..736f8e1d2 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,17 @@ number of different environment variables. certain `TARGET`s, it also is assumed to know about other flags (most common is `-fPIC`). * `AR` - the `ar` (archiver) executable to use to build the static library. -* `CRATE_CC_NO_DEFAULTS` - the default compiler flags may cause conflicts in some cross compiling scenarios. Setting this variable will disable the generation of default compiler flags. +* `CRATE_CC_NO_DEFAULTS` - the default compiler flags may cause conflicts in + some cross compiling scenarios. Setting this variable + will disable the generation of default compiler + flags. * `CXX...` - see [C++ Support](#c-support). +Furthermore, projects using this crate may specify custom environment variables +to be inspected, for example via the `Build::try_flags_from_environment` +function. Consult the project’s own documentation or its use of the `cc` crate +for any additional variables it may use. + Each of these variables can also be supplied with certain prefixes and suffixes, in the following prioritized order: @@ -94,7 +102,7 @@ in the following prioritized order: 3. `_` - for example, `HOST_CC` or `TARGET_CFLAGS` 4. `` - a plain `CC`, `AR` as above. -If none of these variables exist, cc-rs uses built-in defaults +If none of these variables exist, cc-rs uses built-in defaults. In addition to the above optional environment variables, `cc-rs` has some functions with hard requirements on some variables supplied by [cargo's diff --git a/src/lib.rs b/src/lib.rs index e295e9714..a4ba8457e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -563,6 +563,35 @@ impl Build { self } + /// Add flags from the specified environment variable. + /// + /// Normally the `cc` crate will consult with the standard set of environment + /// variables (such as `CFLAGS` and `CXXFLAGS`) to construct the compiler invocation. Use of + /// this method provides additional levers for the end user to use when configuring the build + /// process. + /// + /// Just like the standard variables, this method will search for an environment variable with + /// appropriate target prefixes, when appropriate. + /// + /// # Examples + /// + /// This method is particularly beneficial in introducing the ability to specify crate-specific + /// flags. + /// + /// ```no_run + /// cc::Build::new() + /// .file("src/foo.c") + /// .try_flags_from_environment(concat!(env!("CARGO_PKG_NAME"), "_CFLAGS")) + /// .expect("the environment variable must be specified and UTF-8") + /// .compile("foo"); + /// ``` + /// + pub fn try_flags_from_environment(&mut self, environ_key: &str) -> Result<&mut Build, Error> { + let flags = self.envflags(environ_key)?; + self.flags.extend(flags.into_iter().map(Into::into)); + Ok(self) + } + /// Set the `-shared` flag. /// /// When enabled, the compiler will produce a shared object which can @@ -1471,7 +1500,6 @@ impl Build { let target = self.get_target()?; let mut cmd = self.get_base_compiler()?; - let envflags = self.envflags(if self.cpp { "CXXFLAGS" } else { "CFLAGS" }); // Disable default flag generation via `no_default_flags` or environment variable let no_defaults = self.no_default_flags || self.getenv("CRATE_CC_NO_DEFAULTS").is_some(); @@ -1482,8 +1510,10 @@ impl Build { println!("Info: default compiler flags are disabled"); } - for arg in envflags { - cmd.push_cc_arg(arg.into()); + if let Ok(flags) = self.envflags(if self.cpp { "CXXFLAGS" } else { "CFLAGS" }) { + for arg in flags { + cmd.push_cc_arg(arg.into()); + } } for directory in self.include_directories.iter() { @@ -1990,7 +2020,7 @@ impl Build { fn has_flags(&self) -> bool { let flags_env_var_name = if self.cpp { "CXXFLAGS" } else { "CFLAGS" }; - let flags_env_var_value = self.get_var(flags_env_var_name); + let flags_env_var_value = self.getenv_with_target_prefixes(flags_env_var_name); if let Ok(_) = flags_env_var_value { true } else { @@ -2437,7 +2467,7 @@ impl Build { tool.args.is_empty(), "CUDA compilation currently assumes empty pre-existing args" ); - let nvcc = match self.get_var("NVCC") { + let nvcc = match self.getenv_with_target_prefixes("NVCC") { Err(_) => "nvcc".into(), Ok(nvcc) => nvcc, }; @@ -2509,34 +2539,6 @@ impl Build { Ok(tool) } - fn get_var(&self, var_base: &str) -> Result { - let target = self.get_target()?; - let host = self.get_host()?; - let kind = if host == target { "HOST" } else { "TARGET" }; - let target_u = target.replace("-", "_"); - let res = self - .getenv(&format!("{}_{}", var_base, target)) - .or_else(|| self.getenv(&format!("{}_{}", var_base, target_u))) - .or_else(|| self.getenv(&format!("{}_{}", kind, var_base))) - .or_else(|| self.getenv(var_base)); - - match res { - Some(res) => Ok(res), - None => Err(Error::new( - ErrorKind::EnvVarNotFound, - &format!("Could not find environment variable {}.", var_base), - )), - } - } - - fn envflags(&self, name: &str) -> Vec { - self.get_var(name) - .unwrap_or(String::new()) - .split_ascii_whitespace() - .map(|slice| slice.to_string()) - .collect() - } - /// Returns a fallback `cc_compiler_wrapper` by introspecting `RUSTC_WRAPPER` fn rustc_wrapper_fallback() -> Option { // No explicit CC wrapper was detected, but check if RUSTC_WRAPPER @@ -2557,7 +2559,7 @@ impl Build { /// Returns compiler path, optional modifier name from whitelist, and arguments vec fn env_tool(&self, name: &str) -> Option<(String, Option, Vec)> { - let tool = match self.get_var(name) { + let tool = match self.getenv_with_target_prefixes(name) { Ok(tool) => tool, Err(_) => return None, }; @@ -2628,7 +2630,7 @@ impl Build { match &self.cpp_link_stdlib { Some(s) => Ok(s.as_ref().map(|s| (*s).to_string())), None => { - if let Ok(stdlib) = self.get_var("CXXSTDLIB") { + if let Ok(stdlib) = self.getenv_with_target_prefixes("CXXSTDLIB") { if stdlib.is_empty() { Ok(None) } else { @@ -2691,9 +2693,11 @@ impl Build { fn try_get_archiver_and_flags(&self) -> Result<(Command, String, bool), Error> { let (mut cmd, name) = self.get_base_archiver()?; - let flags = self.envflags("ARFLAGS"); - let mut any_flags = !flags.is_empty(); - cmd.args(flags); + let mut any_flags = false; + if let Ok(flags) = self.envflags("ARFLAGS") { + any_flags = any_flags | !flags.is_empty(); + cmd.args(flags); + } for flag in &self.ar_flags { any_flags = true; cmd.arg(&**flag); @@ -2736,7 +2740,9 @@ impl Build { /// see [`get_ranlib()`] for the complete description. pub fn try_get_ranlib(&self) -> Result { let mut cmd = self.get_base_ranlib()?; - cmd.args(self.envflags("RANLIBFLAGS")); + if let Ok(flags) = self.envflags("RANLIBFLAGS") { + cmd.args(flags); + } Ok(cmd) } @@ -3133,6 +3139,34 @@ impl Build { } } + fn getenv_with_target_prefixes(&self, var_base: &str) -> Result { + let target = self.get_target()?; + let host = self.get_host()?; + let kind = if host == target { "HOST" } else { "TARGET" }; + let target_u = target.replace("-", "_"); + let res = self + .getenv(&format!("{}_{}", var_base, target)) + .or_else(|| self.getenv(&format!("{}_{}", var_base, target_u))) + .or_else(|| self.getenv(&format!("{}_{}", kind, var_base))) + .or_else(|| self.getenv(var_base)); + + match res { + Some(res) => Ok(res), + None => Err(Error::new( + ErrorKind::EnvVarNotFound, + &format!("Could not find environment variable {}.", var_base), + )), + } + } + + fn envflags(&self, name: &str) -> Result, Error> { + Ok(self + .getenv_with_target_prefixes(name)? + .split_ascii_whitespace() + .map(|slice| slice.to_string()) + .collect()) + } + fn print(&self, s: &str) { if self.cargo_metadata { println!("{}", s); From 163eb8e43baa986ec727e1cb51a2df8dd06bd856 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 17 Jul 2023 02:39:09 -0600 Subject: [PATCH 076/138] Use gnu -o flag for obj out path instead of -Fo when using gcc & g++ on Windows (#820) --- src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a4ba8457e..262424287 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -522,12 +522,14 @@ impl Build { let mut cmd = compiler.to_command(); let is_arm = target.contains("aarch64") || target.contains("arm"); let clang = compiler.family == ToolFamily::Clang; + let gnu = compiler.family == ToolFamily::Gnu; command_add_output_file( &mut cmd, &obj, self.cuda, target.contains("msvc"), clang, + gnu, false, is_arm, ); @@ -1377,6 +1379,7 @@ impl Build { let msvc = target.contains("msvc"); let compiler = self.try_get_compiler()?; let clang = compiler.family == ToolFamily::Clang; + let gnu = compiler.family == ToolFamily::Gnu; let (mut cmd, name) = if msvc && asm_ext == Some(AsmFileExt::DotAsm) { self.msvc_macro_assembler()? @@ -1396,7 +1399,9 @@ impl Build { ) }; let is_arm = target.contains("aarch64") || target.contains("arm"); - command_add_output_file(&mut cmd, &obj.dst, self.cuda, msvc, clang, is_asm, is_arm); + command_add_output_file( + &mut cmd, &obj.dst, self.cuda, msvc, clang, gnu, is_asm, is_arm, + ); // armasm and armasm64 don't requrie -c option if !msvc || !is_asm || !is_arm { cmd.arg("-c"); @@ -3578,10 +3583,11 @@ fn command_add_output_file( cuda: bool, msvc: bool, clang: bool, + gnu: bool, is_asm: bool, is_arm: bool, ) { - if msvc && !clang && !cuda && !(is_asm && is_arm) { + if msvc && !clang && !gnu && !cuda && !(is_asm && is_arm) { let mut s = OsString::from("-Fo"); s.push(&dst); cmd.arg(s); From b030a290be3b697de62e2d72687bbf1bdbd2e460 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 20 Jul 2023 14:32:46 +1000 Subject: [PATCH 077/138] Vendor dependency `os_pipe` (#822) Co-authored-by: Josh Triplett --- Cargo.toml | 7 ++- src/lib.rs | 24 +++----- src/os_pipe.rs | 28 ++++++++++ src/os_pipe/unix.rs | 121 +++++++++++++++++++++++++++++++++++++++++ src/os_pipe/windows.rs | 24 ++++++++ 5 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 src/os_pipe.rs create mode 100644 src/os_pipe/unix.rs create mode 100644 src/os_pipe/windows.rs diff --git a/Cargo.toml b/Cargo.toml index 3117b8e21..4158cd277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,12 @@ edition = "2018" [dependencies] jobserver = { version = "0.1.16", optional = true } -os_pipe = "1" + +[target.'cfg(unix)'.dependencies] +libc = "0.2.62" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_System_Pipes", "Win32_Security"] } [features] parallel = ["jobserver"] diff --git a/src/lib.rs b/src/lib.rs index 262424287..0532e5287 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,7 +61,7 @@ use std::collections::{hash_map, HashMap}; use std::env; use std::ffi::{OsStr, OsString}; use std::fmt::{self, Display, Formatter}; -use std::fs; +use std::fs::{self, File}; use std::hash::Hasher; use std::io::{self, BufRead, BufReader, Read, Write}; use std::path::{Component, Path, PathBuf}; @@ -69,6 +69,8 @@ use std::process::{Child, Command, Stdio}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; +mod os_pipe; + // These modules are all glue to support reading the MSVC version from // the registry and from COM interfaces #[cfg(windows)] @@ -3494,11 +3496,7 @@ fn wait_on_child(cmd: &Command, program: &str, child: &mut Child) -> Result<(), } } -fn run_inner( - cmd: &mut Command, - program: &str, - pipe_writer: os_pipe::PipeWriter, -) -> Result<(), Error> { +fn run_inner(cmd: &mut Command, program: &str, pipe_writer: File) -> Result<(), Error> { let mut child = spawn(cmd, program, pipe_writer)?; wait_on_child(cmd, program, &mut child) } @@ -3529,11 +3527,7 @@ fn run_output(cmd: &mut Command, program: &str) -> Result, Error> { Ok(stdout) } -fn spawn( - cmd: &mut Command, - program: &str, - pipe_writer: os_pipe::PipeWriter, -) -> Result { +fn spawn(cmd: &mut Command, program: &str, pipe_writer: File) -> Result { struct ResetStderr<'cmd>(&'cmd mut Command); impl Drop for ResetStderr<'_> { @@ -3773,7 +3767,7 @@ impl AsmFileExt { struct PrintThread { handle: Option>, - pipe_writer: Option, + pipe_writer: Option, } impl PrintThread { @@ -3804,14 +3798,14 @@ impl PrintThread { }) } - fn pipe_writer(&mut self) -> &mut Option { + fn pipe_writer(&mut self) -> &mut Option { &mut self.pipe_writer } - fn pipe_writer_cloned(&self) -> Result, Error> { + fn pipe_writer_cloned(&self) -> Result, Error> { self.pipe_writer .as_ref() - .map(os_pipe::PipeWriter::try_clone) + .map(File::try_clone) .transpose() .map_err(From::from) } diff --git a/src/os_pipe.rs b/src/os_pipe.rs new file mode 100644 index 000000000..218c9bc9f --- /dev/null +++ b/src/os_pipe.rs @@ -0,0 +1,28 @@ +//! Adapted from: +//! - https://doc.rust-lang.org/src/std/sys/unix/pipe.rs.html +//! - https://doc.rust-lang.org/src/std/sys/unix/fd.rs.html#385 +//! - https://github.com/rust-lang/rust/blob/master/library/std/src/sys/mod.rs#L57 +//! - https://github.com/oconnor663/os_pipe.rs +use std::fs::File; + +/// Open a new pipe and return a pair of [`File`] objects for the reader and writer. +/// +/// This corresponds to the `pipe2` library call on Posix and the +/// `CreatePipe` library call on Windows (though these implementation +/// details might change). These pipes are non-inheritable, so new child +/// processes won't receive a copy of them unless they're explicitly +/// passed as stdin/stdout/stderr. +pub fn pipe() -> std::io::Result<(File, File)> { + sys::pipe() +} + +#[cfg(unix)] +#[path = "os_pipe/unix.rs"] +mod sys; + +#[cfg(windows)] +#[path = "os_pipe/windows.rs"] +mod sys; + +#[cfg(all(not(unix), not(windows)))] +compile_error!("Only unix and windows support os_pipe!"); diff --git a/src/os_pipe/unix.rs b/src/os_pipe/unix.rs new file mode 100644 index 000000000..ec4e547f0 --- /dev/null +++ b/src/os_pipe/unix.rs @@ -0,0 +1,121 @@ +use std::{ + fs::File, + io, + os::{raw::c_int, unix::io::FromRawFd}, +}; + +pub(super) fn pipe() -> io::Result<(File, File)> { + let mut fds = [0; 2]; + + // The only known way right now to create atomically set the CLOEXEC flag is + // to use the `pipe2` syscall. This was added to Linux in 2.6.27, glibc 2.9 + // and musl 0.9.3, and some other targets also have it. + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + { + unsafe { + cvt(libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC))?; + } + } + + #[cfg(not(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] + { + unsafe { + cvt(libc::pipe(fds.as_mut_ptr()))?; + } + + cloexec::set_cloexec(fds[0])?; + cloexec::set_cloexec(fds[1])?; + } + + unsafe { Ok((File::from_raw_fd(fds[0]), File::from_raw_fd(fds[1]))) } +} + +fn cvt(t: c_int) -> io::Result { + if t == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(t) + } +} + +#[cfg(not(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" +)))] +mod cloexec { + use super::{c_int, cvt, io}; + + #[cfg(not(any( + target_env = "newlib", + target_os = "solaris", + target_os = "illumos", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "l4re", + target_os = "linux", + target_os = "haiku", + target_os = "redox", + target_os = "vxworks", + target_os = "nto", + )))] + pub(super) fn set_cloexec(fd: c_int) -> io::Result<()> { + unsafe { + cvt(libc::ioctl(fd, libc::FIOCLEX))?; + } + + Ok(()) + } + + #[cfg(any( + all( + target_env = "newlib", + not(any(target_os = "espidf", target_os = "horizon")) + ), + target_os = "solaris", + target_os = "illumos", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "l4re", + target_os = "linux", + target_os = "haiku", + target_os = "redox", + target_os = "vxworks", + target_os = "nto", + ))] + pub(super) fn set_cloexec(fd: c_int) -> io::Result<()> { + unsafe { + let previous = cvt(libc::fcntl(fd, libc::F_GETFD))?; + let new = previous | libc::FD_CLOEXEC; + if new != previous { + cvt(libc::fcntl(fd, libc::F_SETFD, new))?; + } + } + + Ok(()) + } + + // FD_CLOEXEC is not supported in ESP-IDF and Horizon OS but there's no need to, + // because neither supports spawning processes. + #[cfg(any(target_os = "espidf", target_os = "horizon"))] + pub(super) fn set_cloexec(_fd: c_int) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/os_pipe/windows.rs b/src/os_pipe/windows.rs new file mode 100644 index 000000000..4f7a57090 --- /dev/null +++ b/src/os_pipe/windows.rs @@ -0,0 +1,24 @@ +use std::{fs::File, io, os::windows::prelude::*, ptr}; +use windows_sys::Win32::{Foundation::INVALID_HANDLE_VALUE, System::Pipes::CreatePipe}; + +/// NOTE: These pipes do not support IOCP. +/// +/// If IOCP is needed, then you might want to emulate +/// anonymous pipes with CreateNamedPipe, as Rust's stdlib does. +pub(super) fn pipe() -> io::Result<(File, File)> { + let mut read_pipe = INVALID_HANDLE_VALUE; + let mut write_pipe = INVALID_HANDLE_VALUE; + + let ret = unsafe { CreatePipe(&mut read_pipe, &mut write_pipe, ptr::null_mut(), 0) }; + + if ret == 0 { + Err(io::Error::last_os_error()) + } else { + unsafe { + Ok(( + File::from_raw_handle(read_pipe as RawHandle), + File::from_raw_handle(write_pipe as RawHandle), + )) + } + } +} From 803cf9c5e107960a02c250a6f4826f6b9eda224d Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 20 Jul 2023 14:33:38 +1000 Subject: [PATCH 078/138] Optimize `Build::print`: Avoid unnecessary heap alloc (#824) --- src/lib.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0532e5287..e8046eab4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1132,7 +1132,7 @@ impl Build { }); if let Some(atlmfc_lib) = atlmfc_lib { - self.print(&format!( + self.print(&format_args!( "cargo:rustc-link-search=native={}", atlmfc_lib.display() )); @@ -1140,17 +1140,23 @@ impl Build { } if self.link_lib_modifiers.is_empty() { - self.print(&format!("cargo:rustc-link-lib=static={}", lib_name)); + self.print(&format_args!("cargo:rustc-link-lib=static={}", lib_name)); } else { let m = self.link_lib_modifiers.join(","); - self.print(&format!("cargo:rustc-link-lib=static:{}={}", m, lib_name)); + self.print(&format_args!( + "cargo:rustc-link-lib=static:{}={}", + m, lib_name + )); } - self.print(&format!("cargo:rustc-link-search=native={}", dst.display())); + self.print(&format_args!( + "cargo:rustc-link-search=native={}", + dst.display() + )); // Add specific C++ libraries, if enabled. if self.cpp { if let Some(stdlib) = self.get_cpp_link_stdlib()? { - self.print(&format!("cargo:rustc-link-lib={}", stdlib)); + self.print(&format_args!("cargo:rustc-link-lib={}", stdlib)); } } @@ -2328,7 +2334,7 @@ impl Build { ArchSpec::Catalyst(_) => "macosx".to_owned(), }; - self.print(&format!("Detecting {} SDK path for {}", os, sdk)); + self.print(&format_args!("Detecting {} SDK path for {}", os, sdk)); let sdk_path = if let Some(sdkroot) = env::var_os("SDKROOT") { sdkroot } else { @@ -3128,10 +3134,10 @@ impl Build { return val.clone(); } if self.emit_rerun_if_env_changed && !provided_by_cargo(v) { - self.print(&format!("cargo:rerun-if-env-changed={}", v)); + self.print(&format_args!("cargo:rerun-if-env-changed={}", v)); } let r = env::var(v).ok(); - self.print(&format!("{} = {:?}", v, r)); + self.print(&format_args!("{} = {:?}", v, r)); cache.insert(v.to_string(), r.clone()); r } @@ -3174,7 +3180,7 @@ impl Build { .collect()) } - fn print(&self, s: &str) { + fn print(&self, s: &dyn Display) { if self.cargo_metadata { println!("{}", s); } From 1c81432ff556d1fbbb127524a8136dc2a2fac9a3 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 20 Jul 2023 14:34:22 +1000 Subject: [PATCH 079/138] Optimize `Error::new`: Avoid unnecessary heap alloc (#823) --- src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e8046eab4..b175e23a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -157,21 +157,21 @@ pub struct Error { /// Describes the kind of error that occurred. kind: ErrorKind, /// More explanation of error that occurred. - message: String, + message: Cow<'static, str>, } impl Error { - fn new(kind: ErrorKind, message: &str) -> Error { + fn new(kind: ErrorKind, message: impl Into>) -> Error { Error { - kind: kind, - message: message.to_owned(), + kind, + message: message.into(), } } } impl From for Error { fn from(e: io::Error) -> Error { - Error::new(ErrorKind::IOError, &format!("{}", e)) + Error::new(ErrorKind::IOError, format!("{}", e)) } } @@ -2242,7 +2242,7 @@ impl Build { let arch = target.split('-').nth(0).ok_or_else(|| { Error::new( ErrorKind::ArchitectureInvalid, - format!("Unknown architecture for {} target.", os).as_str(), + format!("Unknown architecture for {} target.", os), ) })?; @@ -2292,7 +2292,7 @@ impl Build { _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, - format!("Unknown architecture for {} target.", os).as_str(), + format!("Unknown architecture for {} target.", os), )); } } @@ -3147,7 +3147,7 @@ impl Build { Some(s) => Ok(s), None => Err(Error::new( ErrorKind::EnvVarNotFound, - &format!("Environment variable {} not defined.", v.to_string()), + format!("Environment variable {} not defined.", v), )), } } @@ -3167,7 +3167,7 @@ impl Build { Some(res) => Ok(res), None => Err(Error::new( ErrorKind::EnvVarNotFound, - &format!("Could not find environment variable {}.", var_base), + format!("Could not find environment variable {}.", var_base), )), } } @@ -3480,7 +3480,7 @@ fn wait_on_child(cmd: &Command, program: &str, child: &mut Child) -> Result<(), Err(e) => { return Err(Error::new( ErrorKind::ToolExecError, - &format!( + format!( "Failed to wait on spawned child process, command {:?} with args {:?}: {}.", cmd, program, e ), @@ -3494,7 +3494,7 @@ fn wait_on_child(cmd: &Command, program: &str, child: &mut Child) -> Result<(), } else { Err(Error::new( ErrorKind::ToolExecError, - &format!( + format!( "Command {:?} with args {:?} did not execute successfully (status code {}).", cmd, program, status ), @@ -3559,12 +3559,12 @@ fn spawn(cmd: &mut Command, program: &str, pipe_writer: File) -> Result Err(Error::new( ErrorKind::ToolExecError, - &format!( + format!( "Command {:?} with args {:?} failed to start: {:?}", cmd.0, program, e ), From d616c2ec93027431939cfbb1f881789ceea55a18 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 20 Jul 2023 16:24:48 +1000 Subject: [PATCH 080/138] Optimize `cc::Build::try_compile`: Reuse `PrintThread` (#817) --- src/lib.rs | 59 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b175e23a9..21bcbecdf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1115,8 +1115,11 @@ impl Build { objects.push(Object::new(file.to_path_buf(), obj)); } - self.compile_objects(&objects)?; - self.assemble(lib_name, &dst.join(gnu_lib_name), &objects)?; + + let print = PrintThread::new()?; + + self.compile_objects(&objects, &print)?; + self.assemble(lib_name, &dst.join(gnu_lib_name), &objects, &print)?; if self.get_target()?.contains("msvc") { let compiler = self.get_base_compiler()?; @@ -1257,7 +1260,7 @@ impl Build { } #[cfg(feature = "parallel")] - fn compile_objects(&self, objs: &[Object]) -> Result<(), Error> { + fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { use std::sync::Once; // Limit our parallelism globally with a jobserver. Start off by @@ -1286,7 +1289,6 @@ impl Build { // With all that in mind we compile all objects in a loop here, after we // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let print = PrintThread::new()?; let children = objs .iter() @@ -1369,12 +1371,10 @@ impl Build { } #[cfg(not(feature = "parallel"))] - fn compile_objects(&self, objs: &[Object]) -> Result<(), Error> { - let print = PrintThread::new()?; - + fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { for obj in objs { let (mut cmd, name) = self.create_compile_object_cmd(obj)?; - run_inner(&mut cmd, &name, print.pipe_writer_cloned()?.unwrap())?; + run(&mut cmd, &name, print)?; } Ok(()) @@ -2087,21 +2087,27 @@ impl Build { Ok((cmd, tool.to_string())) } - fn assemble(&self, lib_name: &str, dst: &Path, objs: &[Object]) -> Result<(), Error> { + fn assemble( + &self, + lib_name: &str, + dst: &Path, + objs: &[Object], + print: &PrintThread, + ) -> Result<(), Error> { // Delete the destination if it exists as we want to // create on the first iteration instead of appending. - let _ = fs::remove_file(&dst); + let _ = fs::remove_file(dst); // Add objects to the archive in limited-length batches. This helps keep // the length of the command line within a reasonable length to avoid // blowing system limits on limiting platforms like Windows. let objs: Vec<_> = objs .iter() - .map(|o| o.dst.clone()) - .chain(self.objects.iter().map(|path| (**path).to_owned())) + .map(|o| o.dst.as_path()) + .chain(self.objects.iter().map(std::ops::Deref::deref)) .collect(); for chunk in objs.chunks(100) { - self.assemble_progressive(dst, chunk)?; + self.assemble_progressive(dst, chunk, print)?; } if self.cuda && self.cuda_file_count() > 0 { @@ -2111,12 +2117,9 @@ impl Build { let out_dir = self.get_out_dir()?; let dlink = out_dir.join(lib_name.to_owned() + "_dlink.o"); let mut nvcc = self.get_compiler().to_command(); - nvcc.arg("--device-link") - .arg("-o") - .arg(dlink.clone()) - .arg(dst); - run(&mut nvcc, "nvcc")?; - self.assemble_progressive(dst, &[dlink])?; + nvcc.arg("--device-link").arg("-o").arg(&dlink).arg(dst); + run(&mut nvcc, "nvcc", print)?; + self.assemble_progressive(dst, &[dlink.as_path()], print)?; } let target = self.get_target()?; @@ -2148,13 +2151,18 @@ impl Build { // NOTE: We add `s` even if flags were passed using $ARFLAGS/ar_flag, because `s` // here represents a _mode_, not an arbitrary flag. Further discussion of this choice // can be seen in https://github.com/rust-lang/cc-rs/pull/763. - run(ar.arg("s").arg(dst), &cmd)?; + run(ar.arg("s").arg(dst), &cmd, print)?; } Ok(()) } - fn assemble_progressive(&self, dst: &Path, objs: &[PathBuf]) -> Result<(), Error> { + fn assemble_progressive( + &self, + dst: &Path, + objs: &[&Path], + print: &PrintThread, + ) -> Result<(), Error> { let target = self.get_target()?; if target.contains("msvc") { @@ -2175,7 +2183,7 @@ impl Build { cmd.arg(dst); } cmd.args(objs); - run(&mut cmd, &program)?; + run(&mut cmd, &program, print)?; } else { let (mut ar, cmd, _any_flags) = self.get_ar()?; @@ -2206,7 +2214,7 @@ impl Build { // NOTE: We add cq here regardless of whether $ARFLAGS/ar_flag have been used because // it dictates the _mode_ ar runs in, which the setter of $ARFLAGS/ar_flag can't // dictate. See https://github.com/rust-lang/cc-rs/pull/763 for further discussion. - run(ar.arg("cq").arg(dst).args(objs), &cmd)?; + run(ar.arg("cq").arg(dst).args(objs), &cmd, print)?; } Ok(()) @@ -3507,9 +3515,8 @@ fn run_inner(cmd: &mut Command, program: &str, pipe_writer: File) -> Result<(), wait_on_child(cmd, program, &mut child) } -fn run(cmd: &mut Command, program: &str) -> Result<(), Error> { - let mut print = PrintThread::new()?; - run_inner(cmd, program, print.pipe_writer().take().unwrap())?; +fn run(cmd: &mut Command, program: &str, print: &PrintThread) -> Result<(), Error> { + run_inner(cmd, program, print.pipe_writer_cloned()?.unwrap())?; Ok(()) } From 9042a841c23f6070179bb97974a5a15b881a9250 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 20 Jul 2023 16:24:59 +1000 Subject: [PATCH 081/138] Remove outdated doc about build parallelism (#827) --- src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 21bcbecdf..412dc0701 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,9 +34,6 @@ //! //! Cargo will also set this environment variable when executed with the `-jN` flag. //! -//! If `NUM_JOBS` is not set, the `RAYON_NUM_THREADS` environment variable can -//! also specify the build parallelism. -//! //! # Examples //! //! Use the `Build` struct to compile `src/foo.c`: From d2757965ec10da4df31d2230769ea4263cfeb57f Mon Sep 17 00:00:00 2001 From: he32 Date: Thu, 20 Jul 2023 16:25:04 +0200 Subject: [PATCH 082/138] Use correct ABI on NetBSD/riscv64, and add target entry for same. (#815) --- src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 412dc0701..1cc340d21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1979,6 +1979,9 @@ impl Build { } else if target.contains("freebsd") && arch.starts_with("64") { cmd.args.push(("-march=rv64gc").into()); cmd.args.push("-mabi=lp64d".into()); + } else if target.contains("netbsd") && arch.starts_with("64") { + cmd.args.push(("-march=rv64gc").into()); + cmd.args.push("-mabi=lp64d".into()); } else if target.contains("openbsd") && arch.starts_with("64") { cmd.args.push(("-march=rv64gc").into()); cmd.args.push("-mabi=lp64d".into()); @@ -2992,6 +2995,7 @@ impl Build { "riscv32gc-unknown-linux-gnu" => Some("riscv32-linux-gnu"), "riscv64gc-unknown-linux-musl" => Some("riscv64-linux-musl"), "riscv32gc-unknown-linux-musl" => Some("riscv32-linux-musl"), + "riscv64gc-unknown-netbsd" => Some("riscv64--netbsd"), "s390x-unknown-linux-gnu" => Some("s390x-linux-gnu"), "sparc-unknown-linux-gnu" => Some("sparc-linux-gnu"), "sparc64-unknown-linux-gnu" => Some("sparc64-linux-gnu"), From c83f5ee0d35e5566c187d3e0d0a78bddd941dc41 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Fri, 21 Jul 2023 00:29:58 +1000 Subject: [PATCH 083/138] Fix `Build::compile_objects` deadlock on parallel (#829) --- src/lib.rs | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1cc340d21..0f7fccaa5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1258,7 +1258,7 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::Once; + use std::sync::{mpsc::channel, Once}; // Limit our parallelism globally with a jobserver. Start off by // releasing our own token for this process so we can have a bit of an @@ -1266,7 +1266,8 @@ impl Build { // on Windows with the main implicit token, so we just have a bit extra // parallelism for a bit and don't reacquire later. let server = jobserver(); - let reacquire = server.release_raw().is_ok(); + // Reacquire our process's token on drop + let _reacquire = server.release_raw().ok().map(|_| JobserverToken(server)); // When compiling objects in parallel we do a few dirty tricks to speed // things up: @@ -1287,29 +1288,31 @@ impl Build { // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let children = objs - .iter() - .map(|obj| { - let (mut cmd, program) = self.create_compile_object_cmd(obj)?; - let token = server.acquire()?; + let (tx, rx) = channel::<(_, String, KillOnDrop, _)>(); - let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; + // Since jobserver::Client::acquire can block, waiting + // must be done in parallel so that acquire won't block forever. + let wait_thread = thread::Builder::new().spawn(move || { + for (cmd, program, mut child, _token) in rx { + wait_on_child(&cmd, &program, &mut child.0)?; + } - Ok((cmd, program, KillOnDrop(child), token)) - }) - .collect::, Error>>()?; + Ok(()) + })?; - for (cmd, program, mut child, _token) in children { - wait_on_child(&cmd, &program, &mut child.0)?; - } + for obj in objs { + let (mut cmd, program) = self.create_compile_object_cmd(obj)?; + let token = server.acquire()?; - // Reacquire our process's token before we proceed, which we released - // before entering the loop above. - if reacquire { - server.acquire_raw()?; + let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; + + if tx.send((cmd, program, KillOnDrop(child), token)).is_err() { + break; + } } + drop(tx); - return Ok(()); + return wait_thread.join().expect("wait_thread panics"); /// Returns a suitable `jobserver::Client` used to coordinate /// parallelism between build scripts. @@ -1365,6 +1368,13 @@ impl Build { child.kill().ok(); } } + + struct JobserverToken(&'static jobserver::Client); + impl Drop for JobserverToken { + fn drop(&mut self) { + let _ = self.0.acquire_raw(); + } + } } #[cfg(not(feature = "parallel"))] From 2d9941b2ebed5ba74924579ff35dcc9f5f25c0fd Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Fri, 21 Jul 2023 14:10:02 +1000 Subject: [PATCH 084/138] Cleanup mod com, registry, setup_config and winapi (#828) --- Cargo.toml | 2 +- src/com.rs | 40 ++++++++++--------- src/registry.rs | 94 +++++++++++++------------------------------ src/setup_config.rs | 40 +++++++++++-------- src/winapi.rs | 97 ++++++++------------------------------------- 5 files changed, 89 insertions(+), 184 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4158cd277..bde2eb225 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ jobserver = { version = "0.1.16", optional = true } libc = "0.2.62" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_System_Pipes", "Win32_Security"] } +windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_System_Pipes", "Win32_Security", "Win32_System_Com", "Win32_System_Registry"] } [features] parallel = ["jobserver"] diff --git a/src/com.rs b/src/com.rs index 843247e58..623141274 100644 --- a/src/com.rs +++ b/src/com.rs @@ -7,27 +7,31 @@ #![allow(unused)] -use crate::winapi::CoInitializeEx; -use crate::winapi::IUnknown; -use crate::winapi::Interface; -use crate::winapi::BSTR; -use crate::winapi::COINIT_MULTITHREADED; -use crate::winapi::{SysFreeString, SysStringLen}; -use crate::winapi::{HRESULT, S_FALSE, S_OK}; -use std::ffi::{OsStr, OsString}; -use std::mem::forget; -use std::ops::Deref; -use std::os::windows::ffi::{OsStrExt, OsStringExt}; -use std::ptr::null_mut; -use std::slice::from_raw_parts; +use crate::winapi::{IUnknown, Interface}; +use std::{ + ffi::{OsStr, OsString}, + mem::ManuallyDrop, + ops::Deref, + os::windows::ffi::{OsStrExt, OsStringExt}, + ptr::{null, null_mut}, + slice::from_raw_parts, +}; +use windows_sys::{ + core::{BSTR, HRESULT}, + Win32::{ + Foundation::{SysFreeString, SysStringLen, S_FALSE, S_OK}, + System::Com::{CoInitializeEx, COINIT_MULTITHREADED}, + }, +}; pub fn initialize() -> Result<(), HRESULT> { - let err = unsafe { CoInitializeEx(null_mut(), COINIT_MULTITHREADED) }; + let err = unsafe { CoInitializeEx(null(), COINIT_MULTITHREADED) }; if err != S_OK && err != S_FALSE { // S_FALSE just means COM is already initialized - return Err(err); + Err(err) + } else { + Ok(()) } - Ok(()) } pub struct ComPtr(*mut T) @@ -55,9 +59,7 @@ where /// Extracts the raw pointer. /// You are now responsible for releasing it yourself. pub fn into_raw(self) -> *mut T { - let p = self.0; - forget(self); - p + ManuallyDrop::new(self).0 } /// For internal use only. fn as_unknown(&self) -> &IUnknown { diff --git a/src/registry.rs b/src/registry.rs index cae32219c..bd5fbb2fa 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -8,63 +8,25 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::ffi::{OsStr, OsString}; -use std::io; -use std::ops::RangeFrom; -use std::os::raw; -use std::os::windows::prelude::*; +use std::{ + ffi::{OsStr, OsString}, + io, + ops::RangeFrom, + os::windows::prelude::*, + ptr::null_mut, +}; +use windows_sys::Win32::{ + Foundation::{ERROR_NO_MORE_ITEMS, ERROR_SUCCESS}, + System::Registry::{ + RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, RegQueryValueExW, HKEY, HKEY_LOCAL_MACHINE, + KEY_READ, KEY_WOW64_32KEY, REG_SZ, + }, +}; /// Must never be `HKEY_PERFORMANCE_DATA`. pub(crate) struct RegistryKey(Repr); -type HKEY = *mut u8; type DWORD = u32; -type LPDWORD = *mut DWORD; -type LPCWSTR = *const u16; -type LPWSTR = *mut u16; -type LONG = raw::c_long; -type PHKEY = *mut HKEY; -type PFILETIME = *mut u8; -type LPBYTE = *mut u8; -type REGSAM = u32; - -const ERROR_SUCCESS: DWORD = 0; -const ERROR_NO_MORE_ITEMS: DWORD = 259; -// Sign-extend into 64 bits if needed. -const HKEY_LOCAL_MACHINE: HKEY = 0x80000002u32 as i32 as isize as HKEY; -const REG_SZ: DWORD = 1; -const KEY_READ: DWORD = 0x20019; -const KEY_WOW64_32KEY: DWORD = 0x200; - -#[link(name = "advapi32")] -extern "system" { - fn RegOpenKeyExW( - key: HKEY, - lpSubKey: LPCWSTR, - ulOptions: DWORD, - samDesired: REGSAM, - phkResult: PHKEY, - ) -> LONG; - fn RegEnumKeyExW( - key: HKEY, - dwIndex: DWORD, - lpName: LPWSTR, - lpcName: LPDWORD, - lpReserved: LPDWORD, - lpClass: LPWSTR, - lpcClass: LPDWORD, - lpftLastWriteTime: PFILETIME, - ) -> LONG; - fn RegQueryValueExW( - hKey: HKEY, - lpValueName: LPCWSTR, - lpReserved: LPDWORD, - lpType: LPDWORD, - lpData: LPBYTE, - lpcbData: LPDWORD, - ) -> LONG; - fn RegCloseKey(hKey: HKEY) -> LONG; -} struct OwnedKey(HKEY); @@ -97,7 +59,7 @@ impl RegistryKey { /// Open a sub-key of `self`. pub fn open(&self, key: &OsStr) -> io::Result { let key = key.encode_wide().chain(Some(0)).collect::>(); - let mut ret = 0 as *mut _; + let mut ret = 0; let err = unsafe { RegOpenKeyExW( self.raw(), @@ -107,7 +69,7 @@ impl RegistryKey { &mut ret, ) }; - if err == ERROR_SUCCESS as LONG { + if err == ERROR_SUCCESS { Ok(RegistryKey(Repr::Owned(OwnedKey(ret)))) } else { Err(io::Error::from_raw_os_error(err as i32)) @@ -130,12 +92,12 @@ impl RegistryKey { let err = RegQueryValueExW( self.raw(), name.as_ptr(), - 0 as *mut _, + null_mut(), &mut kind, - 0 as *mut _, + null_mut(), &mut len, ); - if err != ERROR_SUCCESS as LONG { + if err != ERROR_SUCCESS { return Err(io::Error::from_raw_os_error(err as i32)); } if kind != REG_SZ { @@ -156,8 +118,8 @@ impl RegistryKey { let err = RegQueryValueExW( self.raw(), name.as_ptr(), - 0 as *mut _, - 0 as *mut _, + null_mut(), + null_mut(), v.as_mut_ptr() as *mut _, &mut len, ); @@ -165,7 +127,7 @@ impl RegistryKey { // grew between the first and second call to `RegQueryValueExW`), // both because it's extremely unlikely, and this is a bit more // defensive more defensive against weird types of registry keys. - if err != ERROR_SUCCESS as LONG { + if err != ERROR_SUCCESS { return Err(io::Error::from_raw_os_error(err as i32)); } // The length is allowed to change, but should still be even, as @@ -213,14 +175,14 @@ impl<'a> Iterator for Iter<'a> { i, v.as_mut_ptr(), &mut len, - 0 as *mut _, - 0 as *mut _, - 0 as *mut _, - 0 as *mut _, + null_mut(), + null_mut(), + null_mut(), + null_mut(), ); - if ret == ERROR_NO_MORE_ITEMS as LONG { + if ret == ERROR_NO_MORE_ITEMS { None - } else if ret != ERROR_SUCCESS as LONG { + } else if ret != ERROR_SUCCESS { Some(Err(io::Error::from_raw_os_error(ret as i32))) } else { v.set_len(len as usize); diff --git a/src/setup_config.rs b/src/setup_config.rs index 030051ca6..a738b9659 100644 --- a/src/setup_config.rs +++ b/src/setup_config.rs @@ -8,19 +8,25 @@ #![allow(bad_style)] #![allow(unused)] -use crate::winapi::Interface; -use crate::winapi::BSTR; -use crate::winapi::LPCOLESTR; -use crate::winapi::LPSAFEARRAY; -use crate::winapi::S_FALSE; -use crate::winapi::{CoCreateInstance, CLSCTX_ALL}; -use crate::winapi::{IUnknown, IUnknownVtbl}; -use crate::winapi::{HRESULT, LCID, LPCWSTR, PULONGLONG}; -use crate::winapi::{LPFILETIME, ULONG}; -use std::ffi::OsString; -use std::ptr::null_mut; +use crate::{ + com::{BStr, ComPtr}, + winapi::{ + IUnknown, IUnknownVtbl, Interface, LCID, LPCOLESTR, LPCWSTR, LPFILETIME, LPSAFEARRAY, + PULONGLONG, ULONG, + }, +}; -use crate::com::{BStr, ComPtr}; +use std::{ + ffi::OsString, + ptr::{null, null_mut}, +}; +use windows_sys::{ + core::{BSTR, HRESULT}, + Win32::{ + Foundation::S_FALSE, + System::Com::{CoCreateInstance, CLSCTX_ALL}, + }, +}; // Bindings to the Setup.Configuration stuff pub type InstanceState = u32; @@ -212,7 +218,7 @@ impl SetupInstance { SetupInstance(ComPtr::from_raw(obj)) } pub fn instance_id(&self) -> Result { - let mut s = null_mut(); + let mut s = null(); let err = unsafe { self.0.GetInstanceId(&mut s) }; let bstr = unsafe { BStr::from_raw(s) }; if err < 0 { @@ -221,7 +227,7 @@ impl SetupInstance { Ok(bstr.to_osstring()) } pub fn installation_name(&self) -> Result { - let mut s = null_mut(); + let mut s = null(); let err = unsafe { self.0.GetInstallationName(&mut s) }; let bstr = unsafe { BStr::from_raw(s) }; if err < 0 { @@ -230,7 +236,7 @@ impl SetupInstance { Ok(bstr.to_osstring()) } pub fn installation_path(&self) -> Result { - let mut s = null_mut(); + let mut s = null(); let err = unsafe { self.0.GetInstallationPath(&mut s) }; let bstr = unsafe { BStr::from_raw(s) }; if err < 0 { @@ -239,7 +245,7 @@ impl SetupInstance { Ok(bstr.to_osstring()) } pub fn installation_version(&self) -> Result { - let mut s = null_mut(); + let mut s = null(); let err = unsafe { self.0.GetInstallationVersion(&mut s) }; let bstr = unsafe { BStr::from_raw(s) }; if err < 0 { @@ -248,7 +254,7 @@ impl SetupInstance { Ok(bstr.to_osstring()) } pub fn product_path(&self) -> Result { - let mut s = null_mut(); + let mut s = null(); let this = self.0.cast::()?; let err = unsafe { this.GetProductPath(&mut s) }; let bstr = unsafe { BStr::from_raw(s) }; diff --git a/src/winapi.rs b/src/winapi.rs index 8e04ce9cb..93bea3cc4 100644 --- a/src/winapi.rs +++ b/src/winapi.rs @@ -11,20 +11,20 @@ use std::os::raw; pub type wchar_t = u16; -pub type UINT = raw::c_uint; -pub type LPUNKNOWN = *mut IUnknown; +pub use windows_sys::{ + core::GUID, + Win32::{ + Foundation::FILETIME, + System::Com::{SAFEARRAY, SAFEARRAYBOUND}, + }, +}; + pub type REFIID = *const IID; pub type IID = GUID; -pub type REFCLSID = *const IID; -pub type PVOID = *mut raw::c_void; -pub type USHORT = raw::c_ushort; pub type ULONG = raw::c_ulong; -pub type LONG = raw::c_long; pub type DWORD = u32; -pub type LPVOID = *mut raw::c_void; pub type HRESULT = raw::c_long; pub type LPFILETIME = *mut FILETIME; -pub type BSTR = *mut OLECHAR; pub type OLECHAR = WCHAR; pub type WCHAR = wchar_t; pub type LPCOLESTR = *const OLECHAR; @@ -33,75 +33,10 @@ pub type LPCWSTR = *const WCHAR; pub type PULONGLONG = *mut ULONGLONG; pub type ULONGLONG = u64; -pub const S_OK: HRESULT = 0; -pub const S_FALSE: HRESULT = 1; -pub const COINIT_MULTITHREADED: u32 = 0x0; - -pub type CLSCTX = u32; - -pub const CLSCTX_INPROC_SERVER: CLSCTX = 0x1; -pub const CLSCTX_INPROC_HANDLER: CLSCTX = 0x2; -pub const CLSCTX_LOCAL_SERVER: CLSCTX = 0x4; -pub const CLSCTX_REMOTE_SERVER: CLSCTX = 0x10; - -pub const CLSCTX_ALL: CLSCTX = - CLSCTX_INPROC_SERVER | CLSCTX_INPROC_HANDLER | CLSCTX_LOCAL_SERVER | CLSCTX_REMOTE_SERVER; - -#[repr(C)] -#[derive(Copy, Clone)] -pub struct GUID { - pub Data1: raw::c_ulong, - pub Data2: raw::c_ushort, - pub Data3: raw::c_ushort, - pub Data4: [raw::c_uchar; 8], -} - -#[repr(C)] -#[derive(Copy, Clone)] -pub struct FILETIME { - pub dwLowDateTime: DWORD, - pub dwHighDateTime: DWORD, -} - pub trait Interface { fn uuidof() -> GUID; } -#[link(name = "ole32")] -#[link(name = "oleaut32")] -extern "C" {} - -extern "system" { - pub fn CoInitializeEx(pvReserved: LPVOID, dwCoInit: DWORD) -> HRESULT; - pub fn CoCreateInstance( - rclsid: REFCLSID, - pUnkOuter: LPUNKNOWN, - dwClsContext: DWORD, - riid: REFIID, - ppv: *mut LPVOID, - ) -> HRESULT; - pub fn SysFreeString(bstrString: BSTR); - pub fn SysStringLen(pbstr: BSTR) -> UINT; -} - -#[repr(C)] -#[derive(Copy, Clone)] -pub struct SAFEARRAYBOUND { - pub cElements: ULONG, - pub lLbound: LONG, -} - -#[repr(C)] -#[derive(Copy, Clone)] -pub struct SAFEARRAY { - pub cDims: USHORT, - pub fFeatures: USHORT, - pub cbElements: ULONG, - pub cLocks: ULONG, - pub pvData: PVOID, - pub rgsabound: [SAFEARRAYBOUND; 1], -} - pub type LPSAFEARRAY = *mut SAFEARRAY; macro_rules! DEFINE_GUID { @@ -110,10 +45,10 @@ macro_rules! DEFINE_GUID { $b1:expr, $b2:expr, $b3:expr, $b4:expr, $b5:expr, $b6:expr, $b7:expr, $b8:expr ) => { pub const $name: $crate::winapi::GUID = $crate::winapi::GUID { - Data1: $l, - Data2: $w1, - Data3: $w2, - Data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8], + data1: $l, + data2: $w1, + data3: $w2, + data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8], }; }; } @@ -197,10 +132,10 @@ macro_rules! RIDL { #[inline] fn uuidof() -> $crate::winapi::GUID { $crate::winapi::GUID { - Data1: $l, - Data2: $w1, - Data3: $w2, - Data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8], + data1: $l, + data2: $w1, + data3: $w2, + data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8], } } } From f74fe611aa0c7fe0f368139a396d4ed3c4644850 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Fri, 21 Jul 2023 14:11:10 +1000 Subject: [PATCH 085/138] Optimize `Build::get_out_dir`: Return `Cow<'_, Path>` (#831) --- src/lib.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0f7fccaa5..1d104f293 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3121,15 +3121,18 @@ impl Build { self.force_frame_pointer.unwrap_or_else(|| self.get_debug()) } - fn get_out_dir(&self) -> Result { + fn get_out_dir(&self) -> Result, Error> { match &self.out_dir { - Some(p) => Ok((**p).into()), - None => Ok(env::var_os("OUT_DIR").map(PathBuf::from).ok_or_else(|| { - Error::new( - ErrorKind::EnvVarNotFound, - "Environment variable OUT_DIR not defined.", - ) - })?), + Some(p) => Ok(Cow::Borrowed(&**p)), + None => env::var_os("OUT_DIR") + .map(PathBuf::from) + .map(Cow::Owned) + .ok_or_else(|| { + Error::new( + ErrorKind::EnvVarNotFound, + "Environment variable OUT_DIR not defined.", + ) + }), } } From d34e43bed34a53de9220f9a1c65686485cbf0d2e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Fri, 21 Jul 2023 14:31:43 +1000 Subject: [PATCH 086/138] Optimize `Build::env_cache`: Store `Option>` as key (#832) --- src/lib.rs | 63 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1d104f293..cd80ea672 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -126,7 +126,7 @@ pub struct Build { warnings_into_errors: bool, warnings: Option, extra_warnings: Option, - env_cache: Arc>>>, + env_cache: Arc>>>>, apple_sdk_root_cache: Arc>>, emit_rerun_if_env_changed: bool, } @@ -1605,9 +1605,8 @@ impl Build { Some(true) => "-MT", Some(false) => "-MD", None => { - let features = self - .getenv("CARGO_CFG_TARGET_FEATURE") - .unwrap_or(String::new()); + let features = self.getenv("CARGO_CFG_TARGET_FEATURE"); + let features = features.as_deref().unwrap_or_default(); if features.contains("crt-static") { "-MT" } else { @@ -1827,9 +1826,8 @@ impl Build { } if self.static_flag.is_none() { - let features = self - .getenv("CARGO_CFG_TARGET_FEATURE") - .unwrap_or(String::new()); + let features = self.getenv("CARGO_CFG_TARGET_FEATURE"); + let features = features.as_deref().unwrap_or_default(); if features.contains("crt-static") { cmd.args.push("-static".into()); } @@ -2381,6 +2379,7 @@ impl Build { } let host = self.get_host()?; let target = self.get_target()?; + let target = &*target; let (env, msvc, gnu, traditional, clang) = if self.cpp { ("CXX", "cl.exe", "g++", "c++", "clang++") } else { @@ -2412,7 +2411,7 @@ impl Build { // semi-buggy build scripts which are shared in // makefiles/configure scripts (where spaces are far more // lenient) - let mut t = Tool::with_clang_driver(PathBuf::from(tool.trim()), driver_mode); + let mut t = Tool::with_clang_driver(tool, driver_mode); if let Some(cc_wrapper) = wrapper { t.cc_wrapper_path = Some(PathBuf::from(cc_wrapper)); } @@ -2472,7 +2471,7 @@ impl Build { format!("arm-kmc-eabi-{}", gnu) } else if target.starts_with("aarch64-kmc-solid_") { format!("aarch64-kmc-elf-{}", gnu) - } else if self.get_host()? != target { + } else if &*self.get_host()? != target { let prefix = self.prefix_for_target(&target); match prefix { Some(prefix) => { @@ -2499,10 +2498,10 @@ impl Build { "CUDA compilation currently assumes empty pre-existing args" ); let nvcc = match self.getenv_with_target_prefixes("NVCC") { - Err(_) => "nvcc".into(), - Ok(nvcc) => nvcc, + Err(_) => PathBuf::from("nvcc"), + Ok(nvcc) => PathBuf::from(&*nvcc), }; - let mut nvcc_tool = Tool::with_features(PathBuf::from(nvcc), None, self.cuda); + let mut nvcc_tool = Tool::with_features(nvcc, None, self.cuda); nvcc_tool .args .push(format!("-ccbin={}", tool.path.display()).into()); @@ -2589,7 +2588,7 @@ impl Build { } /// Returns compiler path, optional modifier name from whitelist, and arguments vec - fn env_tool(&self, name: &str) -> Option<(String, Option, Vec)> { + fn env_tool(&self, name: &str) -> Option<(PathBuf, Option, Vec)> { let tool = match self.getenv_with_target_prefixes(name) { Ok(tool) => tool, Err(_) => return None, @@ -2598,8 +2597,8 @@ impl Build { // If this is an exact path on the filesystem we don't want to do any // interpretation at all, just pass it on through. This'll hopefully get // us to support spaces-in-paths. - if Path::new(&tool).exists() { - return Some((tool, None, Vec::new())); + if Path::new(&*tool).exists() { + return Some((PathBuf::from(&*tool), None, Vec::new())); } // Ok now we want to handle a couple of scenarios. We'll assume from @@ -2638,7 +2637,7 @@ impl Build { if known_wrappers.contains(&file_stem) { if let Some(compiler) = parts.next() { return Some(( - compiler.to_string(), + compiler.into(), Some(maybe_wrapper.to_string()), parts.map(|s| s.to_string()).collect(), )); @@ -2646,7 +2645,7 @@ impl Build { } Some(( - maybe_wrapper.to_string(), + maybe_wrapper.into(), Self::rustc_wrapper_fallback(), parts.map(|s| s.to_string()).collect(), )) @@ -2665,7 +2664,7 @@ impl Build { if stdlib.is_empty() { Ok(None) } else { - Ok(Some(stdlib)) + Ok(Some(stdlib.to_string())) } } else { let target = self.get_target()?; @@ -3070,30 +3069,30 @@ impl Build { prefixes.first().map(|prefix| *prefix)) } - fn get_target(&self) -> Result, Error> { + fn get_target(&self) -> Result, Error> { match &self.target { - Some(t) => Ok(Cow::Borrowed(&t)), - None => Ok(Cow::Owned(self.getenv_unwrap("TARGET")?)), + Some(t) => Ok(t.clone()), + None => self.getenv_unwrap("TARGET"), } } - fn get_host(&self) -> Result, Error> { + fn get_host(&self) -> Result, Error> { match &self.host { - Some(h) => Ok(Cow::Borrowed(&h)), - None => Ok(Cow::Owned(self.getenv_unwrap("HOST")?)), + Some(h) => Ok(h.clone()), + None => self.getenv_unwrap("HOST"), } } - fn get_opt_level(&self) -> Result, Error> { + fn get_opt_level(&self) -> Result, Error> { match &self.opt_level { - Some(ol) => Ok(Cow::Borrowed(&ol)), - None => Ok(Cow::Owned(self.getenv_unwrap("OPT_LEVEL")?)), + Some(ol) => Ok(ol.clone()), + None => self.getenv_unwrap("OPT_LEVEL"), } } fn get_debug(&self) -> bool { self.debug.unwrap_or_else(|| match self.getenv("DEBUG") { - Some(s) => s != "false", + Some(s) => &*s != "false", None => false, }) } @@ -3136,7 +3135,7 @@ impl Build { } } - fn getenv(&self, v: &str) -> Option { + fn getenv(&self, v: &str) -> Option> { // Returns true for environment variables cargo sets for build scripts: // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts // @@ -3158,13 +3157,13 @@ impl Build { if self.emit_rerun_if_env_changed && !provided_by_cargo(v) { self.print(&format_args!("cargo:rerun-if-env-changed={}", v)); } - let r = env::var(v).ok(); + let r = env::var(v).ok().map(Arc::from); self.print(&format_args!("{} = {:?}", v, r)); cache.insert(v.to_string(), r.clone()); r } - fn getenv_unwrap(&self, v: &str) -> Result { + fn getenv_unwrap(&self, v: &str) -> Result, Error> { match self.getenv(v) { Some(s) => Ok(s), None => Err(Error::new( @@ -3174,7 +3173,7 @@ impl Build { } } - fn getenv_with_target_prefixes(&self, var_base: &str) -> Result { + fn getenv_with_target_prefixes(&self, var_base: &str) -> Result, Error> { let target = self.get_target()?; let host = self.get_host()?; let kind = if host == target { "HOST" } else { "TARGET" }; From a8e091520f5ee5712a3cb05ba042bf89c2b68d3e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Fri, 21 Jul 2023 17:26:28 +1000 Subject: [PATCH 087/138] Optimize `Build::compile_objects` on feature parallel (#833) --- src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index cd80ea672..b2b0ad728 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1260,6 +1260,15 @@ impl Build { fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { use std::sync::{mpsc::channel, Once}; + if objs.len() <= 1 { + for obj in objs { + let (mut cmd, name) = self.create_compile_object_cmd(obj)?; + run(&mut cmd, &name, print)?; + } + + return Ok(()); + } + // Limit our parallelism globally with a jobserver. Start off by // releasing our own token for this process so we can have a bit of an // easier to write loop below. If this fails, though, then we're likely From e84367bb647b9e45dae4d0435aa1a689a4eebbb1 Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sat, 22 Jul 2023 00:27:33 +0200 Subject: [PATCH 088/138] Disambiguate Windows run-times when using clang to compile. (#825) --- src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index b2b0ad728..2a192df74 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1642,6 +1642,13 @@ impl Build { cmd.push_opt_unless_duplicate(format!("-O{}", opt_level).into()); } + if cmd.family == ToolFamily::Clang && target.contains("windows") { + // Disambiguate mingw and msvc on Windows. Problem is that + // depending on the origin clang can default to a mismatchig + // run-time. + cmd.push_cc_arg(format!("--target={}", target).into()); + } + if cmd.family == ToolFamily::Clang && target.contains("android") { // For compatibility with code that doesn't use pre-defined `__ANDROID__` macro. // If compiler used via ndk-build or cmake (officially supported build methods) @@ -2575,6 +2582,10 @@ impl Build { } } + if target.contains("msvc") && tool.family == ToolFamily::Gnu { + println!("cargo:warning=GNU compiler is not supported for this target"); + } + Ok(tool) } From df2f86ceafbe787698b890a3d6bde9f969d5fa88 Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sun, 23 Jul 2023 19:58:43 +0200 Subject: [PATCH 089/138] Harmonize README.md with reality. (#835) --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 736f8e1d2..a58dc68d0 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ you can call them from Rust by declaring them in your Rust code like so: ```rust,no_run -extern { +extern "C" { fn foo_function(); fn bar_function(x: i32) -> i32; } @@ -138,9 +138,9 @@ required varies per platform, but there are three broad categories: * Unix platforms require `cc` to be the C compiler. This can be found by installing cc/clang on Linux distributions and Xcode on macOS, for example. * Windows platforms targeting MSVC (e.g. your target triple ends in `-msvc`) - require `cl.exe` to be available and in `PATH`. This is typically found in - standard Visual Studio installations and the `PATH` can be set up by running - the appropriate developer tools shell. + require Visual Studio to be installed. `cc-rs` attempts to locate it, and + if it fails, `cl.exe` is expected to be available in `PATH`. This can be + set up by running the appropriate developer tools shell. * Windows platforms targeting MinGW (e.g. your target triple ends in `-gnu`) require `cc` to be available in `PATH`. We recommend the [MinGW-w64](https://www.mingw-w64.org/) distribution, which is using the @@ -163,7 +163,7 @@ fn main() { cc::Build::new() .cpp(true) // Switch to C++ library compilation. .file("foo.cpp") - .compile("libfoo.a"); + .compile("foo"); } ``` @@ -178,7 +178,7 @@ The C++ standard library may be linked to the crate target. By default it's `lib .cpp(true) .file("foo.cpp") .cpp_link_stdlib("stdc++") // use libstdc++ - .compile("libfoo.a"); + .compile("foo"); } ``` 2. by setting the `CXXSTDLIB` environment variable. @@ -190,7 +190,7 @@ Remember that C++ does name mangling so `extern "C"` might be required to enable ## CUDA C++ support `cc-rs` also supports compiling CUDA C++ libraries by using the `cuda` method -on `Build` (currently for GNU/Clang toolchains only): +on `Build`: ```rust,no_run fn main() { @@ -208,8 +208,10 @@ fn main() { .flag("-gencode").flag("arch=compute_60,code=sm_60") // Generate code for Pascal (Jetson TX2). .flag("-gencode").flag("arch=compute_62,code=sm_62") + // Generate code in parallel + .flag("-t0") .file("bar.cu") - .compile("libbar.a"); + .compile("bar"); } ``` From b388631f8259674633776d5b7ee558fa348b1658 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 26 Jul 2023 17:21:27 +1000 Subject: [PATCH 090/138] Vendor `windows-sys` crate (#837) --- Cargo.toml | 3 - gen-windows-sys-binding/Cargo.toml | 12 ++ gen-windows-sys-binding/src/main.rs | 62 +++++++ gen-windows-sys-binding/windows_sys.list | 27 ++++ src/com.rs | 15 +- src/lib.rs | 2 + src/os_pipe/windows.rs | 2 +- src/registry.rs | 13 +- src/setup_config.rs | 8 +- src/winapi.rs | 9 +- src/windows_sys.rs | 198 +++++++++++++++++++++++ 11 files changed, 316 insertions(+), 35 deletions(-) create mode 100644 gen-windows-sys-binding/Cargo.toml create mode 100644 gen-windows-sys-binding/src/main.rs create mode 100644 gen-windows-sys-binding/windows_sys.list create mode 100644 src/windows_sys.rs diff --git a/Cargo.toml b/Cargo.toml index bde2eb225..e2784a4f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,6 @@ jobserver = { version = "0.1.16", optional = true } [target.'cfg(unix)'.dependencies] libc = "0.2.62" -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_System_Pipes", "Win32_Security", "Win32_System_Com", "Win32_System_Registry"] } - [features] parallel = ["jobserver"] diff --git a/gen-windows-sys-binding/Cargo.toml b/gen-windows-sys-binding/Cargo.toml new file mode 100644 index 000000000..6a4787b6b --- /dev/null +++ b/gen-windows-sys-binding/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "gen-windows-sys-binding" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +windows-bindgen = "0.49" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] diff --git a/gen-windows-sys-binding/src/main.rs b/gen-windows-sys-binding/src/main.rs new file mode 100644 index 000000000..5dd96cdd6 --- /dev/null +++ b/gen-windows-sys-binding/src/main.rs @@ -0,0 +1,62 @@ +//! Adapted from +//! https://github.com/rust-lang/rust/blob/master/src/tools/generate-windows-sys/src/main.rs + +use std::{ + fs, + io::{self, Write}, +}; + +/// This is printed to the file before the rest of the contents. +const PRELUDE: &str = r#"// This file is autogenerated. +// +// To add bindings, edit windows_sys.lst then run: +// +// ``` +// cd generate-windows-sys/ +// cargo run +// ``` +"#; + +const POSTLUDE: &str = r#" +/// Adapted from +/// [`core::ptr::invalid_mut()`](https://doc.rust-lang.org/src/core/ptr/mod.rs.html#600-607). +/// +/// This function should actually use `core::mem::transmute` but due to msrv +/// we use `as` casting instead. +/// +/// Once msrv is bumped to 1.56, replace this with `core::mem::transmute` since +/// it is const stablised in 1.56 +/// +/// NOTE that once supports `strict_provenance` we would also have to update +/// this. +const fn invalid_mut(addr: usize) -> *mut T { + addr as *mut T +} +"#; + +fn main() -> io::Result<()> { + // Load the list of APIs + let buffer = fs::read_to_string("windows_sys.list")?; + let names: Vec<&str> = buffer + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with("//") { + None + } else { + Some(line) + } + }) + .collect(); + + // Write the bindings to windows-sys.rs + let bindings = + windows_bindgen::standalone_std(&names).replace("::core::ptr::invalid_mut", "invalid_mut"); + + let mut f = fs::File::create("../src/windows_sys.rs")?; + f.write_all(PRELUDE.as_bytes())?; + f.write_all(bindings.as_bytes())?; + f.write_all(POSTLUDE.as_bytes())?; + + Ok(()) +} diff --git a/gen-windows-sys-binding/windows_sys.list b/gen-windows-sys-binding/windows_sys.list new file mode 100644 index 000000000..82d11c9de --- /dev/null +++ b/gen-windows-sys-binding/windows_sys.list @@ -0,0 +1,27 @@ +Windows.Win32.Foundation.FILETIME +Windows.Win32.Foundation.INVALID_HANDLE_VALUE +Windows.Win32.Foundation.ERROR_NO_MORE_ITEMS +Windows.Win32.Foundation.ERROR_SUCCESS +Windows.Win32.Foundation.SysFreeString +Windows.Win32.Foundation.SysStringLen +Windows.Win32.Foundation.S_FALSE +Windows.Win32.Foundation.S_OK + +Windows.Win32.System.Com.SAFEARRAY +Windows.Win32.System.Com.SAFEARRAYBOUND +Windows.Win32.System.Com.CLSCTX_ALL +Windows.Win32.System.Com.COINIT_MULTITHREADED +Windows.Win32.System.Com.CoCreateInstance +Windows.Win32.System.Com.CoInitializeEx + +Windows.Win32.System.Pipes.CreatePipe + +Windows.Win32.System.Registry.RegCloseKey +Windows.Win32.System.Registry.RegEnumKeyExW +Windows.Win32.System.Registry.RegOpenKeyExW +Windows.Win32.System.Registry.RegQueryValueExW +Windows.Win32.System.Registry.HKEY +Windows.Win32.System.Registry.HKEY_LOCAL_MACHINE +Windows.Win32.System.Registry.KEY_READ +Windows.Win32.System.Registry.KEY_WOW64_32KEY +Windows.Win32.System.Registry.REG_SZ diff --git a/src/com.rs b/src/com.rs index 623141274..cd2284427 100644 --- a/src/com.rs +++ b/src/com.rs @@ -7,7 +7,13 @@ #![allow(unused)] -use crate::winapi::{IUnknown, Interface}; +use crate::{ + winapi::{IUnknown, Interface}, + windows_sys::{ + CoInitializeEx, SysFreeString, SysStringLen, BSTR, COINIT_MULTITHREADED, HRESULT, S_FALSE, + S_OK, + }, +}; use std::{ ffi::{OsStr, OsString}, mem::ManuallyDrop, @@ -16,13 +22,6 @@ use std::{ ptr::{null, null_mut}, slice::from_raw_parts, }; -use windows_sys::{ - core::{BSTR, HRESULT}, - Win32::{ - Foundation::{SysFreeString, SysStringLen, S_FALSE, S_OK}, - System::Com::{CoInitializeEx, COINIT_MULTITHREADED}, - }, -}; pub fn initialize() -> Result<(), HRESULT> { let err = unsafe { CoInitializeEx(null(), COINIT_MULTITHREADED) }; diff --git a/src/lib.rs b/src/lib.rs index 2a192df74..91536c3ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,6 +81,8 @@ mod com; mod setup_config; #[cfg(windows)] mod vs_instances; +#[cfg(windows)] +mod windows_sys; pub mod windows_registry; diff --git a/src/os_pipe/windows.rs b/src/os_pipe/windows.rs index 4f7a57090..212632e43 100644 --- a/src/os_pipe/windows.rs +++ b/src/os_pipe/windows.rs @@ -1,5 +1,5 @@ +use crate::windows_sys::{CreatePipe, INVALID_HANDLE_VALUE}; use std::{fs::File, io, os::windows::prelude::*, ptr}; -use windows_sys::Win32::{Foundation::INVALID_HANDLE_VALUE, System::Pipes::CreatePipe}; /// NOTE: These pipes do not support IOCP. /// diff --git a/src/registry.rs b/src/registry.rs index bd5fbb2fa..7c1c2dd79 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -8,6 +8,10 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +use crate::windows_sys::{ + RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, RegQueryValueExW, ERROR_NO_MORE_ITEMS, + ERROR_SUCCESS, HKEY, HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_32KEY, REG_SZ, +}; use std::{ ffi::{OsStr, OsString}, io, @@ -15,13 +19,6 @@ use std::{ os::windows::prelude::*, ptr::null_mut, }; -use windows_sys::Win32::{ - Foundation::{ERROR_NO_MORE_ITEMS, ERROR_SUCCESS}, - System::Registry::{ - RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, RegQueryValueExW, HKEY, HKEY_LOCAL_MACHINE, - KEY_READ, KEY_WOW64_32KEY, REG_SZ, - }, -}; /// Must never be `HKEY_PERFORMANCE_DATA`. pub(crate) struct RegistryKey(Repr); @@ -59,7 +56,7 @@ impl RegistryKey { /// Open a sub-key of `self`. pub fn open(&self, key: &OsStr) -> io::Result { let key = key.encode_wide().chain(Some(0)).collect::>(); - let mut ret = 0; + let mut ret = null_mut(); let err = unsafe { RegOpenKeyExW( self.raw(), diff --git a/src/setup_config.rs b/src/setup_config.rs index a738b9659..fe2c03096 100644 --- a/src/setup_config.rs +++ b/src/setup_config.rs @@ -14,19 +14,13 @@ use crate::{ IUnknown, IUnknownVtbl, Interface, LCID, LPCOLESTR, LPCWSTR, LPFILETIME, LPSAFEARRAY, PULONGLONG, ULONG, }, + windows_sys::{CoCreateInstance, BSTR, CLSCTX_ALL, HRESULT, S_FALSE}, }; use std::{ ffi::OsString, ptr::{null, null_mut}, }; -use windows_sys::{ - core::{BSTR, HRESULT}, - Win32::{ - Foundation::S_FALSE, - System::Com::{CoCreateInstance, CLSCTX_ALL}, - }, -}; // Bindings to the Setup.Configuration stuff pub type InstanceState = u32; diff --git a/src/winapi.rs b/src/winapi.rs index 93bea3cc4..223599f62 100644 --- a/src/winapi.rs +++ b/src/winapi.rs @@ -11,19 +11,12 @@ use std::os::raw; pub type wchar_t = u16; -pub use windows_sys::{ - core::GUID, - Win32::{ - Foundation::FILETIME, - System::Com::{SAFEARRAY, SAFEARRAYBOUND}, - }, -}; +pub use crate::windows_sys::{FILETIME, GUID, HRESULT, SAFEARRAY, SAFEARRAYBOUND}; pub type REFIID = *const IID; pub type IID = GUID; pub type ULONG = raw::c_ulong; pub type DWORD = u32; -pub type HRESULT = raw::c_long; pub type LPFILETIME = *mut FILETIME; pub type OLECHAR = WCHAR; pub type WCHAR = wchar_t; diff --git a/src/windows_sys.rs b/src/windows_sys.rs new file mode 100644 index 000000000..ee4704d25 --- /dev/null +++ b/src/windows_sys.rs @@ -0,0 +1,198 @@ +// This file is autogenerated. +// +// To add bindings, edit windows_sys.lst then run: +// +// ``` +// cd generate-windows-sys/ +// cargo run +// ``` +// Bindings generated by `windows-bindgen` 0.49.0 + +#![allow( + non_snake_case, + non_upper_case_globals, + non_camel_case_types, + dead_code, + clippy::all +)] +#[link(name = "advapi32")] +extern "system" { + pub fn RegCloseKey(hkey: HKEY) -> WIN32_ERROR; +} +#[link(name = "advapi32")] +extern "system" { + pub fn RegEnumKeyExW( + hkey: HKEY, + dwindex: u32, + lpname: PWSTR, + lpcchname: *mut u32, + lpreserved: *const u32, + lpclass: PWSTR, + lpcchclass: *mut u32, + lpftlastwritetime: *mut FILETIME, + ) -> WIN32_ERROR; +} +#[link(name = "advapi32")] +extern "system" { + pub fn RegOpenKeyExW( + hkey: HKEY, + lpsubkey: PCWSTR, + uloptions: u32, + samdesired: REG_SAM_FLAGS, + phkresult: *mut HKEY, + ) -> WIN32_ERROR; +} +#[link(name = "advapi32")] +extern "system" { + pub fn RegQueryValueExW( + hkey: HKEY, + lpvaluename: PCWSTR, + lpreserved: *const u32, + lptype: *mut REG_VALUE_TYPE, + lpdata: *mut u8, + lpcbdata: *mut u32, + ) -> WIN32_ERROR; +} +#[link(name = "kernel32")] +extern "system" { + pub fn CreatePipe( + hreadpipe: *mut HANDLE, + hwritepipe: *mut HANDLE, + lppipeattributes: *const SECURITY_ATTRIBUTES, + nsize: u32, + ) -> BOOL; +} +#[link(name = "ole32")] +extern "system" { + pub fn CoCreateInstance( + rclsid: *const GUID, + punkouter: IUnknown, + dwclscontext: CLSCTX, + riid: *const GUID, + ppv: *mut *mut ::core::ffi::c_void, + ) -> HRESULT; +} +#[link(name = "ole32")] +extern "system" { + pub fn CoInitializeEx(pvreserved: *const ::core::ffi::c_void, dwcoinit: COINIT) -> HRESULT; +} +#[link(name = "oleaut32")] +extern "system" { + pub fn SysFreeString(bstrstring: BSTR) -> (); +} +#[link(name = "oleaut32")] +extern "system" { + pub fn SysStringLen(pbstr: BSTR) -> u32; +} +pub type ADVANCED_FEATURE_FLAGS = u16; +pub type BOOL = i32; +pub type BSTR = *const u16; +pub type CLSCTX = u32; +pub const CLSCTX_ALL: CLSCTX = 23u32; +pub type COINIT = i32; +pub const COINIT_MULTITHREADED: COINIT = 0i32; +pub const ERROR_NO_MORE_ITEMS: WIN32_ERROR = 259u32; +pub const ERROR_SUCCESS: WIN32_ERROR = 0u32; +#[repr(C)] +pub struct FILETIME { + pub dwLowDateTime: u32, + pub dwHighDateTime: u32, +} +impl ::core::marker::Copy for FILETIME {} +impl ::core::clone::Clone for FILETIME { + fn clone(&self) -> Self { + *self + } +} +#[repr(C)] +pub struct GUID { + pub data1: u32, + pub data2: u16, + pub data3: u16, + pub data4: [u8; 8], +} +impl GUID { + pub const fn from_u128(uuid: u128) -> Self { + Self { + data1: (uuid >> 96) as u32, + data2: (uuid >> 80 & 0xffff) as u16, + data3: (uuid >> 64 & 0xffff) as u16, + data4: (uuid as u64).to_be_bytes(), + } + } +} +impl ::core::marker::Copy for GUID {} +impl ::core::clone::Clone for GUID { + fn clone(&self) -> Self { + *self + } +} +pub type HANDLE = *mut ::core::ffi::c_void; +pub type HKEY = *mut ::core::ffi::c_void; +pub const HKEY_LOCAL_MACHINE: HKEY = invalid_mut(-2147483646i32 as _); +pub type HRESULT = i32; +pub const INVALID_HANDLE_VALUE: HANDLE = invalid_mut(-1i32 as _); +pub type IUnknown = *mut ::core::ffi::c_void; +pub const KEY_READ: REG_SAM_FLAGS = 131097u32; +pub const KEY_WOW64_32KEY: REG_SAM_FLAGS = 512u32; +pub type PCWSTR = *const u16; +pub type PWSTR = *mut u16; +pub type REG_SAM_FLAGS = u32; +pub const REG_SZ: REG_VALUE_TYPE = 1u32; +pub type REG_VALUE_TYPE = u32; +#[repr(C)] +pub struct SAFEARRAY { + pub cDims: u16, + pub fFeatures: ADVANCED_FEATURE_FLAGS, + pub cbElements: u32, + pub cLocks: u32, + pub pvData: *mut ::core::ffi::c_void, + pub rgsabound: [SAFEARRAYBOUND; 1], +} +impl ::core::marker::Copy for SAFEARRAY {} +impl ::core::clone::Clone for SAFEARRAY { + fn clone(&self) -> Self { + *self + } +} +#[repr(C)] +pub struct SAFEARRAYBOUND { + pub cElements: u32, + pub lLbound: i32, +} +impl ::core::marker::Copy for SAFEARRAYBOUND {} +impl ::core::clone::Clone for SAFEARRAYBOUND { + fn clone(&self) -> Self { + *self + } +} +#[repr(C)] +pub struct SECURITY_ATTRIBUTES { + pub nLength: u32, + pub lpSecurityDescriptor: *mut ::core::ffi::c_void, + pub bInheritHandle: BOOL, +} +impl ::core::marker::Copy for SECURITY_ATTRIBUTES {} +impl ::core::clone::Clone for SECURITY_ATTRIBUTES { + fn clone(&self) -> Self { + *self + } +} +pub const S_FALSE: HRESULT = 1i32; +pub const S_OK: HRESULT = 0i32; +pub type WIN32_ERROR = u32; + +/// Adapted from +/// [`core::ptr::invalid_mut()`](https://doc.rust-lang.org/src/core/ptr/mod.rs.html#600-607). +/// +/// This function should actually use `core::mem::transmute` but due to msrv +/// we use `as` casting instead. +/// +/// Once msrv is bumped to 1.56, replace this with `core::mem::transmute` since +/// it is const stablised in 1.56 +/// +/// NOTE that once supports `strict_provenance` we would also have to update +/// this. +const fn invalid_mut(addr: usize) -> *mut T { + addr as *mut T +} From f074cd9900ab76454d25b52dc9bb9fbd4a0e0e47 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 1 Aug 2023 12:03:58 -0700 Subject: [PATCH 091/138] Handle `x86_64h-apple-darwin` target (#840) --- src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 91536c3ea..e3885034b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3737,7 +3737,9 @@ fn autodetect_android_compiler(target: &str, host: &str, gnu: &str, clang: &str) // Rust and clang/cc don't agree on how to name the target. fn map_darwin_target_from_rust_to_compiler_architecture(target: &str) -> Option<&'static str> { - if target.contains("x86_64") { + if target.contains("x86_64h") { + Some("x86_64h") + } else if target.contains("x86_64") { Some("x86_64") } else if target.contains("arm64e") { Some("arm64e") From 7adebb9d8c235a53800c4ac3a160615aa1c5028c Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 1 Aug 2023 17:06:13 -0700 Subject: [PATCH 092/138] Bump version to 1.0.80 (#834) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e2784a4f3..2ab2fd393 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.79" +version = "1.0.80" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 3eb7c38722fdae6413774cf71e2917c9d9df1f77 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 2 Aug 2023 15:50:59 +1000 Subject: [PATCH 093/138] Fix `PrintThread`: Read stderr as bytes instead of str (#842) --- src/lib.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e3885034b..35f5de00e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3830,14 +3830,20 @@ impl PrintThread { // location to another. let print = thread::spawn(move || { let mut stderr = BufReader::with_capacity(4096, pipe_reader); - let mut line = String::with_capacity(20); - let mut stdout = io::stdout(); + let mut line = Vec::with_capacity(20); + let stdout = io::stdout(); - // read_line returns 0 on Eof - while stderr.read_line(&mut line).unwrap() != 0 { - writeln!(&mut stdout, "cargo:warning={}", line).ok(); + // read_until returns 0 on Eof + while stderr.read_until(b'\n', &mut line).unwrap() != 0 { + { + let mut stdout = stdout.lock(); + + stdout.write_all(b"cargo:warning=").unwrap(); + stdout.write_all(&line).unwrap(); + stdout.write_all(b"\n").unwrap(); + } - // read_line does not clear the buffer + // read_until does not clear the buffer line.clear(); } }); From 05fa16018fd5deae32a1de52f17827ee36b74645 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 2 Aug 2023 07:04:09 -0700 Subject: [PATCH 094/138] Bump version to 1.0.81 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2ab2fd393..f11f2de99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.80" +version = "1.0.81" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 6f43cf38af88b7532d1ce741e759fb951804ec6f Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 7 Aug 2023 13:47:13 +1000 Subject: [PATCH 095/138] Fix `Build::compile_objects` perf regression and deadlock (#849) --- src/lib.rs | 124 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 35f5de00e..05030ce2a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1260,7 +1260,7 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::{mpsc::channel, Once}; + use std::sync::{mpsc, Once}; if objs.len() <= 1 { for obj in objs { @@ -1299,16 +1299,88 @@ impl Build { // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let (tx, rx) = channel::<(_, String, KillOnDrop, _)>(); + let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, _)>(); // Since jobserver::Client::acquire can block, waiting // must be done in parallel so that acquire won't block forever. let wait_thread = thread::Builder::new().spawn(move || { - for (cmd, program, mut child, _token) in rx { - wait_on_child(&cmd, &program, &mut child.0)?; - } + let mut error = None; + let mut pendings = Vec::new(); + // Buffer the stdout + let mut stdout = io::BufWriter::with_capacity(128, io::stdout()); + let mut backoff_cnt = 0; + + loop { + let mut has_made_progress = false; + + // Reading new pending tasks + loop { + match rx.try_recv() { + Ok(pending) => { + has_made_progress = true; + pendings.push(pending) + } + Err(mpsc::TryRecvError::Disconnected) if pendings.is_empty() => { + let _ = stdout.flush(); + return if let Some(err) = error { + Err(err) + } else { + Ok(()) + }; + } + _ => break, + } + } - Ok(()) + // Try waiting on them. + pendings.retain_mut(|(cmd, program, child, _)| { + match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { + Ok(Some(())) => { + // Task done, remove the entry + has_made_progress = true; + false + } + Ok(None) => true, // Task still not finished, keep the entry + Err(err) => { + // Task fail, remove the entry. + has_made_progress = true; + + // Since we can only return one error, log the error to make + // sure users always see all the compilation failures. + let _ = writeln!(stdout, "cargo:warning={}", err); + error = Some(err); + + false + } + } + }); + + if !has_made_progress { + if backoff_cnt > 3 { + // We have yielded at least three times without making' + // any progress, so we will sleep for a while. + let duration = + std::time::Duration::from_millis(100 * (backoff_cnt - 3).min(10)); + thread::sleep(duration); + } else { + // Given that we spawned a lot of compilation tasks, it is unlikely + // that OS cannot find other ready task to execute. + // + // If all of them are done, then we will yield them and spawn more, + // or simply returns. + // + // Thus this will not be turned into a busy-wait loop and it will not + // waste CPU resource. + thread::yield_now(); + } + } + + backoff_cnt = if has_made_progress { + 0 + } else { + backoff_cnt + 1 + }; + } })?; for obj in objs { @@ -1317,10 +1389,10 @@ impl Build { let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; - if tx.send((cmd, program, KillOnDrop(child), token)).is_err() { - break; - } + tx.send((cmd, program, KillOnDrop(child), token)) + .expect("Wait thread must be alive until all compilation jobs are done, otherwise we risk deadlock"); } + // Drop tx so that the wait_thread could return drop(tx); return wait_thread.join().expect("wait_thread panics"); @@ -3545,6 +3617,40 @@ fn wait_on_child(cmd: &Command, program: &str, child: &mut Child) -> Result<(), } } +#[cfg(feature = "parallel")] +fn try_wait_on_child( + cmd: &Command, + program: &str, + child: &mut Child, + stdout: &mut dyn io::Write, +) -> Result, Error> { + match child.try_wait() { + Ok(Some(status)) => { + let _ = writeln!(stdout, "{}", status); + + if status.success() { + Ok(Some(())) + } else { + Err(Error::new( + ErrorKind::ToolExecError, + format!( + "Command {:?} with args {:?} did not execute successfully (status code {}).", + cmd, program, status + ), + )) + } + } + Ok(None) => Ok(None), + Err(e) => Err(Error::new( + ErrorKind::ToolExecError, + format!( + "Failed to wait on spawned child process, command {:?} with args {:?}: {}.", + cmd, program, e + ), + )), + } +} + fn run_inner(cmd: &mut Command, program: &str, pipe_writer: File) -> Result<(), Error> { let mut child = spawn(cmd, program, pipe_writer)?; wait_on_child(cmd, program, &mut child) From bf4f709765016a16d2f4966305d6c181bdf05325 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 7 Aug 2023 14:14:08 +1000 Subject: [PATCH 096/138] Release cc v1.0.82 (#850) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f11f2de99..493df0eab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.81" +version = "1.0.82" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From e6cc89b735710de9eb89c8dde1ed1c1103661313 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sat, 12 Aug 2023 01:44:54 -0700 Subject: [PATCH 097/138] fix: fix the usage of include directories --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 05030ce2a..3288e9198 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1621,7 +1621,7 @@ impl Build { for directory in self.include_directories.iter() { cmd.args.push("-I".into()); - cmd.args.push((**directory).into()); + cmd.args.push(directory.as_os_str().into()); } // If warnings and/or extra_warnings haven't been explicitly set, From 360754b91441395713c24db6e5a5e3a91b298527 Mon Sep 17 00:00:00 2001 From: hikari_no_yume Date: Thu, 17 Aug 2023 17:08:08 +0200 Subject: [PATCH 098/138] Add method for specifying C/C++ standard version (#761) --- src/lib.rs | 41 +++++++++++++++++++++++++++++++++++++++++ tests/test.rs | 16 ++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 05030ce2a..2fe30b98c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,6 +108,7 @@ pub struct Build { cpp_set_stdlib: Option>, cuda: bool, cudart: Option>, + std: Option>, target: Option>, host: Option>, out_dir: Option>, @@ -314,6 +315,7 @@ impl Build { cpp_set_stdlib: None, cuda: false, cudart: None, + std: None, target: None, host: None, out_dir: None, @@ -708,6 +710,37 @@ impl Build { self } + /// Specify the C or C++ language standard version. + /// + /// These values are common to modern versions of GCC, Clang and MSVC: + /// - `c11` for ISO/IEC 9899:2011 + /// - `c17` for ISO/IEC 9899:2018 + /// - `c++14` for ISO/IEC 14882:2014 + /// - `c++17` for ISO/IEC 14882:2017 + /// - `c++20` for ISO/IEC 14882:2020 + /// + /// Other values have less broad support, e.g. MSVC does not support `c++11` + /// (`c++14` is the minimum), `c89` (omit the flag instead) or `c99`. + /// + /// For compiling C++ code, you should also set `.cpp(true)`. + /// + /// The default is that no standard flag is passed to the compiler, so the + /// language version will be the compiler's default. + /// + /// # Example + /// + /// ```no_run + /// cc::Build::new() + /// .file("src/modern.cpp") + /// .cpp(true) + /// .std("c++17") + /// .compile("modern"); + /// ``` + pub fn std(&mut self, std: &str) -> &mut Build { + self.std = Some(std.into()); + self + } + /// Set warnings into errors flag. /// /// Disabled by default. @@ -1613,6 +1646,14 @@ impl Build { println!("Info: default compiler flags are disabled"); } + if let Some(ref std) = self.std { + let separator = match cmd.family { + ToolFamily::Msvc { .. } => ':', + ToolFamily::Gnu | ToolFamily::Clang => '=', + }; + cmd.push_cc_arg(format!("-std{}{}", separator, std).into()); + } + if let Ok(flags) = self.envflags(if self.cpp { "CXXFLAGS" } else { "CFLAGS" }) { for arg in flags { cmd.push_cc_arg(arg.into()); diff --git a/tests/test.rs b/tests/test.rs index 161abd8ab..7f1ddb218 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -366,6 +366,14 @@ fn gnu_no_dash_dash() { test.cmd(0).must_not_have("--"); } +#[test] +fn gnu_std_c() { + let test = Test::gnu(); + test.gcc().file("foo.c").std("c11").compile("foo"); + + test.cmd(0).must_have("-std=c11"); +} + #[test] fn msvc_smoke() { reset_env(); @@ -443,6 +451,14 @@ fn msvc_no_dash_dash() { test.cmd(0).must_not_have("--"); } +#[test] +fn msvc_std_c() { + let test = Test::msvc(); + test.gcc().file("foo.c").std("c11").compile("foo"); + + test.cmd(0).must_have("-std:c11"); +} + // Disable this test with the parallel feature because the execution // order is not deterministic. #[cfg(not(feature = "parallel"))] From d8a653b1b91922685b8d2889f9d6a3cd5c51eff9 Mon Sep 17 00:00:00 2001 From: kadiwa Date: Sun, 20 Aug 2023 20:46:14 +0200 Subject: [PATCH 099/138] Stop using feature "libc/std" (#860) --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 493df0eab..7e77025f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,9 @@ edition = "2018" jobserver = { version = "0.1.16", optional = true } [target.'cfg(unix)'.dependencies] -libc = "0.2.62" +# Don't turn on the feature "std" for this, see https://github.com/rust-lang/cargo/issues/4866 +# which is still an issue with `resolver = "1"`. +libc = { version = "0.2.62", default-features = false } [features] parallel = ["jobserver"] From bfc3ba414e896f48a70d55d9be1097e2945e17ad Mon Sep 17 00:00:00 2001 From: kadiwa Date: Sun, 20 Aug 2023 20:55:37 +0200 Subject: [PATCH 100/138] Release version 1.0.83 (#861) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7e77025f8..89945c56f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.82" +version = "1.0.83" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 0d082a201188da0770b4237a2df6496dbb01ebd6 Mon Sep 17 00:00:00 2001 From: Robin Lambertz Date: Fri, 1 Sep 2023 03:58:52 +0200 Subject: [PATCH 101/138] Separate source args with -- when compiling assembly with clang-cl (#857) --- src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2fe30b98c..8dd19e10d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1512,7 +1512,8 @@ impl Build { let clang = compiler.family == ToolFamily::Clang; let gnu = compiler.family == ToolFamily::Gnu; - let (mut cmd, name) = if msvc && asm_ext == Some(AsmFileExt::DotAsm) { + let is_assembler_msvc = msvc && asm_ext == Some(AsmFileExt::DotAsm); + let (mut cmd, name) = if is_assembler_msvc { self.msvc_macro_assembler()? } else { let mut cmd = compiler.to_command(); @@ -1543,7 +1544,7 @@ impl Build { if is_asm { cmd.args(self.asm_flags.iter().map(std::ops::Deref::deref)); } - if compiler.family == (ToolFamily::Msvc { clang_cl: true }) && !is_asm { + if compiler.family == (ToolFamily::Msvc { clang_cl: true }) && !is_assembler_msvc { // #513: For `clang-cl`, separate flags/options from the input file. // When cross-compiling macOS -> Windows, this avoids interpreting // common `/Users/...` paths as the `/U` flag and triggering From fc93ea43c8510d7136dff942d47327c7bd8689c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 02:47:34 +0000 Subject: [PATCH 102/138] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f85288d4d..7aef82c15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: rust: stable-x86_64 target: x86_64-pc-windows-msvc steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Rust (rustup) run: | set -euxo pipefail @@ -96,7 +96,7 @@ jobs: name: Test CUDA support runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install cuda-minimal-build-11-8 shell: bash run: | @@ -117,7 +117,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Rust run: | rustup toolchain install 1.46.0 --no-self-update --profile minimal @@ -129,7 +129,7 @@ jobs: name: Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Rust run: | rustup toolchain install stable --no-self-update --profile minimal --component rustfmt From daab9244b03e244c4f2511944870d719c443f61f Mon Sep 17 00:00:00 2001 From: BlackHoleFox Date: Thu, 14 Sep 2023 13:53:18 -0600 Subject: [PATCH 103/138] Use rustc's Apple deployment target defaults when available (#848) * use MACOSX_DEPLOYMENT_TARGET when set * Use rustc's minimum Apple deployment targets when available * Obtain deployment target consistently Fix the now-unified default watchOS deployment target --------- Co-authored-by: Jeong YunWon --- src/lib.rs | 186 +++++++++++++++++++++++++++++++++++--------------- tests/test.rs | 18 +++++ 2 files changed, 150 insertions(+), 54 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8dd19e10d..f3e921bf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1832,8 +1832,8 @@ impl Build { if let Some(arch) = map_darwin_target_from_rust_to_compiler_architecture(target) { - let deployment_target = env::var("IPHONEOS_DEPLOYMENT_TARGET") - .unwrap_or_else(|_| "7.0".into()); + let deployment_target = + self.apple_deployment_version(AppleOs::Ios, target, None); cmd.args.push( format!( "--target={}-apple-ios{}-simulator", @@ -1846,8 +1846,8 @@ impl Build { if let Some(arch) = map_darwin_target_from_rust_to_compiler_architecture(target) { - let deployment_target = env::var("WATCHOS_DEPLOYMENT_TARGET") - .unwrap_or_else(|_| "5.0".into()); + let deployment_target = + self.apple_deployment_version(AppleOs::WatchOs, target, None); cmd.args.push( format!( "--target={}-apple-watchos{}-simulator", @@ -2141,8 +2141,8 @@ impl Build { } } - if target.contains("apple-ios") || target.contains("apple-watchos") { - self.ios_watchos_flags(cmd)?; + if target.contains("-apple-") { + self.apple_flags(cmd)?; } if self.static_flag.unwrap_or(false) { @@ -2360,37 +2360,30 @@ impl Build { Ok(()) } - fn ios_watchos_flags(&self, cmd: &mut Tool) -> Result<(), Error> { + fn apple_flags(&self, cmd: &mut Tool) -> Result<(), Error> { enum ArchSpec { Device(&'static str), Simulator(&'static str), Catalyst(&'static str), } - enum Os { - Ios, - WatchOs, - } - impl Display for Os { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - Os::Ios => f.write_str("iOS"), - Os::WatchOs => f.write_str("WatchOS"), - } - } - } - let target = self.get_target()?; - let os = if target.contains("-watchos") { - Os::WatchOs + let os = if target.contains("-darwin") { + AppleOs::MacOs + } else if target.contains("-watchos") { + AppleOs::WatchOs } else { - Os::Ios + AppleOs::Ios + }; + let is_mac = match os { + AppleOs::MacOs => true, + _ => false, }; - let arch = target.split('-').nth(0).ok_or_else(|| { + let arch_str = target.split('-').nth(0).ok_or_else(|| { Error::new( ErrorKind::ArchitectureInvalid, - format!("Unknown architecture for {} target.", os), + format!("Unknown architecture for {:?} target.", os), ) })?; @@ -2404,11 +2397,22 @@ impl Build { None => false, }; - let arch = if is_catalyst { - match arch { + let arch = if is_mac { + match arch_str { + "i686" => ArchSpec::Device("-m32"), + "x86_64" | "x86_64h" | "aarch64" => ArchSpec::Device("-m64"), + _ => { + return Err(Error::new( + ErrorKind::ArchitectureInvalid, + "Unknown architecture for macOS target.", + )); + } + } + } else if is_catalyst { + match arch_str { "arm64e" => ArchSpec::Catalyst("arm64e"), "arm64" | "aarch64" => ArchSpec::Catalyst("arm64"), - "x86_64" => ArchSpec::Catalyst("-m64"), + "x86_64" | "x86_64h" => ArchSpec::Catalyst("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, @@ -2417,9 +2421,9 @@ impl Build { } } } else if is_sim { - match arch { + match arch_str { "arm64" | "aarch64" => ArchSpec::Simulator("arm64"), - "x86_64" => ArchSpec::Simulator("-m64"), + "x86_64" | "x86_64h" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, @@ -2428,7 +2432,7 @@ impl Build { } } } else { - match arch { + match arch_str { "arm" | "armv7" | "thumbv7" => ArchSpec::Device("armv7"), "armv7k" => ArchSpec::Device("armv7k"), "armv7s" | "thumbv7s" => ArchSpec::Device("armv7s"), @@ -2436,30 +2440,29 @@ impl Build { "arm64" | "aarch64" => ArchSpec::Device("arm64"), "arm64_32" => ArchSpec::Device("arm64_32"), "i386" | "i686" => ArchSpec::Simulator("-m32"), - "x86_64" => ArchSpec::Simulator("-m64"), + "x86_64" | "x86_64h" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, - format!("Unknown architecture for {} target.", os), + format!("Unknown architecture for {:?} target.", os), )); } } }; - let (sdk_prefix, sim_prefix, min_version) = match os { - Os::Ios => ( - "iphone", - "ios-", - std::env::var("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "7.0".into()), - ), - Os::WatchOs => ( - "watch", - "watch", - std::env::var("WATCHOS_DEPLOYMENT_TARGET").unwrap_or_else(|_| "2.0".into()), - ), + let min_version = self.apple_deployment_version(os, &target, Some(arch_str)); + let (sdk_prefix, sim_prefix) = match os { + AppleOs::MacOs => ("macosx", ""), + AppleOs::Ios => ("iphone", "ios-"), + AppleOs::WatchOs => ("watch", "watch"), }; let sdk = match arch { + ArchSpec::Device(_) if is_mac => { + cmd.args + .push(format!("-mmacosx-version-min={}", min_version).into()); + "macosx".to_owned() + } ArchSpec::Device(arch) => { cmd.args.push("-arch".into()); cmd.args.push(arch.into()); @@ -2482,17 +2485,19 @@ impl Build { ArchSpec::Catalyst(_) => "macosx".to_owned(), }; - self.print(&format_args!("Detecting {} SDK path for {}", os, sdk)); - let sdk_path = if let Some(sdkroot) = env::var_os("SDKROOT") { - sdkroot - } else { - self.apple_sdk_root(sdk.as_str())? - }; + if !is_mac { + self.print(&format_args!("Detecting {:?} SDK path for {}", os, sdk)); + let sdk_path = if let Some(sdkroot) = env::var_os("SDKROOT") { + sdkroot + } else { + self.apple_sdk_root(sdk.as_str())? + }; - cmd.args.push("-isysroot".into()); - cmd.args.push(sdk_path); - // TODO: Remove this once Apple stops accepting apps built with Xcode 13 - cmd.args.push("-fembed-bitcode".into()); + cmd.args.push("-isysroot".into()); + cmd.args.push(sdk_path); + // TODO: Remove this once Apple stops accepting apps built with Xcode 13 + cmd.args.push("-fembed-bitcode".into()); + } Ok(()) } @@ -3399,6 +3404,63 @@ impl Build { Ok(ret) } + fn apple_deployment_version( + &self, + os: AppleOs, + target: &str, + arch_str: Option<&str>, + ) -> String { + fn rustc_provided_target(rustc: Option<&str>, target: &str) -> Option { + let rustc = rustc?; + let output = Command::new(rustc) + .args(&["--target", target]) + .args(&["--print", "deployment-target"]) + .output() + .ok()?; + + if output.status.success() { + std::str::from_utf8(&output.stdout) + .unwrap() + .strip_prefix("deployment_target=") + .map(|v| v.trim()) + .map(ToString::to_string) + } else { + // rustc is < 1.71 + None + } + } + + let rustc = self.getenv("RUSTC"); + let rustc = rustc.as_deref(); + // note the hardcoded minimums here are subject to change in a future compiler release, + // so the ones rustc knows about are preferred. For any compiler version that has bumped them + // though, `--print deployment-target` will be present and the fallbacks won't be used. + // + // the ordering of env -> rustc -> old defaults is intentional for performance when using + // an explicit target + match os { + AppleOs::MacOs => env::var("MACOSX_DEPLOYMENT_TARGET") + .ok() + .or_else(|| rustc_provided_target(rustc, target)) + .unwrap_or_else(|| { + if arch_str == Some("aarch64") { + "11.0" + } else { + "10.7" + } + .into() + }), + AppleOs::Ios => env::var("IPHONEOS_DEPLOYMENT_TARGET") + .ok() + .or_else(|| rustc_provided_target(rustc, target)) + .unwrap_or_else(|| "7.0".into()), + AppleOs::WatchOs => env::var("WATCHOS_DEPLOYMENT_TARGET") + .ok() + .or_else(|| rustc_provided_target(rustc, target)) + .unwrap_or_else(|| "5.0".into()), + } + } + fn cuda_file_count(&self) -> usize { self.files .iter() @@ -3786,6 +3848,22 @@ fn command_add_output_file( } } +#[derive(Clone, Copy)] +enum AppleOs { + MacOs, + Ios, + WatchOs, +} +impl std::fmt::Debug for AppleOs { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + AppleOs::MacOs => f.write_str("macOS"), + AppleOs::Ios => f.write_str("iOS"), + AppleOs::WatchOs => f.write_str("WatchOS"), + } + } +} + // Use by default minimum available API level // See note about naming here // https://android.googlesource.com/platform/ndk/+/refs/heads/ndk-release-r21/docs/BuildSystemMaintainers.md#Clang diff --git a/tests/test.rs b/tests/test.rs index 7f1ddb218..b1c10c10f 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -475,3 +475,21 @@ fn asm_flags() { test.cmd(1).must_have("--abc"); test.cmd(2).must_have("--abc"); } + +#[test] +fn gnu_apple_darwin() { + for (arch, version) in &[("x86_64", "10.7"), ("aarch64", "11.0")] { + let target = format!("{}-apple-darwin", arch); + let test = Test::gnu(); + test.gcc() + .target(&target) + .host(&target) + // Avoid test maintainence when minimum supported OSes change. + .__set_env("MACOSX_DEPLOYMENT_TARGET", version) + .file("foo.c") + .compile("foo"); + + test.cmd(0) + .must_have(format!("-mmacosx-version-min={}", version)); + } +} From 7f3acb516cf4552a8ed3cfea22139c7e86f7264e Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sat, 23 Sep 2023 04:01:00 +0200 Subject: [PATCH 104/138] Don't pass compiler flags to MSVC assembers. (#867) While ml[64] ignores the compiler flags with warnings, armasm[64] rejects them and fails. --- src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f3e921bf4..85a1169cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2220,9 +2220,6 @@ impl Build { if target.contains("i686") || target.contains("i586") { cmd.arg("-safeseh"); } - for flag in self.flags.iter() { - cmd.arg(&**flag); - } Ok((cmd, tool.to_string())) } From 7ee0f86182f9645b0be92afb0b72b415e33a0d1a Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sat, 23 Sep 2023 04:03:46 +0200 Subject: [PATCH 105/138] Pass self.definitions to armasm[64].exe. (#868) --- src/lib.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 85a1169cd..e9beae309 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2202,7 +2202,20 @@ impl Build { cmd.arg("-g"); } - println!("cargo:warning=The MSVC ARM assemblers do not support -D flags"); + for &(ref key, ref value) in self.definitions.iter() { + cmd.arg("-PreDefine"); + if let Some(ref value) = *value { + if let Ok(i) = value.parse::() { + cmd.arg(&format!("{} SETA {}", key, i)); + } else if value.starts_with('"') && value.ends_with('"') { + cmd.arg(&format!("{} SETS {}", key, value)); + } else { + cmd.arg(&format!("{} SETS \"{}\"", key, value)); + } + } else { + cmd.arg(&format!("{} SETL {}", key, "{TRUE}")); + } + } } else { if self.get_debug() { cmd.arg("-Zi"); From 17186eaca771c7381c0441c463821593cf664e3e Mon Sep 17 00:00:00 2001 From: Andy Polyakov <9038069+dot-asm@users.noreply.github.com> Date: Sat, 23 Sep 2023 17:23:03 +0200 Subject: [PATCH 106/138] Be more specific about not passing -c to armasm[64].exe. (#869) 4ce411709170197228d353aa2bbf385efe3b958a passes non-.asm files to the C compiler driver, which unlike armasm[64].exe does require -c flag. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e9beae309..3dc25b23b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1535,7 +1535,7 @@ impl Build { &mut cmd, &obj.dst, self.cuda, msvc, clang, gnu, is_asm, is_arm, ); // armasm and armasm64 don't requrie -c option - if !msvc || !is_asm || !is_arm { + if !is_assembler_msvc || !is_arm { cmd.arg("-c"); } if self.cuda && self.cuda_file_count() > 1 { From a211d910173ed625a3581beee9a70d6c0af5792d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sun, 24 Sep 2023 00:24:12 +0900 Subject: [PATCH 107/138] Add AppleTVOS support (#704) --- src/lib.rs | 34 ++++++++++++++++++++++++++++++---- tests/test.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3dc25b23b..aa5eef4b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1774,7 +1774,10 @@ impl Build { cmd.push_opt_unless_duplicate("-DANDROID".into()); } - if !target.contains("apple-ios") && !target.contains("apple-watchos") { + if !target.contains("apple-ios") + && !target.contains("apple-watchos") + && !target.contains("apple-tvos") + { cmd.push_cc_arg("-ffunction-sections".into()); cmd.push_cc_arg("-fdata-sections".into()); } @@ -1856,6 +1859,20 @@ impl Build { .into(), ); } + } else if target.contains("x86_64-apple-tvos") { + if let Some(arch) = + map_darwin_target_from_rust_to_compiler_architecture(target) + { + let deployment_target = + self.apple_deployment_version(AppleOs::TvOs, target, None); + cmd.args.push( + format!( + "--target={}-apple-tvos{}-simulator", + arch, deployment_target + ) + .into(), + ); + } } else if target.starts_with("riscv64gc-") { cmd.args.push( format!("--target={}", target.replace("riscv64gc", "riscv64")).into(), @@ -2382,6 +2399,8 @@ impl Build { AppleOs::MacOs } else if target.contains("-watchos") { AppleOs::WatchOs + } else if target.contains("-tvos") { + AppleOs::TvOs } else { AppleOs::Ios }; @@ -2402,7 +2421,7 @@ impl Build { None => false, }; - let is_sim = match target.split('-').nth(3) { + let is_arm_sim = match target.split('-').nth(3) { Some(v) => v == "sim", None => false, }; @@ -2430,14 +2449,14 @@ impl Build { )); } } - } else if is_sim { + } else if is_arm_sim { match arch_str { "arm64" | "aarch64" => ArchSpec::Simulator("arm64"), "x86_64" | "x86_64h" => ArchSpec::Simulator("-m64"), _ => { return Err(Error::new( ErrorKind::ArchitectureInvalid, - "Unknown architecture for iOS simulator target.", + "Unknown architecture for simulator target.", )); } } @@ -2465,6 +2484,7 @@ impl Build { AppleOs::MacOs => ("macosx", ""), AppleOs::Ios => ("iphone", "ios-"), AppleOs::WatchOs => ("watch", "watch"), + AppleOs::TvOs => ("appletv", "appletv"), }; let sdk = match arch { @@ -3468,6 +3488,10 @@ impl Build { .ok() .or_else(|| rustc_provided_target(rustc, target)) .unwrap_or_else(|| "5.0".into()), + AppleOs::TvOs => env::var("TVOS_DEPLOYMENT_TARGET") + .ok() + .or_else(|| rustc_provided_target(rustc, target)) + .unwrap_or_else(|| "9.0".into()), } } @@ -3863,6 +3887,7 @@ enum AppleOs { MacOs, Ios, WatchOs, + TvOs, } impl std::fmt::Debug for AppleOs { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { @@ -3870,6 +3895,7 @@ impl std::fmt::Debug for AppleOs { AppleOs::MacOs => f.write_str("macOS"), AppleOs::Ios => f.write_str("iOS"), AppleOs::WatchOs => f.write_str("WatchOS"), + AppleOs::TvOs => f.write_str("AppleTVOS"), } } } diff --git a/tests/test.rs b/tests/test.rs index b1c10c10f..c85f25e69 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -493,3 +493,33 @@ fn gnu_apple_darwin() { .must_have(format!("-mmacosx-version-min={}", version)); } } + +#[cfg(target_os = "macos")] +#[test] +fn apple_tvos() { + for target in &["aarch64-apple-tvos"] { + let test = Test::gnu(); + test.gcc() + .target(&target) + .host(&target) + .file("foo.c") + .compile("foo"); + + test.cmd(0).must_have("-mappletvos-version-min=9.0"); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn apple_tvsimulator() { + for target in &["x86_64-apple-tvos"] { + let test = Test::gnu(); + test.gcc() + .target(&target) + .host(&target) + .file("foo.c") + .compile("foo"); + + test.cmd(0).must_have("-mappletvsimulator-version-min=9.0"); + } +} From 0fc80910ec62d1427cbda617c5c58dbef83a4152 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sun, 24 Sep 2023 01:30:52 +0900 Subject: [PATCH 108/138] Fix xctoolchain AppleClang to corrrectly use -isysroot (#703) * Add -isysroot for xctoolchain cc * -isysroot priors to $SDKROOT --- src/lib.rs | 32 +++++++++++++++++--------------- tests/test.rs | 2 ++ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index aa5eef4b7..1759c051d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2515,7 +2515,8 @@ impl Build { ArchSpec::Catalyst(_) => "macosx".to_owned(), }; - if !is_mac { + // AppleClang sometimes requires sysroot even for darwin + if cmd.is_xctoolchain_clang() || !target.ends_with("-darwin") { self.print(&format_args!("Detecting {:?} SDK path for {}", os, sdk)); let sdk_path = if let Some(sdkroot) = env::var_os("SDKROOT") { sdkroot @@ -2525,7 +2526,10 @@ impl Build { cmd.args.push("-isysroot".into()); cmd.args.push(sdk_path); - // TODO: Remove this once Apple stops accepting apps built with Xcode 13 + } + + // TODO: Remove this once Apple stops accepting apps built with Xcode 13 + if !is_mac { cmd.args.push("-fembed-bitcode".into()); } @@ -3382,19 +3386,6 @@ impl Build { let target = self.get_target()?; let host = self.get_host()?; if host.contains("apple-darwin") && target.contains("apple-darwin") { - // If, for example, `cargo` runs during the build of an XCode project, then `SDKROOT` environment variable - // would represent the current target, and this is the problem for us, if we want to compile something - // for the host, when host != target. - // We can not just remove `SDKROOT`, because, again, for example, XCode add to PATH - // /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin - // and `cc` from this path can not find system include files, like `pthread.h`, if `SDKROOT` - // is not set - if let Ok(sdkroot) = env::var("SDKROOT") { - if !sdkroot.contains("MacOSX") { - let macos_sdk = self.apple_sdk_root("macosx")?; - cmd.env("SDKROOT", macos_sdk); - } - } // Additionally, `IPHONEOS_DEPLOYMENT_TARGET` must not be set when using the Xcode linker at // "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld", // although this is apparently ignored when using the linker at "/usr/bin/ld". @@ -3718,6 +3709,17 @@ impl Tool { self.family == ToolFamily::Clang } + /// Whether the tool is AppleClang under .xctoolchain + #[cfg(target_vendor = "apple")] + fn is_xctoolchain_clang(&self) -> bool { + let path = self.path.to_string_lossy(); + path.contains(".xctoolchain/") + } + #[cfg(not(target_vendor = "apple"))] + fn is_xctoolchain_clang(&self) -> bool { + false + } + /// Whether the tool is MSVC-like. pub fn is_like_msvc(&self) -> bool { match self.family { diff --git a/tests/test.rs b/tests/test.rs index c85f25e69..a0c258efe 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -489,8 +489,10 @@ fn gnu_apple_darwin() { .file("foo.c") .compile("foo"); + let cmd = test.cmd(0); test.cmd(0) .must_have(format!("-mmacosx-version-min={}", version)); + cmd.must_not_have("-isysroot"); } } From d765f309f1d29d89a05b1a2cb4e33e4259949cf9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sun, 24 Sep 2023 01:32:27 +0900 Subject: [PATCH 109/138] detect clang/gcc using --version (#709) * put m32/m64 always to gcc/clang * try to detect clang/gcc * Set target to apple-darwin for macos to avoid cross-compile * Apply suggestions from code review Co-authored-by: Thom Chiovoloni --------- Co-authored-by: Thom Chiovoloni --- src/lib.rs | 51 +++++++++++++++++++++++++++++++++++--------- tests/support/mod.rs | 6 +++++- tests/test.rs | 20 ++++++++++++++--- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1759c051d..c87e9b0d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1811,6 +1811,16 @@ impl Build { family.add_force_frame_pointer(cmd); } + if !cmd.is_like_msvc() { + if target.contains("i686") || target.contains("i586") { + cmd.args.push("-m32".into()); + } else if target == "x86_64-unknown-linux-gnux32" { + cmd.args.push("-mx32".into()); + } else if target.contains("x86_64") || target.contains("powerpc64") { + cmd.args.push("-m64".into()); + } + } + // Target flags match cmd.family { ToolFamily::Clang => { @@ -1954,14 +1964,6 @@ impl Build { } } ToolFamily::Gnu => { - if target.contains("i686") || target.contains("i586") { - cmd.args.push("-m32".into()); - } else if target == "x86_64-unknown-linux-gnux32" { - cmd.args.push("-mx32".into()); - } else if target.contains("x86_64") || target.contains("powerpc64") { - cmd.args.push("-m64".into()); - } - if target.contains("darwin") { if let Some(arch) = map_darwin_target_from_rust_to_compiler_architecture(target) { @@ -3525,6 +3527,35 @@ impl Tool { } fn with_features(path: PathBuf, clang_driver: Option<&str>, cuda: bool) -> Self { + fn detect_family(path: &Path) -> ToolFamily { + let mut cmd = Command::new(path); + cmd.arg("--version"); + + let stdout = match run_output(&mut cmd, &path.to_string_lossy()) + .ok() + .and_then(|o| String::from_utf8(o).ok()) + { + Some(s) => s, + None => { + // --version failed. fallback to gnu + println!("cargo-warning:Failed to run: {:?}", cmd); + return ToolFamily::Gnu; + } + }; + if stdout.contains("clang") { + ToolFamily::Clang + } else if stdout.contains("GCC") { + ToolFamily::Gnu + } else { + // --version doesn't include clang for GCC + println!( + "cargo-warning:Compiler version doesn't include clang or GCC: {:?}", + cmd + ); + ToolFamily::Gnu + } + } + // Try to detect family of the tool from its name, falling back to Gnu. let family = if let Some(fname) = path.file_name().and_then(|p| p.to_str()) { if fname.contains("clang-cl") { @@ -3537,10 +3568,10 @@ impl Tool { _ => ToolFamily::Clang, } } else { - ToolFamily::Gnu + detect_family(&path) } } else { - ToolFamily::Gnu + detect_family(&path) }; Tool { diff --git a/tests/support/mod.rs b/tests/support/mod.rs index f3c04405a..3ec191113 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -76,7 +76,11 @@ impl Test { let target = if self.msvc { "x86_64-pc-windows-msvc" } else { - "x86_64-unknown-linux-gnu" + if cfg!(target_os = "macos") { + "x86_64-apple-darwin" + } else { + "x86_64-unknown-linux-gnu" + } }; cfg.target(target) diff --git a/tests/test.rs b/tests/test.rs index a0c258efe..ab2a3c1ce 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -55,7 +55,11 @@ fn gnu_opt_level_s() { #[test] fn gnu_debug() { let test = Test::gnu(); - test.gcc().debug(true).file("foo.c").compile("foo"); + test.gcc() + .target("x86_64-unknown-linux") + .debug(true) + .file("foo.c") + .compile("foo"); test.cmd(0).must_have("-gdwarf-4"); let test = Test::gnu(); @@ -70,7 +74,11 @@ fn gnu_debug() { #[test] fn gnu_debug_fp_auto() { let test = Test::gnu(); - test.gcc().debug(true).file("foo.c").compile("foo"); + test.gcc() + .target("x86_64-unknown-linux") + .debug(true) + .file("foo.c") + .compile("foo"); test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_have("-fno-omit-frame-pointer"); } @@ -78,7 +86,11 @@ fn gnu_debug_fp_auto() { #[test] fn gnu_debug_fp() { let test = Test::gnu(); - test.gcc().debug(true).file("foo.c").compile("foo"); + test.gcc() + .target("x86_64-unknown-linux") + .debug(true) + .file("foo.c") + .compile("foo"); test.cmd(0).must_have("-gdwarf-4"); test.cmd(0).must_have("-fno-omit-frame-pointer"); } @@ -89,6 +101,7 @@ fn gnu_debug_nofp() { let test = Test::gnu(); test.gcc() + .target("x86_64-unknown-linux") .debug(true) .force_frame_pointer(false) .file("foo.c") @@ -98,6 +111,7 @@ fn gnu_debug_nofp() { let test = Test::gnu(); test.gcc() + .target("x86_64-unknown-linux") .force_frame_pointer(false) .debug(true) .file("foo.c") From ffdf35676298d4ae3626415c4e68f11b60f00116 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Sat, 23 Sep 2023 10:04:32 -0700 Subject: [PATCH 110/138] Remove -fembed-bitcode flag (#812) Apple has recently stopped accepting apps built with any Xcode version below 14.1, and rejects apps built with bitcode starting with Xcode 14. --- src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c87e9b0d9..ca8c9b8a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2530,11 +2530,6 @@ impl Build { cmd.args.push(sdk_path); } - // TODO: Remove this once Apple stops accepting apps built with Xcode 13 - if !is_mac { - cmd.args.push("-fembed-bitcode".into()); - } - Ok(()) } From c0f211891cd9d8bbb34257d00656dcda1874f5eb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sun, 24 Sep 2023 02:56:47 +0900 Subject: [PATCH 111/138] minimal maxosx deployment target is 10.9 when cpp (#872) --- src/lib.rs | 6 +++++- tests/test.rs | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ca8c9b8a5..6257fa0df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3464,7 +3464,11 @@ impl Build { if arch_str == Some("aarch64") { "11.0" } else { - "10.7" + if self.cpp { + "10.9" + } else { + "10.7" + } } .into() }), diff --git a/tests/test.rs b/tests/test.rs index ab2a3c1ce..f712c4368 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -343,11 +343,9 @@ fn gnu_flag_if_supported() { .must_not_have("-std=c++11"); } +#[cfg(not(windows))] #[test] fn gnu_flag_if_supported_cpp() { - if cfg!(windows) { - return; - } let test = Test::gnu(); test.gcc() .cpp(true) From 755e4fe6fbf4b86679aa07d02e0228315321d534 Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Sat, 9 Jul 2022 18:49:46 +0100 Subject: [PATCH 112/138] Detect msvc version if there is no default --- src/windows_registry.rs | 47 +++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/src/windows_registry.rs b/src/windows_registry.rs index 07a9c4db6..00040289c 100644 --- a/src/windows_registry.rs +++ b/src/windows_registry.rs @@ -450,14 +450,10 @@ mod impl_ { fn vs15plus_vc_paths( target: &str, - instance_path: &PathBuf, + instance_path: &Path, ) -> Option<(PathBuf, PathBuf, PathBuf, PathBuf, PathBuf)> { - let version_path = - instance_path.join(r"VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt"); - let mut version_file = File::open(version_path).ok()?; - let mut version = String::new(); - version_file.read_to_string(&mut version).ok()?; - let version = version.trim(); + let version = vs15plus_vc_read_version(instance_path)?; + let host = match host_arch() { X86 => "X86", X86_64 => "X64", @@ -487,6 +483,43 @@ mod impl_ { Some((path, bin_path, host_dylib_path, lib_path, include_path)) } + fn vs15plus_vc_read_version(dir: &Path) -> Option { + // Try to open the default version file. + let mut version_path: PathBuf = + dir.join(r"VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt"); + let mut version_file = if let Ok(f) = File::open(&version_path) { + f + } else { + // If the default doesn't exist, search for other version files. + // These are in the form Microsoft.VCToolsVersion.v143.default.txt + // where `143` is any three decimal digit version number. + // This sorts versions by lexical order and selects the highest version. + let mut version_file = String::new(); + version_path.pop(); + for file in version_path.read_dir().ok()? { + let name = file.ok()?.file_name(); + let name = name.to_str()?; + if name.starts_with("Microsoft.VCToolsVersion.v") + && name.ends_with(".default.txt") + && name > &version_file + { + version_file.replace_range(.., name); + } + } + if version_file.is_empty() { + return None; + } + version_path.push(version_file); + File::open(version_path).ok()? + }; + + // Get the version string from the file we found. + let mut version = String::new(); + version_file.read_to_string(&mut version).ok()?; + version.truncate(version.trim_end().len()); + Some(version) + } + fn atl_paths(target: &str, path: &Path) -> Option<(PathBuf, PathBuf)> { let atl_path = path.join("atlmfc"); let sub = lib_subdir(target)?; From 0c87751a251922bc05c7371b09aff36ed30881df Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Fri, 29 Sep 2023 11:32:57 +0200 Subject: [PATCH 113/138] feat: Do not preemptively drop implicit job token. --- src/lib.rs | 79 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6257fa0df..384b2b3ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1293,7 +1293,12 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::{mpsc, Once}; + use std::sync::{ + mpsc::{self, Receiver, Sender}, + Once, + }; + + use jobserver::{Acquired, Client}; if objs.len() <= 1 { for obj in objs { @@ -1309,10 +1314,54 @@ impl Build { // easier to write loop below. If this fails, though, then we're likely // on Windows with the main implicit token, so we just have a bit extra // parallelism for a bit and don't reacquire later. - let server = jobserver(); + let server = jobserver().clone(); // Reacquire our process's token on drop - let _reacquire = server.release_raw().ok().map(|_| JobserverToken(server)); - + //let _reacquire = server.release_raw().ok().map(|_| JobserverToken(server)); + struct JobToken { + owned: Option, + pool: Sender>, + should_return_to_queue: bool, + } + impl Drop for JobToken { + fn drop(&mut self) { + if self.should_return_to_queue { + let _ = self.pool.send(self.owned.take()); + } + } + } + struct JobTokenServer { + helper: jobserver::HelperThread, + tx: Sender>, + rx: Receiver>, + } + impl JobTokenServer { + fn new(client: Client) -> Result { + let (tx, rx) = std::sync::mpsc::channel(); + tx.send(None).unwrap(); + let pool = tx.clone(); + let helper = client.into_helper_thread(move |acq| { + let _ = pool.send(Some(acq.unwrap())); + })?; + Ok(Self { helper, tx, rx }) + } + fn acquire(&mut self) -> JobToken { + if let Ok(token) = self.rx.try_recv() { + JobToken { + owned: token, + pool: self.tx.clone(), + should_return_to_queue: true, + } + } else { + self.helper.request_token(); + let token = self.rx.recv().unwrap(); + JobToken { + owned: token, + pool: self.tx.clone(), + should_return_to_queue: true, + } + } + } + } // When compiling objects in parallel we do a few dirty tricks to speed // things up: // @@ -1332,7 +1381,7 @@ impl Build { // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, _)>(); + let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, JobToken)>(); // Since jobserver::Client::acquire can block, waiting // must be done in parallel so that acquire won't block forever. @@ -1345,7 +1394,7 @@ impl Build { loop { let mut has_made_progress = false; - + let mut is_disconnected = false; // Reading new pending tasks loop { match rx.try_recv() { @@ -1361,14 +1410,21 @@ impl Build { Ok(()) }; } + Err(mpsc::TryRecvError::Disconnected) => { + is_disconnected = true; + break; + } _ => break, } } // Try waiting on them. - pendings.retain_mut(|(cmd, program, child, _)| { + pendings.retain_mut(|(cmd, program, child, token)| { match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { Ok(Some(())) => { + if is_disconnected { + token.should_return_to_queue = false; + } // Task done, remove the entry has_made_progress = true; false @@ -1377,7 +1433,9 @@ impl Build { Err(err) => { // Task fail, remove the entry. has_made_progress = true; - + if is_disconnected { + token.should_return_to_queue = false; + } // Since we can only return one error, log the error to make // sure users always see all the compilation failures. let _ = writeln!(stdout, "cargo:warning={}", err); @@ -1415,11 +1473,10 @@ impl Build { }; } })?; - + let mut tokens = JobTokenServer::new(server)?; for obj in objs { let (mut cmd, program) = self.create_compile_object_cmd(obj)?; - let token = server.acquire()?; - + let token = tokens.acquire(); let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; tx.send((cmd, program, KillOnDrop(child), token)) From d0a8a49fa8f5ece55d2685e7d06cb6ad479019c6 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sat, 30 Sep 2023 02:29:00 +0200 Subject: [PATCH 114/138] Move implementation to a separate module and document a bunch --- src/job_token.rs | 66 ++++++++++++++++++++++++++++++ src/lib.rs | 104 +++++++---------------------------------------- 2 files changed, 81 insertions(+), 89 deletions(-) create mode 100644 src/job_token.rs diff --git a/src/job_token.rs b/src/job_token.rs new file mode 100644 index 000000000..4c3a51e3e --- /dev/null +++ b/src/job_token.rs @@ -0,0 +1,66 @@ +use jobserver::{Acquired, Client, HelperThread}; +use std::sync::mpsc::{self, Receiver, Sender}; + +pub(crate) struct JobToken { + /// The token can either be a fresh token obtained from the jobserver or - if `token` is None - an implicit token for this process. + /// Both are valid values to put into queue. + token: Option, + pool: Sender>, + should_return_to_queue: bool, +} + +impl Drop for JobToken { + fn drop(&mut self) { + if self.should_return_to_queue { + let _ = self.pool.send(self.token.take()); + } + } +} + +impl JobToken { + /// Ensure that this token is not put back into queue once it's dropped. + /// This also leads to releasing it sooner for other processes to use, which is a good thing to do once you know that + /// you're never going to request a token in this process again. + pub(crate) fn forget(&mut self) { + self.should_return_to_queue = false; + } +} + +/// A thin wrapper around jobserver's Client. +/// It would be perfectly fine to just use that, but we also want to reuse our own implicit token assigned for this build script. +/// This struct manages that and gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. +/// Furthermore, instead of giving up job tokens, it keeps them around for reuse if we know we're going to request another token after freeing the current one. +pub(crate) struct JobTokenServer { + helper: HelperThread, + tx: Sender>, + rx: Receiver>, +} + +impl JobTokenServer { + pub(crate) fn new(client: Client) -> Result { + let (tx, rx) = mpsc::channel(); + // Initialize the + tx.send(None).unwrap(); + let pool = tx.clone(); + let helper = client.into_helper_thread(move |acq| { + let _ = pool.send(Some(acq.unwrap())); + })?; + Ok(Self { helper, tx, rx }) + } + + pub(crate) fn acquire(&mut self) -> JobToken { + let token = if let Ok(token) = self.rx.try_recv() { + // Opportunistically check if we already have a token for our own reuse. + token + } else { + // Cold path, request a token and block + self.helper.request_token(); + self.rx.recv().unwrap() + }; + JobToken { + token, + pool: self.tx.clone(), + should_return_to_queue: true, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 384b2b3ba..7ffdfcd6c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,8 +66,9 @@ use std::process::{Child, Command, Stdio}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; +#[cfg(feature = "parallel")] +mod job_token; mod os_pipe; - // These modules are all glue to support reading the MSVC version from // the registry and from COM interfaces #[cfg(windows)] @@ -1293,12 +1294,7 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::{ - mpsc::{self, Receiver, Sender}, - Once, - }; - - use jobserver::{Acquired, Client}; + use std::sync::mpsc; if objs.len() <= 1 { for obj in objs { @@ -1309,59 +1305,10 @@ impl Build { return Ok(()); } - // Limit our parallelism globally with a jobserver. Start off by - // releasing our own token for this process so we can have a bit of an - // easier to write loop below. If this fails, though, then we're likely - // on Windows with the main implicit token, so we just have a bit extra - // parallelism for a bit and don't reacquire later. - let server = jobserver().clone(); + // Limit our parallelism globally with a jobserver. + let server = unsafe { default_jobserver() }; // Reacquire our process's token on drop - //let _reacquire = server.release_raw().ok().map(|_| JobserverToken(server)); - struct JobToken { - owned: Option, - pool: Sender>, - should_return_to_queue: bool, - } - impl Drop for JobToken { - fn drop(&mut self) { - if self.should_return_to_queue { - let _ = self.pool.send(self.owned.take()); - } - } - } - struct JobTokenServer { - helper: jobserver::HelperThread, - tx: Sender>, - rx: Receiver>, - } - impl JobTokenServer { - fn new(client: Client) -> Result { - let (tx, rx) = std::sync::mpsc::channel(); - tx.send(None).unwrap(); - let pool = tx.clone(); - let helper = client.into_helper_thread(move |acq| { - let _ = pool.send(Some(acq.unwrap())); - })?; - Ok(Self { helper, tx, rx }) - } - fn acquire(&mut self) -> JobToken { - if let Ok(token) = self.rx.try_recv() { - JobToken { - owned: token, - pool: self.tx.clone(), - should_return_to_queue: true, - } - } else { - self.helper.request_token(); - let token = self.rx.recv().unwrap(); - JobToken { - owned: token, - pool: self.tx.clone(), - should_return_to_queue: true, - } - } - } - } + // When compiling objects in parallel we do a few dirty tricks to speed // things up: // @@ -1381,7 +1328,7 @@ impl Build { // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, JobToken)>(); + let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, crate::job_token::JobToken)>(); // Since jobserver::Client::acquire can block, waiting // must be done in parallel so that acquire won't block forever. @@ -1394,6 +1341,10 @@ impl Build { loop { let mut has_made_progress = false; + // If the other end of the pipe is already disconnected, then we're not gonna get any new jobs, + // so it doesn't make sense to reuse the tokens; in fact, releasing them as soon as possible (once we know that the other end is disconnected) is beneficial. + // Imagine that the last file built takes an hour to finish; in this scenario, by not releasing the tokens before other builds are done we'd effectively block other processes from + // starting sooner - even though we only need one token, not however many we've acquired. let mut is_disconnected = false; // Reading new pending tasks loop { @@ -1422,10 +1373,10 @@ impl Build { pendings.retain_mut(|(cmd, program, child, token)| { match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { Ok(Some(())) => { + // Task done, remove the entry if is_disconnected { - token.should_return_to_queue = false; + token.forget(); } - // Task done, remove the entry has_made_progress = true; false } @@ -1434,7 +1385,7 @@ impl Build { // Task fail, remove the entry. has_made_progress = true; if is_disconnected { - token.should_return_to_queue = false; + token.forget(); } // Since we can only return one error, log the error to make // sure users always see all the compilation failures. @@ -1473,7 +1424,7 @@ impl Build { }; } })?; - let mut tokens = JobTokenServer::new(server)?; + let mut tokens = crate::job_token::JobTokenServer::new(server)?; for obj in objs { let (mut cmd, program) = self.create_compile_object_cmd(obj)?; let token = tokens.acquire(); @@ -1487,24 +1438,6 @@ impl Build { return wait_thread.join().expect("wait_thread panics"); - /// Returns a suitable `jobserver::Client` used to coordinate - /// parallelism between build scripts. - fn jobserver() -> &'static jobserver::Client { - static INIT: Once = Once::new(); - static mut JOBSERVER: Option = None; - - fn _assert_sync() {} - _assert_sync::(); - - unsafe { - INIT.call_once(|| { - let server = default_jobserver(); - JOBSERVER = Some(server); - }); - JOBSERVER.as_ref().unwrap() - } - } - unsafe fn default_jobserver() -> jobserver::Client { // Try to use the environmental jobserver which Cargo typically // initializes for us... @@ -1541,13 +1474,6 @@ impl Build { child.kill().ok(); } } - - struct JobserverToken(&'static jobserver::Client); - impl Drop for JobserverToken { - fn drop(&mut self) { - let _ = self.0.acquire_raw(); - } - } } #[cfg(not(feature = "parallel"))] From ee5dee70fe58966abc45364a101c03d95ecc5881 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sat, 30 Sep 2023 02:37:52 +0200 Subject: [PATCH 115/138] Readd reusable global jobserver connection --- src/lib.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7ffdfcd6c..cf180d134 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1294,7 +1294,7 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::mpsc; + use std::sync::{mpsc, Once}; if objs.len() <= 1 { for obj in objs { @@ -1306,7 +1306,7 @@ impl Build { } // Limit our parallelism globally with a jobserver. - let server = unsafe { default_jobserver() }; + let server = jobserver(); // Reacquire our process's token on drop // When compiling objects in parallel we do a few dirty tricks to speed @@ -1438,6 +1438,24 @@ impl Build { return wait_thread.join().expect("wait_thread panics"); + /// Returns a suitable `jobserver::Client` used to coordinate + /// parallelism between build scripts. + fn jobserver() -> jobserver::Client { + static INIT: Once = Once::new(); + static mut JOBSERVER: Option = None; + + fn _assert_sync() {} + _assert_sync::(); + + unsafe { + INIT.call_once(|| { + let server = default_jobserver(); + JOBSERVER = Some(server); + }); + JOBSERVER.clone().unwrap() + } + } + unsafe fn default_jobserver() -> jobserver::Client { // Try to use the environmental jobserver which Cargo typically // initializes for us... From 36371277d24d7a8baecd77f8bb951eda8aee5479 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sat, 30 Sep 2023 02:43:04 +0200 Subject: [PATCH 116/138] Fix up a comment --- src/job_token.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/job_token.rs b/src/job_token.rs index 4c3a51e3e..afb97cde4 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -39,7 +39,8 @@ pub(crate) struct JobTokenServer { impl JobTokenServer { pub(crate) fn new(client: Client) -> Result { let (tx, rx) = mpsc::channel(); - // Initialize the + // Push the implicit token. Since JobTokens only give back what they got, + // there should be at most one global implicit token in the wild. tx.send(None).unwrap(); let pool = tx.clone(); let helper = client.into_helper_thread(move |acq| { From 4ca780838a8f20f5e954f70f2be7a626276754e9 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sat, 30 Sep 2023 03:02:49 +0200 Subject: [PATCH 117/138] Further documentation refinements --- src/job_token.rs | 15 +++++++++------ src/lib.rs | 8 +++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index afb97cde4..28ec43f8a 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -19,17 +19,20 @@ impl Drop for JobToken { impl JobToken { /// Ensure that this token is not put back into queue once it's dropped. - /// This also leads to releasing it sooner for other processes to use, which is a good thing to do once you know that - /// you're never going to request a token in this process again. + /// This also leads to releasing it sooner for other processes to use, + /// which is a correct thing to do once it is known that there won't be + /// any more token acquisitions. pub(crate) fn forget(&mut self) { self.should_return_to_queue = false; } } /// A thin wrapper around jobserver's Client. -/// It would be perfectly fine to just use that, but we also want to reuse our own implicit token assigned for this build script. -/// This struct manages that and gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. -/// Furthermore, instead of giving up job tokens, it keeps them around for reuse if we know we're going to request another token after freeing the current one. +/// It would be perfectly fine to just use jobserver's Client, but we also want to reuse +/// our own implicit token assigned for this build script. This struct manages that and +/// gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. +/// Furthermore, instead of giving up job tokens, it keeps them around +/// for reuse if we know we're going to request another token after freeing the current one. pub(crate) struct JobTokenServer { helper: HelperThread, tx: Sender>, @@ -51,7 +54,7 @@ impl JobTokenServer { pub(crate) fn acquire(&mut self) -> JobToken { let token = if let Ok(token) = self.rx.try_recv() { - // Opportunistically check if we already have a token for our own reuse. + // Opportunistically check if there's a token that can be reused. token } else { // Cold path, request a token and block diff --git a/src/lib.rs b/src/lib.rs index cf180d134..ed7d1bf66 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1342,9 +1342,11 @@ impl Build { loop { let mut has_made_progress = false; // If the other end of the pipe is already disconnected, then we're not gonna get any new jobs, - // so it doesn't make sense to reuse the tokens; in fact, releasing them as soon as possible (once we know that the other end is disconnected) is beneficial. - // Imagine that the last file built takes an hour to finish; in this scenario, by not releasing the tokens before other builds are done we'd effectively block other processes from - // starting sooner - even though we only need one token, not however many we've acquired. + // so it doesn't make sense to reuse the tokens; in fact, + // releasing them as soon as possible (once we know that the other end is disconnected) is beneficial. + // Imagine that the last file built takes an hour to finish; in this scenario, + // by not releasing the tokens before that last file is done we would effectively block other processes from + // starting sooner - even though we only need one token for that last file, not N others that were acquired. let mut is_disconnected = false; // Reading new pending tasks loop { From 1b025d810d87258fa2af1f9043b9bf99ab7ae326 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sun, 1 Oct 2023 18:33:01 +0200 Subject: [PATCH 118/138] Move jobserver func into job_token module --- src/job_token.rs | 53 +++++++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 49 ++------------------------------------------ 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index 28ec43f8a..9f0518f41 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -1,5 +1,11 @@ use jobserver::{Acquired, Client, HelperThread}; -use std::sync::mpsc::{self, Receiver, Sender}; +use std::{ + env, + sync::{ + mpsc::{self, Receiver, Sender}, + Once, + }, +}; pub(crate) struct JobToken { /// The token can either be a fresh token obtained from the jobserver or - if `token` is None - an implicit token for this process. @@ -68,3 +74,48 @@ impl JobTokenServer { } } } + +/// Returns a suitable `jobserver::Client` used to coordinate +/// parallelism between build scripts. +pub(super) fn jobserver() -> jobserver::Client { + static INIT: Once = Once::new(); + static mut JOBSERVER: Option = None; + + fn _assert_sync() {} + _assert_sync::(); + + unsafe { + INIT.call_once(|| { + let server = default_jobserver(); + JOBSERVER = Some(server); + }); + JOBSERVER.clone().unwrap() + } +} + +unsafe fn default_jobserver() -> jobserver::Client { + // Try to use the environmental jobserver which Cargo typically + // initializes for us... + if let Some(client) = jobserver::Client::from_env() { + return client; + } + + // ... but if that fails for whatever reason select something + // reasonable and crate a new jobserver. Use `NUM_JOBS` if set (it's + // configured by Cargo) and otherwise just fall back to a + // semi-reasonable number. Note that we could use `num_cpus` here + // but it's an extra dependency that will almost never be used, so + // it's generally not too worth it. + let mut parallelism = 4; + if let Ok(amt) = env::var("NUM_JOBS") { + if let Ok(amt) = amt.parse() { + parallelism = amt; + } + } + + // If we create our own jobserver then be sure to reserve one token + // for ourselves. + let client = jobserver::Client::new(parallelism).expect("failed to create jobserver"); + client.acquire_raw().expect("failed to acquire initial"); + return client; +} diff --git a/src/lib.rs b/src/lib.rs index ed7d1bf66..4be136757 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1294,7 +1294,7 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::{mpsc, Once}; + use std::sync::mpsc; if objs.len() <= 1 { for obj in objs { @@ -1306,7 +1306,7 @@ impl Build { } // Limit our parallelism globally with a jobserver. - let server = jobserver(); + let server = job_token::jobserver(); // Reacquire our process's token on drop // When compiling objects in parallel we do a few dirty tricks to speed @@ -1440,51 +1440,6 @@ impl Build { return wait_thread.join().expect("wait_thread panics"); - /// Returns a suitable `jobserver::Client` used to coordinate - /// parallelism between build scripts. - fn jobserver() -> jobserver::Client { - static INIT: Once = Once::new(); - static mut JOBSERVER: Option = None; - - fn _assert_sync() {} - _assert_sync::(); - - unsafe { - INIT.call_once(|| { - let server = default_jobserver(); - JOBSERVER = Some(server); - }); - JOBSERVER.clone().unwrap() - } - } - - unsafe fn default_jobserver() -> jobserver::Client { - // Try to use the environmental jobserver which Cargo typically - // initializes for us... - if let Some(client) = jobserver::Client::from_env() { - return client; - } - - // ... but if that fails for whatever reason select something - // reasonable and crate a new jobserver. Use `NUM_JOBS` if set (it's - // configured by Cargo) and otherwise just fall back to a - // semi-reasonable number. Note that we could use `num_cpus` here - // but it's an extra dependency that will almost never be used, so - // it's generally not too worth it. - let mut parallelism = 4; - if let Ok(amt) = env::var("NUM_JOBS") { - if let Ok(amt) = amt.parse() { - parallelism = amt; - } - } - - // If we create our own jobserver then be sure to reserve one token - // for ourselves. - let client = jobserver::Client::new(parallelism).expect("failed to create jobserver"); - client.acquire_raw().expect("failed to acquire initial"); - return client; - } - struct KillOnDrop(Child); impl Drop for KillOnDrop { From 52afaf1080f1032ad0e44c07501f7517f7403ebf Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Sun, 1 Oct 2023 18:39:10 +0200 Subject: [PATCH 119/138] Remove should_return_to_queue member in favor of wrapping pool in an Option --- src/job_token.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index 9f0518f41..c485f8bed 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -11,14 +11,15 @@ pub(crate) struct JobToken { /// The token can either be a fresh token obtained from the jobserver or - if `token` is None - an implicit token for this process. /// Both are valid values to put into queue. token: Option, - pool: Sender>, - should_return_to_queue: bool, + /// A pool to which `token` should be returned. `pool` is optional, as one might want to release a token straight away instead + /// of storing it back in the pool - see [`Self::forget()`] function for that. + pool: Option>>, } impl Drop for JobToken { fn drop(&mut self) { - if self.should_return_to_queue { - let _ = self.pool.send(self.token.take()); + if let Some(pool) = &self.pool { + let _ = pool.send(self.token.take()); } } } @@ -29,7 +30,7 @@ impl JobToken { /// which is a correct thing to do once it is known that there won't be /// any more token acquisitions. pub(crate) fn forget(&mut self) { - self.should_return_to_queue = false; + self.pool.take(); } } @@ -69,8 +70,7 @@ impl JobTokenServer { }; JobToken { token, - pool: self.tx.clone(), - should_return_to_queue: true, + pool: Some(self.tx.clone()), } } } From 3d458416426402d403b1ff41d61b53625cfc39f8 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:28:03 +0200 Subject: [PATCH 120/138] Make jobserver initialization private in job_token mod --- src/job_token.rs | 7 ++++--- src/lib.rs | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index c485f8bed..f5e69b8f7 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -1,4 +1,4 @@ -use jobserver::{Acquired, Client, HelperThread}; +use jobserver::{Acquired, HelperThread}; use std::{ env, sync::{ @@ -47,7 +47,8 @@ pub(crate) struct JobTokenServer { } impl JobTokenServer { - pub(crate) fn new(client: Client) -> Result { + pub(crate) fn new() -> Result { + let client = jobserver(); let (tx, rx) = mpsc::channel(); // Push the implicit token. Since JobTokens only give back what they got, // there should be at most one global implicit token in the wild. @@ -77,7 +78,7 @@ impl JobTokenServer { /// Returns a suitable `jobserver::Client` used to coordinate /// parallelism between build scripts. -pub(super) fn jobserver() -> jobserver::Client { +fn jobserver() -> jobserver::Client { static INIT: Once = Once::new(); static mut JOBSERVER: Option = None; diff --git a/src/lib.rs b/src/lib.rs index 4be136757..d3ed7dc8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1306,8 +1306,7 @@ impl Build { } // Limit our parallelism globally with a jobserver. - let server = job_token::jobserver(); - // Reacquire our process's token on drop + let mut tokens = crate::job_token::JobTokenServer::new()?; // When compiling objects in parallel we do a few dirty tricks to speed // things up: @@ -1426,7 +1425,6 @@ impl Build { }; } })?; - let mut tokens = crate::job_token::JobTokenServer::new(server)?; for obj in objs { let (mut cmd, program) = self.create_compile_object_cmd(obj)?; let token = tokens.acquire(); From 280c673b1e0d31bfe381a93e4af4f25dba061686 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:01:05 +0200 Subject: [PATCH 121/138] Remove unnecessary mut on acquire fn --- src/job_token.rs | 4 ++-- src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index f5e69b8f7..9deaf82e8 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -40,7 +40,7 @@ impl JobToken { /// gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. /// Furthermore, instead of giving up job tokens, it keeps them around /// for reuse if we know we're going to request another token after freeing the current one. -pub(crate) struct JobTokenServer { +pub(crate) struct GlobalJobTokenServer { helper: HelperThread, tx: Sender>, rx: Receiver>, @@ -60,7 +60,7 @@ impl JobTokenServer { Ok(Self { helper, tx, rx }) } - pub(crate) fn acquire(&mut self) -> JobToken { + pub(crate) fn acquire(&self) -> JobToken { let token = if let Ok(token) = self.rx.try_recv() { // Opportunistically check if there's a token that can be reused. token diff --git a/src/lib.rs b/src/lib.rs index d3ed7dc8f..d2cac31e6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1306,7 +1306,7 @@ impl Build { } // Limit our parallelism globally with a jobserver. - let mut tokens = crate::job_token::JobTokenServer::new()?; + let tokens = crate::job_token::JobTokenServer::new()?; // When compiling objects in parallel we do a few dirty tricks to speed // things up: From d55c382da734a0198f8e92bbeddec0314fb1b0b9 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:15:06 +0200 Subject: [PATCH 122/138] Use shared global JobTokenServer --- src/job_token.rs | 28 ++++++++++++++++++---------- src/lib.rs | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index 9deaf82e8..f1c218875 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -1,4 +1,4 @@ -use jobserver::{Acquired, HelperThread}; +use jobserver::{Acquired, Client, HelperThread}; use std::{ env, sync::{ @@ -40,15 +40,17 @@ impl JobToken { /// gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. /// Furthermore, instead of giving up job tokens, it keeps them around /// for reuse if we know we're going to request another token after freeing the current one. -pub(crate) struct GlobalJobTokenServer { +pub(crate) struct JobTokenServer { helper: HelperThread, tx: Sender>, rx: Receiver>, } impl JobTokenServer { - pub(crate) fn new() -> Result { - let client = jobserver(); + pub(crate) fn new() -> &'static Self { + jobserver() + } + fn new_inner(client: Client) -> Result { let (tx, rx) = mpsc::channel(); // Push the implicit token. Since JobTokens only give back what they got, // there should be at most one global implicit token in the wild. @@ -76,11 +78,16 @@ impl JobTokenServer { } } -/// Returns a suitable `jobserver::Client` used to coordinate -/// parallelism between build scripts. -fn jobserver() -> jobserver::Client { +/// Returns a suitable `JobTokenServer` used to coordinate +/// parallelism between build scripts. A global `JobTokenServer` is used as this ensures +/// that only one implicit job token is used in the wild. +/// Having multiple separate job token servers would lead to each of them assuming that they have control +/// over the implicit job token. +/// As it stands, each caller of `jobserver` can receive an implicit job token and there will be at most +/// one implicit job token in the wild. +fn jobserver() -> &'static JobTokenServer { static INIT: Once = Once::new(); - static mut JOBSERVER: Option = None; + static mut JOBSERVER: Option = None; fn _assert_sync() {} _assert_sync::(); @@ -88,9 +95,10 @@ fn jobserver() -> jobserver::Client { unsafe { INIT.call_once(|| { let server = default_jobserver(); - JOBSERVER = Some(server); + JOBSERVER = + Some(JobTokenServer::new_inner(server).expect("Job server initialization failed")); }); - JOBSERVER.clone().unwrap() + JOBSERVER.as_ref().unwrap() } } diff --git a/src/lib.rs b/src/lib.rs index d2cac31e6..93915f6eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1306,7 +1306,7 @@ impl Build { } // Limit our parallelism globally with a jobserver. - let tokens = crate::job_token::JobTokenServer::new()?; + let tokens = crate::job_token::JobTokenServer::new(); // When compiling objects in parallel we do a few dirty tricks to speed // things up: From 985dbaeb928093c47402133b9be3402554372fe4 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:00:33 +0200 Subject: [PATCH 123/138] Change Option to MaybeUninit --- src/job_token.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index f1c218875..006586bf3 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -1,6 +1,7 @@ use jobserver::{Acquired, Client, HelperThread}; use std::{ env, + mem::MaybeUninit, sync::{ mpsc::{self, Receiver, Sender}, Once, @@ -87,7 +88,7 @@ impl JobTokenServer { /// one implicit job token in the wild. fn jobserver() -> &'static JobTokenServer { static INIT: Once = Once::new(); - static mut JOBSERVER: Option = None; + static mut JOBSERVER: MaybeUninit = MaybeUninit::uninit(); fn _assert_sync() {} _assert_sync::(); @@ -95,10 +96,12 @@ fn jobserver() -> &'static JobTokenServer { unsafe { INIT.call_once(|| { let server = default_jobserver(); - JOBSERVER = - Some(JobTokenServer::new_inner(server).expect("Job server initialization failed")); + JOBSERVER = MaybeUninit::new( + JobTokenServer::new_inner(server).expect("Job server initialization failed"), + ); }); - JOBSERVER.as_ref().unwrap() + // Poor man's assume_init_ref, as that'd require a MSRV of 1.55. + &*JOBSERVER.as_ptr() } } From 65ab3711cffba13dd5188ae4fbcc1bd3c37ef5d5 Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:10:40 +0200 Subject: [PATCH 124/138] Convert internal channel to use Result and do not panic in helper thread --- src/job_token.rs | 22 ++++++++++++++-------- src/lib.rs | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/job_token.rs b/src/job_token.rs index 006586bf3..818917c8d 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -14,13 +14,14 @@ pub(crate) struct JobToken { token: Option, /// A pool to which `token` should be returned. `pool` is optional, as one might want to release a token straight away instead /// of storing it back in the pool - see [`Self::forget()`] function for that. - pool: Option>>, + pool: Option>>>, } impl Drop for JobToken { fn drop(&mut self) { if let Some(pool) = &self.pool { - let _ = pool.send(self.token.take()); + // Always send back an Ok() variant as we know that the acquisition for this token has succeeded. + let _ = pool.send(self.token.take().map(|token| Ok(token))); } } } @@ -43,8 +44,8 @@ impl JobToken { /// for reuse if we know we're going to request another token after freeing the current one. pub(crate) struct JobTokenServer { helper: HelperThread, - tx: Sender>, - rx: Receiver>, + tx: Sender>>, + rx: Receiver>>, } impl JobTokenServer { @@ -58,12 +59,12 @@ impl JobTokenServer { tx.send(None).unwrap(); let pool = tx.clone(); let helper = client.into_helper_thread(move |acq| { - let _ = pool.send(Some(acq.unwrap())); + let _ = pool.send(Some(acq.map_err(|e| e.into()))); })?; Ok(Self { helper, tx, rx }) } - pub(crate) fn acquire(&self) -> JobToken { + pub(crate) fn acquire(&self) -> Result { let token = if let Ok(token) = self.rx.try_recv() { // Opportunistically check if there's a token that can be reused. token @@ -72,10 +73,15 @@ impl JobTokenServer { self.helper.request_token(); self.rx.recv().unwrap() }; - JobToken { + let token = if let Some(token) = token { + Some(token?) + } else { + None + }; + Ok(JobToken { token, pool: Some(self.tx.clone()), - } + }) } } diff --git a/src/lib.rs b/src/lib.rs index 93915f6eb..21d4e0e45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1427,7 +1427,7 @@ impl Build { })?; for obj in objs { let (mut cmd, program) = self.create_compile_object_cmd(obj)?; - let token = tokens.acquire(); + let token = tokens.acquire()?; let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; tx.send((cmd, program, KillOnDrop(child), token)) From 53564e00498156c9be00361c4b039952cbf7ff59 Mon Sep 17 00:00:00 2001 From: Arc-blroth <45273859+Arc-blroth@users.noreply.github.com> Date: Wed, 27 Jul 2022 13:42:07 -0700 Subject: [PATCH 125/138] Fix `--target` getting passed twice to the Android NDK clang on Windows --- src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6257fa0df..219d82cd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,6 +200,7 @@ pub struct Tool { family: ToolFamily, cuda: bool, removed_args: Vec, + has_internal_target_arg: bool, } /// Represents the family of tools this tool belongs to. @@ -1824,9 +1825,7 @@ impl Build { // Target flags match cmd.family { ToolFamily::Clang => { - if !(target.contains("android") - && android_clang_compiler_uses_target_arg_internally(&cmd.path)) - { + if !(target.contains("android") && cmd.has_internal_target_arg) { if target.contains("darwin") { if let Some(arch) = map_darwin_target_from_rust_to_compiler_architecture(target) @@ -2696,6 +2695,7 @@ impl Build { let file_name = path.to_str().unwrap().to_owned(); let (target, clang) = file_name.split_at(file_name.rfind("-").unwrap()); + tool.has_internal_target_arg = true; tool.path.set_file_name(clang.trim_start_matches("-")); tool.path.set_extension("exe"); tool.args.push(format!("--target={}", target).into()); @@ -3522,6 +3522,7 @@ impl Tool { family: family, cuda: false, removed_args: Vec::new(), + has_internal_target_arg: false, } } @@ -3582,6 +3583,7 @@ impl Tool { family: family, cuda: cuda, removed_args: Vec::new(), + has_internal_target_arg: false, } } From 2adc336e1c95704dc8e6af5a455d915cba6b186a Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sat, 21 Oct 2023 17:00:20 +1100 Subject: [PATCH 126/138] Fix MSRV: Remove use of `Vec::retain_mut` And replace that with our own implementation of `retain_unordered_mut` which is usable under any rust version after v1.0 Signed-off-by: Jiahao XU --- src/lib.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 5b6a37ccb..32ca00630 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1372,7 +1372,7 @@ impl Build { } // Try waiting on them. - pendings.retain_mut(|(cmd, program, child, token)| { + retain_unordered_mut(&mut pendings, |(cmd, program, child, token)| { match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { Ok(Some(())) => { // Task done, remove the entry @@ -4127,3 +4127,19 @@ impl Drop for PrintThread { self.handle.take().unwrap().join().unwrap(); } } + +/// Remove all element in `vec` which `f(element)` returns `false`. +#[cfg(feature = "parallel")] +fn retain_unordered_mut(vec: &mut Vec, mut f: F) +where + F: FnMut(&mut T) -> bool, +{ + let mut i = 0; + while i < vec.len() { + if f(&mut vec[i]) { + i += 1; + } else { + vec.swap_remove(i); + } + } +} From fd2330cf0e5635d258c9956004933bc825dca49e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sat, 21 Oct 2023 17:17:02 +1100 Subject: [PATCH 127/138] Add a TODO for `retain_unordered_mut` It can be removed once cc MSRV is bumped to 1.61 Signed-off-by: Jiahao XU --- src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 32ca00630..c13ccd125 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4129,6 +4129,8 @@ impl Drop for PrintThread { } /// Remove all element in `vec` which `f(element)` returns `false`. +/// +/// TODO: Remove this once the MSRV is bumped to v1.61 #[cfg(feature = "parallel")] fn retain_unordered_mut(vec: &mut Vec, mut f: F) where From 9b569aeea214f21a7b6015d3ea71f86559334553 Mon Sep 17 00:00:00 2001 From: Silas Groh Date: Fri, 3 Nov 2023 22:26:34 +0800 Subject: [PATCH 128/138] fix(flag_check): never link to avoid false positives (#839) --- src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c13ccd125..ecc5d48f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -539,11 +539,8 @@ impl Build { is_arm, ); - // We need to explicitly tell msvc not to link and create an exe - // in the root directory of the crate - if target.contains("msvc") && !self.cuda { - cmd.arg("-c"); - } + // Checking for compiler flags does not require linking + cmd.arg("-c"); cmd.arg(&src); From 59255ceac58f7211635bdbbd41e66fcd756a90c7 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 3 Nov 2023 15:17:02 -0700 Subject: [PATCH 129/138] Pass -Wno-unused-command-line-argument to clang in `is_flag_supported` (#886) --- src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index ecc5d48f5..77f1a1db1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -523,6 +523,11 @@ impl Build { if compiler.family.verbose_stderr() { compiler.remove_arg("-v".into()); } + if compiler.family == ToolFamily::Clang { + // Avoid reporting that the arg is unsupported just because the + // compiler complains that it wasn't used. + compiler.push_cc_arg("-Wno-unused-command-line-argument".into()); + } let mut cmd = compiler.to_command(); let is_arm = target.contains("aarch64") || target.contains("arm"); From 6c9be16e97bc18390e50f15f8156fcaeffd4abc7 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 5 Nov 2023 09:19:21 +1000 Subject: [PATCH 130/138] Fix msrv CI: Check for `--all-features` (#890) * Fix msrv CI: Check for `--all-features` and macos To make sure that enabling `parallel` feature would not require bumping msrv. Signed-off-by: Jiahao XU * Use minimal-versions and fetch deps crates using stable Signed-off-by: Jiahao XU * Fix fetching deps in msrv Signed-off-by: Jiahao XU * Cache downlaoded crates since 1.46 is really slow in fetching Signed-off-by: Jiahao XU * Skip msrv for macos-latest since newer xcode breaks 1.46 linking Signed-off-by: Jiahao XU * CI: Disable fail-fast for msrv check Signed-off-by: Jiahao XU * CI: Add feature to cache key of msrv Signed-off-by: Jiahao XU * CI: Rm matrix features of msrv Since most of the time is spent on installing Rust, there's no point in parallelizing it. Signed-off-by: Jiahao XU --------- Signed-off-by: Jiahao XU --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7aef82c15..4dff9b2d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -114,6 +114,7 @@ jobs: name: MSRV runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] steps: @@ -121,9 +122,15 @@ jobs: - name: Install Rust run: | rustup toolchain install 1.46.0 --no-self-update --profile minimal + rustup toolchain install nightly --no-self-update --profile minimal rustup default 1.46.0 shell: bash + - name: Create Cargo.lock with minimal version + run: cargo +nightly update -Zminimal-versions + - name: Cache downloaded crates since 1.46 is really slow in fetching + uses: Swatinem/rust-cache@v2 - run: cargo check --lib + - run: cargo check --lib --all-features rustfmt: name: Rustfmt From 022713ca4e3de0229473fc47b42d2eaaad37e503 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 5 Nov 2023 10:29:31 +1100 Subject: [PATCH 131/138] Bump MSRV to 1.53.0 and re-enable macos MSRV check Signed-off-by: Jiahao XU --- .github/workflows/main.yml | 6 +++--- Cargo.toml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dff9b2d4..724b9a618 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,14 +116,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - name: Install Rust run: | - rustup toolchain install 1.46.0 --no-self-update --profile minimal + rustup toolchain install 1.53.0 --no-self-update --profile minimal rustup toolchain install nightly --no-self-update --profile minimal - rustup default 1.46.0 + rustup default 1.53.0 shell: bash - name: Create Cargo.lock with minimal version run: cargo +nightly update -Zminimal-versions diff --git a/Cargo.toml b/Cargo.toml index 89945c56f..c7ba95a00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ readme = "README.md" categories = ["development-tools::build-utils"] exclude = ["/.github"] edition = "2018" +rust-version = "1.53" [dependencies] jobserver = { version = "0.1.16", optional = true } From 3ff23685f8c797e3fc3e601ebe979475dc09c20c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 5 Nov 2023 10:36:54 +1100 Subject: [PATCH 132/138] Disable msrv checking on macos-latest Since 1.53 still can't link on latest macos Signed-off-by: Jiahao XU --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 724b9a618..6f2286952 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,7 +116,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v4 - name: Install Rust @@ -127,7 +127,7 @@ jobs: shell: bash - name: Create Cargo.lock with minimal version run: cargo +nightly update -Zminimal-versions - - name: Cache downloaded crates since 1.46 is really slow in fetching + - name: Cache downloaded crates since 1.53 is really slow in fetching uses: Swatinem/rust-cache@v2 - run: cargo check --lib - run: cargo check --lib --all-features From fcedb005d38e7a1917a20c8d921de7f30c31c6d9 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Sun, 12 Nov 2023 08:49:51 +1000 Subject: [PATCH 133/138] Optimization: Vendor jobserver impl and rm thread spawning in parallel compile_objects (#889) * Impl vendored jobserver implementation It supports non-blocking `try_acquire` and is much simpler than the one provided by `jobserver` Signed-off-by: Jiahao XU * Convert parallel `compile_objects` to use future instead of threads Also fixed compilation errors in mod `job_token` Signed-off-by: Jiahao XU * Optimize parallel `compile_objects` Remove use of mpsc since the future is executed on one single thread only. Signed-off-by: Jiahao XU * Fix `job_token`: Remove mpsc and make sure tokens are relased The mpsc is stored in a global variable and Rust never calls `Drop::drop` on global variables, so they are never released. This commit removes the mpsc and replaces that with an `AtomicBool` for the implicit token to fix this, also dramatically simplifies the code. Signed-off-by: Jiahao XU * Optimize `job_token`: Make `JobToken` zero-sized Signed-off-by: Jiahao XU * Fix `windows::JobServerClient::try_acquire` impl Return `Ok(None)` instead of `Err()` if no token is ready. Signed-off-by: Jiahao XU * Fix `unix::JobServerClient::from_pipe`: Accept more fd access modes `O_RDWR` is a valid access mode for both read and write end of the pipe. Signed-off-by: Jiahao XU * Rm unnecessary `'static` bound in parameter of `job_token` Signed-off-by: Jiahao XU * Optimize parallel `compile_objects`: Sleep/yield if no progress is made Signed-off-by: Jiahao XU * Fix windows implementation: Match all return value explicitly Signed-off-by: Jiahao XU * Use Result::ok() in job_token.rs Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> * Fix grammer in comments Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> * simplify job_token impl Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> * Add more comment explaining the design choice Signed-off-by: Jiahao XU * Refactor: Extract new mod `async_executor` Signed-off-by: Jiahao XU * Update src/job_token/unix.rs Co-authored-by: Thom Chiovoloni * Remove outdated comment Signed-off-by: Jiahao XU * Do not check for `--jobserver-fds` on windows Since the manual specifies that only `--jobsewrver-auth` will be used and windows does not have the concept of fds anyway. Signed-off-by: Jiahao XU * Accept ASCII only in windows `JobServerClient::open` impl Signed-off-by: Jiahao XU * Use acquire and release ordering for atomic operation in `JobServer` Signed-off-by: Jiahao XU * Add a TODO for use of `NUM_JOBS` Signed-off-by: Jiahao XU * Simplify windows jobserver `WAIT_ABANDONED` errmsg Signed-off-by: Jiahao XU --------- Signed-off-by: Jiahao XU Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Co-authored-by: Thom Chiovoloni --- Cargo.toml | 5 +- gen-windows-sys-binding/windows_sys.list | 13 ++ src/async_executor.rs | 118 ++++++++++ src/job_token.rs | 264 +++++++++++++---------- src/job_token/unix.rs | 185 ++++++++++++++++ src/job_token/windows.rs | 76 +++++++ src/lib.rs | 163 +++++++------- src/windows_sys.rs | 26 +++ 8 files changed, 644 insertions(+), 206 deletions(-) create mode 100644 src/async_executor.rs create mode 100644 src/job_token/unix.rs create mode 100644 src/job_token/windows.rs diff --git a/Cargo.toml b/Cargo.toml index c7ba95a00..9a6c573cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,16 +18,13 @@ exclude = ["/.github"] edition = "2018" rust-version = "1.53" -[dependencies] -jobserver = { version = "0.1.16", optional = true } - [target.'cfg(unix)'.dependencies] # Don't turn on the feature "std" for this, see https://github.com/rust-lang/cargo/issues/4866 # which is still an issue with `resolver = "1"`. libc = { version = "0.2.62", default-features = false } [features] -parallel = ["jobserver"] +parallel = [] [dev-dependencies] tempfile = "3" diff --git a/gen-windows-sys-binding/windows_sys.list b/gen-windows-sys-binding/windows_sys.list index 82d11c9de..f5ed55e16 100644 --- a/gen-windows-sys-binding/windows_sys.list +++ b/gen-windows-sys-binding/windows_sys.list @@ -6,6 +6,12 @@ Windows.Win32.Foundation.SysFreeString Windows.Win32.Foundation.SysStringLen Windows.Win32.Foundation.S_FALSE Windows.Win32.Foundation.S_OK +Windows.Win32.Foundation.FALSE +Windows.Win32.Foundation.HANDLE +Windows.Win32.Foundation.WAIT_OBJECT_0 +Windows.Win32.Foundation.WAIT_TIMEOUT +Windows.Win32.Foundation.WAIT_FAILED +Windows.Win32.Foundation.WAIT_ABANDONED Windows.Win32.System.Com.SAFEARRAY Windows.Win32.System.Com.SAFEARRAYBOUND @@ -25,3 +31,10 @@ Windows.Win32.System.Registry.HKEY_LOCAL_MACHINE Windows.Win32.System.Registry.KEY_READ Windows.Win32.System.Registry.KEY_WOW64_32KEY Windows.Win32.System.Registry.REG_SZ + +Windows.Win32.System.Threading.ReleaseSemaphore +Windows.Win32.System.Threading.WaitForSingleObject +Windows.Win32.System.Threading.SEMAPHORE_MODIFY_STATE +Windows.Win32.System.Threading.THREAD_SYNCHRONIZE + +Windows.Win32.System.WindowsProgramming.OpenSemaphoreA diff --git a/src/async_executor.rs b/src/async_executor.rs new file mode 100644 index 000000000..ad9e62a65 --- /dev/null +++ b/src/async_executor.rs @@ -0,0 +1,118 @@ +use std::{ + cell::Cell, + future::Future, + pin::Pin, + ptr, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, + thread, + time::Duration, +}; + +use crate::Error; + +const NOOP_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| NOOP_RAW_WAKER, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, +); +const NOOP_RAW_WAKER: RawWaker = RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE); + +#[derive(Default)] +pub(super) struct YieldOnce(bool); + +impl Future for YieldOnce { + type Output = (); + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> { + let flag = &mut std::pin::Pin::into_inner(self).0; + if !*flag { + *flag = true; + Poll::Pending + } else { + Poll::Ready(()) + } + } +} + +/// Execute the futures and return when they are all done. +/// +/// Here we use our own homebrew async executor since cc is used in the build +/// script of many popular projects, pulling in additional dependencies would +/// significantly slow down its compilation. +pub(super) fn block_on( + mut fut1: Fut1, + mut fut2: Fut2, + has_made_progress: &Cell, +) -> Result<(), Error> +where + Fut1: Future>, + Fut2: Future>, +{ + // Shadows the future so that it can never be moved and is guaranteed + // to be pinned. + // + // The same trick used in `pin!` macro. + // + // TODO: Once MSRV is bumped to 1.68, replace this with `std::pin::pin!` + let mut fut1 = Some(unsafe { Pin::new_unchecked(&mut fut1) }); + let mut fut2 = Some(unsafe { Pin::new_unchecked(&mut fut2) }); + + // TODO: Once `Waker::noop` stablised and our MSRV is bumped to the version + // which it is stablised, replace this wth `Waker::noop`. + let waker = unsafe { Waker::from_raw(NOOP_RAW_WAKER) }; + let mut context = Context::from_waker(&waker); + + let mut backoff_cnt = 0; + + loop { + has_made_progress.set(false); + + if let Some(fut) = fut2.as_mut() { + if let Poll::Ready(res) = fut.as_mut().poll(&mut context) { + fut2 = None; + res?; + } + } + + if let Some(fut) = fut1.as_mut() { + if let Poll::Ready(res) = fut.as_mut().poll(&mut context) { + fut1 = None; + res?; + } + } + + if fut1.is_none() && fut2.is_none() { + return Ok(()); + } + + if !has_made_progress.get() { + if backoff_cnt > 3 { + // We have yielded at least three times without making' + // any progress, so we will sleep for a while. + let duration = Duration::from_millis(100 * (backoff_cnt - 3).min(10)); + thread::sleep(duration); + } else { + // Given that we spawned a lot of compilation tasks, it is unlikely + // that OS cannot find other ready task to execute. + // + // If all of them are done, then we will yield them and spawn more, + // or simply return. + // + // Thus this will not be turned into a busy-wait loop and it will not + // waste CPU resource. + thread::yield_now(); + } + } + + backoff_cnt = if has_made_progress.get() { + 0 + } else { + backoff_cnt + 1 + }; + } +} diff --git a/src/job_token.rs b/src/job_token.rs index 818917c8d..4b4fa989e 100644 --- a/src/job_token.rs +++ b/src/job_token.rs @@ -1,139 +1,171 @@ -use jobserver::{Acquired, Client, HelperThread}; -use std::{ - env, - mem::MaybeUninit, - sync::{ - mpsc::{self, Receiver, Sender}, - Once, - }, -}; - -pub(crate) struct JobToken { - /// The token can either be a fresh token obtained from the jobserver or - if `token` is None - an implicit token for this process. - /// Both are valid values to put into queue. - token: Option, - /// A pool to which `token` should be returned. `pool` is optional, as one might want to release a token straight away instead - /// of storing it back in the pool - see [`Self::forget()`] function for that. - pool: Option>>>, -} +use std::{mem::MaybeUninit, sync::Once}; + +use crate::Error; + +#[cfg(unix)] +#[path = "job_token/unix.rs"] +mod sys; + +#[cfg(windows)] +#[path = "job_token/windows.rs"] +mod sys; + +pub(super) struct JobToken(); impl Drop for JobToken { fn drop(&mut self) { - if let Some(pool) = &self.pool { - // Always send back an Ok() variant as we know that the acquisition for this token has succeeded. - let _ = pool.send(self.token.take().map(|token| Ok(token))); + match JobTokenServer::new() { + JobTokenServer::Inherited(jobserver) => jobserver.release_token_raw(), + JobTokenServer::InProcess(jobserver) => jobserver.release_token_raw(), } } } -impl JobToken { - /// Ensure that this token is not put back into queue once it's dropped. - /// This also leads to releasing it sooner for other processes to use, - /// which is a correct thing to do once it is known that there won't be - /// any more token acquisitions. - pub(crate) fn forget(&mut self) { - self.pool.take(); - } -} - -/// A thin wrapper around jobserver's Client. -/// It would be perfectly fine to just use jobserver's Client, but we also want to reuse -/// our own implicit token assigned for this build script. This struct manages that and -/// gives out tokens without exposing whether they're implicit tokens or tokens from jobserver. -/// Furthermore, instead of giving up job tokens, it keeps them around -/// for reuse if we know we're going to request another token after freeing the current one. -pub(crate) struct JobTokenServer { - helper: HelperThread, - tx: Sender>>, - rx: Receiver>>, +pub(super) enum JobTokenServer { + Inherited(inherited_jobserver::JobServer), + InProcess(inprocess_jobserver::JobServer), } impl JobTokenServer { + /// This function returns a static reference to the jobserver because + /// - creating a jobserver from env is a bit fd-unsafe (e.g. the fd might + /// be closed by other jobserver users in the process) and better do it + /// at the start of the program. + /// - in case a jobserver cannot be created from env (e.g. it's not + /// present), we will create a global in-process only jobserver + /// that has to be static so that it will be shared by all cc + /// compilation. pub(crate) fn new() -> &'static Self { - jobserver() - } - fn new_inner(client: Client) -> Result { - let (tx, rx) = mpsc::channel(); - // Push the implicit token. Since JobTokens only give back what they got, - // there should be at most one global implicit token in the wild. - tx.send(None).unwrap(); - let pool = tx.clone(); - let helper = client.into_helper_thread(move |acq| { - let _ = pool.send(Some(acq.map_err(|e| e.into()))); - })?; - Ok(Self { helper, tx, rx }) - } + static INIT: Once = Once::new(); + static mut JOBSERVER: MaybeUninit = MaybeUninit::uninit(); - pub(crate) fn acquire(&self) -> Result { - let token = if let Ok(token) = self.rx.try_recv() { - // Opportunistically check if there's a token that can be reused. - token - } else { - // Cold path, request a token and block - self.helper.request_token(); - self.rx.recv().unwrap() - }; - let token = if let Some(token) = token { - Some(token?) - } else { - None - }; - Ok(JobToken { - token, - pool: Some(self.tx.clone()), - }) + unsafe { + INIT.call_once(|| { + let server = inherited_jobserver::JobServer::from_env() + .map(Self::Inherited) + .unwrap_or_else(|| Self::InProcess(inprocess_jobserver::JobServer::new())); + JOBSERVER = MaybeUninit::new(server); + }); + // TODO: Poor man's assume_init_ref, as that'd require a MSRV of 1.55. + &*JOBSERVER.as_ptr() + } } -} -/// Returns a suitable `JobTokenServer` used to coordinate -/// parallelism between build scripts. A global `JobTokenServer` is used as this ensures -/// that only one implicit job token is used in the wild. -/// Having multiple separate job token servers would lead to each of them assuming that they have control -/// over the implicit job token. -/// As it stands, each caller of `jobserver` can receive an implicit job token and there will be at most -/// one implicit job token in the wild. -fn jobserver() -> &'static JobTokenServer { - static INIT: Once = Once::new(); - static mut JOBSERVER: MaybeUninit = MaybeUninit::uninit(); - - fn _assert_sync() {} - _assert_sync::(); - - unsafe { - INIT.call_once(|| { - let server = default_jobserver(); - JOBSERVER = MaybeUninit::new( - JobTokenServer::new_inner(server).expect("Job server initialization failed"), - ); - }); - // Poor man's assume_init_ref, as that'd require a MSRV of 1.55. - &*JOBSERVER.as_ptr() + pub(crate) fn try_acquire(&self) -> Result, Error> { + match self { + Self::Inherited(jobserver) => jobserver.try_acquire(), + Self::InProcess(jobserver) => Ok(jobserver.try_acquire()), + } } } -unsafe fn default_jobserver() -> jobserver::Client { - // Try to use the environmental jobserver which Cargo typically - // initializes for us... - if let Some(client) = jobserver::Client::from_env() { - return client; +mod inherited_jobserver { + use super::{sys, Error, JobToken}; + + use std::{ + env::var_os, + sync::atomic::{ + AtomicBool, + Ordering::{AcqRel, Acquire}, + }, + }; + + pub(crate) struct JobServer { + /// Implicit token for this process which is obtained and will be + /// released in parent. Since JobTokens only give back what they got, + /// there should be at most one global implicit token in the wild. + /// + /// Since Rust does not execute any `Drop` for global variables, + /// we can't just put it back to jobserver and then re-acquire it at + /// the end of the process. + global_implicit_token: AtomicBool, + inner: sys::JobServerClient, } - // ... but if that fails for whatever reason select something - // reasonable and crate a new jobserver. Use `NUM_JOBS` if set (it's - // configured by Cargo) and otherwise just fall back to a - // semi-reasonable number. Note that we could use `num_cpus` here - // but it's an extra dependency that will almost never be used, so - // it's generally not too worth it. - let mut parallelism = 4; - if let Ok(amt) = env::var("NUM_JOBS") { - if let Ok(amt) = amt.parse() { - parallelism = amt; + impl JobServer { + pub(super) unsafe fn from_env() -> Option { + let var = var_os("CARGO_MAKEFLAGS") + .or_else(|| var_os("MAKEFLAGS")) + .or_else(|| var_os("MFLAGS"))?; + + let inner = sys::JobServerClient::open(var)?; + + Some(Self { + inner, + global_implicit_token: AtomicBool::new(true), + }) + } + + pub(super) fn try_acquire(&self) -> Result, Error> { + if !self.global_implicit_token.swap(false, AcqRel) { + // Cold path, no global implicit token, obtain one + if self.inner.try_acquire()?.is_none() { + return Ok(None); + } + } + Ok(Some(JobToken())) + } + + pub(super) fn release_token_raw(&self) { + // All tokens will be put back into the jobserver immediately + // and they cannot be cached, since Rust does not call `Drop::drop` + // on global variables. + if self + .global_implicit_token + .compare_exchange(false, true, AcqRel, Acquire) + .is_err() + { + // There's already a global implicit token, so this token must + // be released back into jobserver + let _ = self.inner.release(); + } } } +} + +mod inprocess_jobserver { + use super::JobToken; + + use std::{ + env::var, + sync::atomic::{ + AtomicU32, + Ordering::{AcqRel, Acquire}, + }, + }; + + pub(crate) struct JobServer(AtomicU32); - // If we create our own jobserver then be sure to reserve one token - // for ourselves. - let client = jobserver::Client::new(parallelism).expect("failed to create jobserver"); - client.acquire_raw().expect("failed to acquire initial"); - return client; + impl JobServer { + pub(super) fn new() -> Self { + // Use `NUM_JOBS` if set (it's configured by Cargo) and otherwise + // just fall back to a semi-reasonable number. + // + // Note that we could use `num_cpus` here but it's an extra + // dependency that will almost never be used, so + // it's generally not too worth it. + let mut parallelism = 4; + // TODO: Use std::thread::available_parallelism as an upper bound + // when MSRV is bumped. + if let Ok(amt) = var("NUM_JOBS") { + if let Ok(amt) = amt.parse() { + parallelism = amt; + } + } + + Self(AtomicU32::new(parallelism)) + } + + pub(super) fn try_acquire(&self) -> Option { + let res = self + .0 + .fetch_update(AcqRel, Acquire, |tokens| tokens.checked_sub(1)); + + res.ok().map(|_| JobToken()) + } + + pub(super) fn release_token_raw(&self) { + self.0.fetch_add(1, AcqRel); + } + } } diff --git a/src/job_token/unix.rs b/src/job_token/unix.rs new file mode 100644 index 000000000..af626c24d --- /dev/null +++ b/src/job_token/unix.rs @@ -0,0 +1,185 @@ +use std::{ + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{self, Read, Write}, + mem::ManuallyDrop, + os::{ + raw::c_int, + unix::{ + ffi::{OsStrExt, OsStringExt}, + prelude::*, + }, + }, + path::Path, +}; + +pub(super) struct JobServerClient { + read: File, + write: Option, +} + +impl JobServerClient { + pub(super) unsafe fn open(var: OsString) -> Option { + let bytes = var.into_vec(); + + let s = bytes + .split(u8::is_ascii_whitespace) + .filter_map(|arg| { + arg.strip_prefix(b"--jobserver-fds=") + .or_else(|| arg.strip_prefix(b"--jobserver-auth=")) + }) + .find(|bytes| !bytes.is_empty())?; + + if let Some(fifo) = s.strip_prefix(b"fifo:") { + Self::from_fifo(Path::new(OsStr::from_bytes(fifo))) + } else { + Self::from_pipe(OsStr::from_bytes(s).to_str()?) + } + } + + /// `--jobserver-auth=fifo:PATH` + fn from_fifo(path: &Path) -> Option { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .ok()?; + + if is_pipe(&file)? { + // File in Rust is always closed-on-exec as long as it's opened by + // `File::open` or `fs::OpenOptions::open`. + set_nonblocking(&file)?; + + Some(Self { + read: file, + write: None, + }) + } else { + None + } + } + + /// `--jobserver-auth=fd-for-R,fd-for-W` + unsafe fn from_pipe(s: &str) -> Option { + let (read, write) = s.split_once(',')?; + + let read = read.parse().ok()?; + let write = write.parse().ok()?; + + let read = ManuallyDrop::new(File::from_raw_fd(read)); + let write = ManuallyDrop::new(File::from_raw_fd(write)); + + // Ok so we've got two integers that look like file descriptors, but + // for extra sanity checking let's see if they actually look like + // instances of a pipe before we return the client. + // + // If we're called from `make` *without* the leading + on our rule + // then we'll have `MAKEFLAGS` env vars but won't actually have + // access to the file descriptors. + match ( + is_pipe(&read), + is_pipe(&write), + get_access_mode(&read), + get_access_mode(&write), + ) { + ( + Some(true), + Some(true), + Some(libc::O_RDONLY) | Some(libc::O_RDWR), + Some(libc::O_WRONLY) | Some(libc::O_RDWR), + ) => { + let read = read.try_clone().ok()?; + let write = write.try_clone().ok()?; + + // Set read and write end to nonblocking + set_nonblocking(&read)?; + set_nonblocking(&write)?; + + Some(Self { + read, + write: Some(write), + }) + } + _ => None, + } + } + + pub(super) fn try_acquire(&self) -> io::Result> { + let mut fds = [libc::pollfd { + fd: self.read.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }]; + + let ret = cvt(unsafe { libc::poll(fds.as_mut_ptr(), 1, 0) })?; + if ret == 1 { + let mut buf = [0]; + match (&self.read).read(&mut buf) { + Ok(1) => Ok(Some(())), + Ok(_) => Ok(None), // 0, eof + Err(e) + if e.kind() == io::ErrorKind::Interrupted + || e.kind() == io::ErrorKind::WouldBlock => + { + Ok(None) + } + Err(e) => Err(e), + } + } else { + Ok(None) + } + } + + pub(super) fn release(&self) -> io::Result<()> { + // For write to block, this would mean that pipe is full. + // If all every release are pair with an acquire, then this cannot + // happen. + // + // If it does happen, it is likely a bug in the program using this + // crate or some other programs that use the same jobserver have a + // bug in their code. + // + // If that turns out to not be the case we'll get an error anyway! + let mut write = self.write.as_ref().unwrap_or(&self.read); + match write.write(&[b'+'])? { + 1 => Ok(()), + _ => Err(io::Error::from(io::ErrorKind::UnexpectedEof)), + } + } +} + +fn set_nonblocking(file: &File) -> Option<()> { + // F_SETFL can only set the O_APPEND, O_ASYNC, O_DIRECT, O_NOATIME, and + // O_NONBLOCK flags. + // + // For pipe, only O_NONBLOCK is meaningful, so it is ok to + // not issue a F_GETFL fcntl syscall. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK) }; + + if ret == -1 { + None + } else { + Some(()) + } +} + +fn cvt(t: c_int) -> io::Result { + if t == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(t) + } +} + +fn is_pipe(file: &File) -> Option { + Some(file.metadata().ok()?.file_type().is_fifo()) +} + +fn get_access_mode(file: &File) -> Option { + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) }; + if ret == -1 { + return None; + } + + Some(ret & libc::O_ACCMODE) +} diff --git a/src/job_token/windows.rs b/src/job_token/windows.rs new file mode 100644 index 000000000..03ec8869b --- /dev/null +++ b/src/job_token/windows.rs @@ -0,0 +1,76 @@ +use std::{ + ffi::{CString, OsString}, + io, ptr, +}; + +use crate::windows_sys::{ + OpenSemaphoreA, ReleaseSemaphore, WaitForSingleObject, FALSE, HANDLE, SEMAPHORE_MODIFY_STATE, + THREAD_SYNCHRONIZE, WAIT_ABANDONED, WAIT_FAILED, WAIT_OBJECT_0, WAIT_TIMEOUT, +}; + +pub(super) struct JobServerClient { + sem: HANDLE, +} + +unsafe impl Sync for JobServerClient {} +unsafe impl Send for JobServerClient {} + +impl JobServerClient { + pub(super) unsafe fn open(var: OsString) -> Option { + let var = var.to_str()?; + if !var.is_ascii() { + // `OpenSemaphoreA` only accepts ASCII, not utf-8. + // + // Upstream implementation jobserver and jobslot also uses the + // same function and they works without problem, so there's no + // motivation to support utf-8 here using `OpenSemaphoreW` + // which only makes the code harder to maintain by making it more + // different than upstream. + return None; + } + + let s = var + .split_ascii_whitespace() + .filter_map(|arg| arg.strip_prefix("--jobserver-auth=")) + .find(|s| !s.is_empty())?; + + let name = CString::new(s).ok()?; + + let sem = OpenSemaphoreA( + THREAD_SYNCHRONIZE | SEMAPHORE_MODIFY_STATE, + FALSE, + name.as_bytes().as_ptr(), + ); + if sem != ptr::null_mut() { + Some(Self { sem }) + } else { + None + } + } + + pub(super) fn try_acquire(&self) -> io::Result> { + match unsafe { WaitForSingleObject(self.sem, 0) } { + WAIT_OBJECT_0 => Ok(Some(())), + WAIT_TIMEOUT => Ok(None), + WAIT_FAILED => Err(io::Error::last_os_error()), + // We believe this should be impossible for a semaphore, but still + // check the error code just in case it happens. + WAIT_ABANDONED => Err(io::Error::new( + io::ErrorKind::Other, + "Wait on jobserver semaphore returned WAIT_ABANDONED", + )), + _ => unreachable!("Unexpected return value from WaitForSingleObject"), + } + } + + pub(super) fn release(&self) -> io::Result<()> { + // SAFETY: ReleaseSemaphore will write to prev_count is it is Some + // and release semaphore self.sem by 1. + let r = unsafe { ReleaseSemaphore(self.sem, 1, ptr::null_mut()) }; + if r != 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 77f1a1db1..86e55f438 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,6 +66,8 @@ use std::process::{Child, Command, Stdio}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; +#[cfg(feature = "parallel")] +mod async_executor; #[cfg(feature = "parallel")] mod job_token; mod os_pipe; @@ -1297,7 +1299,9 @@ impl Build { #[cfg(feature = "parallel")] fn compile_objects(&self, objs: &[Object], print: &PrintThread) -> Result<(), Error> { - use std::sync::mpsc; + use std::cell::Cell; + + use async_executor::{block_on, YieldOnce}; if objs.len() <= 1 { for obj in objs { @@ -1330,116 +1334,93 @@ impl Build { // acquire the appropriate tokens, Once all objects have been compiled // we wait on all the processes and propagate the results of compilation. - let (tx, rx) = mpsc::channel::<(_, String, KillOnDrop, crate::job_token::JobToken)>(); + let pendings = Cell::new(Vec::<( + Command, + String, + KillOnDrop, + crate::job_token::JobToken, + )>::new()); + let is_disconnected = Cell::new(false); + let has_made_progress = Cell::new(false); - // Since jobserver::Client::acquire can block, waiting - // must be done in parallel so that acquire won't block forever. - let wait_thread = thread::Builder::new().spawn(move || { + let wait_future = async { let mut error = None; - let mut pendings = Vec::new(); // Buffer the stdout let mut stdout = io::BufWriter::with_capacity(128, io::stdout()); - let mut backoff_cnt = 0; loop { - let mut has_made_progress = false; // If the other end of the pipe is already disconnected, then we're not gonna get any new jobs, // so it doesn't make sense to reuse the tokens; in fact, // releasing them as soon as possible (once we know that the other end is disconnected) is beneficial. // Imagine that the last file built takes an hour to finish; in this scenario, // by not releasing the tokens before that last file is done we would effectively block other processes from // starting sooner - even though we only need one token for that last file, not N others that were acquired. - let mut is_disconnected = false; - // Reading new pending tasks - loop { - match rx.try_recv() { - Ok(pending) => { - has_made_progress = true; - pendings.push(pending) - } - Err(mpsc::TryRecvError::Disconnected) if pendings.is_empty() => { - let _ = stdout.flush(); - return if let Some(err) = error { - Err(err) - } else { - Ok(()) - }; - } - Err(mpsc::TryRecvError::Disconnected) => { - is_disconnected = true; - break; - } - _ => break, - } - } - // Try waiting on them. - retain_unordered_mut(&mut pendings, |(cmd, program, child, token)| { - match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { - Ok(Some(())) => { - // Task done, remove the entry - if is_disconnected { - token.forget(); - } - has_made_progress = true; - false - } - Ok(None) => true, // Task still not finished, keep the entry - Err(err) => { - // Task fail, remove the entry. - has_made_progress = true; - if is_disconnected { - token.forget(); + let mut pendings_is_empty = false; + + cell_update(&pendings, |mut pendings| { + // Try waiting on them. + retain_unordered_mut(&mut pendings, |(cmd, program, child, _token)| { + match try_wait_on_child(cmd, program, &mut child.0, &mut stdout) { + Ok(Some(())) => { + // Task done, remove the entry + has_made_progress.set(true); + false } - // Since we can only return one error, log the error to make - // sure users always see all the compilation failures. - let _ = writeln!(stdout, "cargo:warning={}", err); - error = Some(err); + Ok(None) => true, // Task still not finished, keep the entry + Err(err) => { + // Task fail, remove the entry. + // Since we can only return one error, log the error to make + // sure users always see all the compilation failures. + has_made_progress.set(true); - false + let _ = writeln!(stdout, "cargo:warning={}", err); + error = Some(err); + + false + } } - } + }); + pendings_is_empty = pendings.is_empty(); + pendings }); - if !has_made_progress { - if backoff_cnt > 3 { - // We have yielded at least three times without making' - // any progress, so we will sleep for a while. - let duration = - std::time::Duration::from_millis(100 * (backoff_cnt - 3).min(10)); - thread::sleep(duration); + if pendings_is_empty && is_disconnected.get() { + break if let Some(err) = error { + Err(err) } else { - // Given that we spawned a lot of compilation tasks, it is unlikely - // that OS cannot find other ready task to execute. - // - // If all of them are done, then we will yield them and spawn more, - // or simply returns. - // - // Thus this will not be turned into a busy-wait loop and it will not - // waste CPU resource. - thread::yield_now(); - } + Ok(()) + }; } - backoff_cnt = if has_made_progress { - 0 - } else { - backoff_cnt + 1 + YieldOnce::default().await; + } + }; + let spawn_future = async { + for obj in objs { + let (mut cmd, program) = self.create_compile_object_cmd(obj)?; + let token = loop { + if let Some(token) = tokens.try_acquire()? { + break token; + } else { + YieldOnce::default().await + } }; + let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; + + cell_update(&pendings, |mut pendings| { + pendings.push((cmd, program, KillOnDrop(child), token)); + pendings + }); + + has_made_progress.set(true); } - })?; - for obj in objs { - let (mut cmd, program) = self.create_compile_object_cmd(obj)?; - let token = tokens.acquire()?; - let child = spawn(&mut cmd, &program, print.pipe_writer_cloned()?.unwrap())?; + is_disconnected.set(true); - tx.send((cmd, program, KillOnDrop(child), token)) - .expect("Wait thread must be alive until all compilation jobs are done, otherwise we risk deadlock"); - } - // Drop tx so that the wait_thread could return - drop(tx); + Ok::<_, Error>(()) + }; - return wait_thread.join().expect("wait_thread panics"); + return block_on(wait_future, spawn_future, &has_made_progress); struct KillOnDrop(Child); @@ -1450,6 +1431,16 @@ impl Build { child.kill().ok(); } } + + fn cell_update(cell: &Cell, f: F) + where + T: Default, + F: FnOnce(T) -> T, + { + let old = cell.take(); + let new = f(old); + cell.set(new); + } } #[cfg(not(feature = "parallel"))] diff --git a/src/windows_sys.rs b/src/windows_sys.rs index ee4704d25..20a256076 100644 --- a/src/windows_sys.rs +++ b/src/windows_sys.rs @@ -62,6 +62,22 @@ extern "system" { nsize: u32, ) -> BOOL; } +#[link(name = "kernel32")] +extern "system" { + pub fn OpenSemaphoreA(dwdesiredaccess: u32, binherithandle: BOOL, lpname: PCSTR) -> HANDLE; +} +#[link(name = "kernel32")] +extern "system" { + pub fn ReleaseSemaphore( + hsemaphore: HANDLE, + lreleasecount: i32, + lppreviouscount: *mut i32, + ) -> BOOL; +} +#[link(name = "kernel32")] +extern "system" { + pub fn WaitForSingleObject(hhandle: HANDLE, dwmilliseconds: u32) -> WIN32_ERROR; +} #[link(name = "ole32")] extern "system" { pub fn CoCreateInstance( @@ -93,6 +109,7 @@ pub type COINIT = i32; pub const COINIT_MULTITHREADED: COINIT = 0i32; pub const ERROR_NO_MORE_ITEMS: WIN32_ERROR = 259u32; pub const ERROR_SUCCESS: WIN32_ERROR = 0u32; +pub const FALSE: BOOL = 0i32; #[repr(C)] pub struct FILETIME { pub dwLowDateTime: u32, @@ -135,6 +152,7 @@ pub const INVALID_HANDLE_VALUE: HANDLE = invalid_mut(-1i32 as _); pub type IUnknown = *mut ::core::ffi::c_void; pub const KEY_READ: REG_SAM_FLAGS = 131097u32; pub const KEY_WOW64_32KEY: REG_SAM_FLAGS = 512u32; +pub type PCSTR = *const u8; pub type PCWSTR = *const u16; pub type PWSTR = *mut u16; pub type REG_SAM_FLAGS = u32; @@ -178,8 +196,16 @@ impl ::core::clone::Clone for SECURITY_ATTRIBUTES { *self } } +pub const SEMAPHORE_MODIFY_STATE: SYNCHRONIZATION_ACCESS_RIGHTS = 2u32; +pub type SYNCHRONIZATION_ACCESS_RIGHTS = u32; pub const S_FALSE: HRESULT = 1i32; pub const S_OK: HRESULT = 0i32; +pub type THREAD_ACCESS_RIGHTS = u32; +pub const THREAD_SYNCHRONIZE: THREAD_ACCESS_RIGHTS = 1048576u32; +pub const WAIT_ABANDONED: WIN32_ERROR = 128u32; +pub const WAIT_FAILED: WIN32_ERROR = 4294967295u32; +pub const WAIT_OBJECT_0: WIN32_ERROR = 0u32; +pub const WAIT_TIMEOUT: WIN32_ERROR = 258u32; pub type WIN32_ERROR = u32; /// Adapted from From 4c0062da5dbb73d39f303a8ba1ca7361243e8a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Cruz?= Date: Sat, 11 Nov 2023 22:51:11 +0000 Subject: [PATCH 134/138] fix: Add apple tvos support (#881) * fix: Add apple tvos support * ci: Add apple tvOS build job --- .github/workflows/main.yml | 18 ++++++++++++++++++ src/lib.rs | 12 ++++++++++++ tests/support/mod.rs | 6 ++++++ tests/test.rs | 8 ++++---- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f2286952..1fafe834e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,6 +92,24 @@ jobs: - run: cargo test ${{ matrix.no_run }} --manifest-path cc-test/Cargo.toml --target ${{ matrix.target }} --features parallel - run: cargo test ${{ matrix.no_run }} --manifest-path cc-test/Cargo.toml --target ${{ matrix.target }} --release + check-tvos: + name: Test aarch64-apple-tvos + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust (rustup) + run: | + set -euxo pipefail + rustup toolchain install nightly --no-self-update --profile minimal + rustup component add rust-src --toolchain nightly + rustup default nightly + shell: bash + - run: cargo test -Z build-std=std --no-run --target aarch64-apple-tvos + - run: cargo test -Z build-std=std --no-run --features parallel --target aarch64-apple-tvos + - run: cargo test -Z build-std=std --no-run --manifest-path cc-test/Cargo.toml --target aarch64-apple-tvos + - run: cargo test -Z build-std=std --no-run --manifest-path cc-test/Cargo.toml --target aarch64-apple-tvos --features parallel + - run: cargo test -Z build-std=std --no-run --manifest-path cc-test/Cargo.toml --target aarch64-apple-tvos --release + cuda: name: Test CUDA support runs-on: ubuntu-20.04 diff --git a/src/lib.rs b/src/lib.rs index 86e55f438..58356598a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1831,6 +1831,16 @@ impl Build { .into(), ); } + } else if target.contains("aarch64-apple-tvos") { + if let Some(arch) = + map_darwin_target_from_rust_to_compiler_architecture(target) + { + let deployment_target = + self.apple_deployment_version(AppleOs::TvOs, target, None); + cmd.args.push( + format!("--target={}-apple-tvos{}", arch, deployment_target).into(), + ); + } } else if target.starts_with("riscv64gc-") { cmd.args.push( format!("--target={}", target.replace("riscv64gc", "riscv64")).into(), @@ -2568,6 +2578,8 @@ impl Build { clang.to_string() } else if target.contains("apple-watchos") { clang.to_string() + } else if target.contains("apple-tvos") { + clang.to_string() } else if target.contains("android") { autodetect_android_compiler(&target, &host, gnu, clang) } else if target.contains("cloudabi") { diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 3ec191113..94b51f653 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -61,6 +61,12 @@ impl Test { t } + pub fn clang() -> Test { + let t = Test::new(); + t.shim("clang").shim("clang++").shim("ar"); + t + } + pub fn shim(&self, name: &str) -> &Test { let name = if name.ends_with(env::consts::EXE_SUFFIX) { name.to_string() diff --git a/tests/test.rs b/tests/test.rs index f712c4368..fc07e6253 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -510,9 +510,9 @@ fn gnu_apple_darwin() { #[cfg(target_os = "macos")] #[test] -fn apple_tvos() { +fn clang_apple_tvos() { for target in &["aarch64-apple-tvos"] { - let test = Test::gnu(); + let test = Test::clang(); test.gcc() .target(&target) .host(&target) @@ -525,9 +525,9 @@ fn apple_tvos() { #[cfg(target_os = "macos")] #[test] -fn apple_tvsimulator() { +fn clang_apple_tvsimulator() { for target in &["x86_64-apple-tvos"] { - let test = Test::gnu(); + let test = Test::clang(); test.gcc() .target(&target) .host(&target) From bb1220d170bbc5f0ac71db9788f31081f7ff40a1 Mon Sep 17 00:00:00 2001 From: ori raisfeld Date: Sun, 12 Nov 2023 01:08:46 +0200 Subject: [PATCH 135/138] added a function to remove flags (#885) * added a function to remove flags * Reword/expand `remove_flag` doc comment. --------- Co-authored-by: Thom Chiovoloni --- src/lib.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 58356598a..ae89a1b86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -431,6 +431,24 @@ impl Build { self } + /// Removes a compiler flag that was added by [`Build::flag`]. + /// + /// Will not remove flags added by other means (default flags, + /// flags from env, and so on). + /// + /// # Example + /// ``` + /// cc::Build::new() + /// .file("src/foo.c") + /// .flag("unwanted_flag") + /// .remove_flag("unwanted_flag"); + /// ``` + + pub fn remove_flag(&mut self, flag: &str) -> &mut Build { + self.flags.retain(|other_flag| &**other_flag != flag); + self + } + /// Add a flag to the invocation of the ar /// /// # Example From 17f9e01ffc2b73cb2706dc7c1047fd564aad1134 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 13 Nov 2023 04:59:19 +1000 Subject: [PATCH 136/138] Release cc v1.0.84 (#898) Resolve #893 Signed-off-by: Jiahao XU --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9a6c573cf..9e646aa64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cc" -version = "1.0.83" +version = "1.0.84" authors = ["Alex Crichton "] license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/cc-rs" From 2d6a3b2119cf5eacc01e1f2877e064a7aede7819 Mon Sep 17 00:00:00 2001 From: BlackHoleFox Date: Tue, 14 Nov 2023 02:12:09 -0600 Subject: [PATCH 137/138] Fix Apple deployment version floor when linking C++ (#901) --- src/lib.rs | 86 +++++++++++++++++++++++++++++++++++++++++---------- tests/test.rs | 39 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d8effdbaf..0123bb16c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3407,6 +3407,8 @@ impl Build { target: &str, arch_str: Option<&str>, ) -> String { + const OLD_IOS_MINIMUM_VERSION: &str = "7.0"; + fn rustc_provided_target(rustc: Option<&str>, target: &str) -> Option { let rustc = rustc?; let output = Command::new(rustc) @@ -3427,6 +3429,62 @@ impl Build { } } + let deployment_from_env = |name: &str| { + // note this isn't hit in production codepaths, its mostly just for tests which don't + // set the real env + if let Some((_, v)) = self.env.iter().find(|(k, _)| &**k == OsStr::new(name)) { + Some(v.to_str().unwrap().to_string()) + } else { + env::var(name).ok() + } + }; + + // Determines if the acquired deployment target is too low to support modern C++ on some Apple platform. + // + // A long time ago they used libstdc++, but since macOS 10.9 and iOS 7 libc++ has been the library the SDKs provide to link against. + // If a `cc`` config wants to use C++, we round up to these versions as the baseline. + let maybe_cpp_version_baseline = |deployment_target_ver: String| -> String { + if !self.cpp { + return deployment_target_ver; + } + + let mut deployment_target = deployment_target_ver + .split('.') + .map(|v| v.parse::().expect("integer version")); + + match os { + AppleOs::MacOs => { + let major = deployment_target.next().unwrap_or(0); + let minor = deployment_target.next().unwrap_or(0); + + // If below 10.9, we round up. + if major == 10 && minor < 9 { + println!( + "cargo-warning: macOS deployment target ({}) too low, it will be increased", + deployment_target_ver + ); + return String::from("10.9"); + } + } + AppleOs::Ios => { + let major = deployment_target.next().unwrap_or(0); + + if major < 7 { + println!( + "cargo-warning: iOS deployment target ({}) too low, it will be increased", + deployment_target_ver + ); + return String::from(OLD_IOS_MINIMUM_VERSION); + } + } + // watchOS, tvOS, and others are all new enough that libc++ is their baseline. + _ => {} + } + + // If the deployment target met or exceeded the C++ baseline + deployment_target_ver + }; + let rustc = self.getenv("RUSTC"); let rustc = rustc.as_deref(); // note the hardcoded minimums here are subject to change in a future compiler release, @@ -3436,31 +3494,27 @@ impl Build { // the ordering of env -> rustc -> old defaults is intentional for performance when using // an explicit target match os { - AppleOs::MacOs => env::var("MACOSX_DEPLOYMENT_TARGET") - .ok() + AppleOs::MacOs => deployment_from_env("MACOSX_DEPLOYMENT_TARGET") .or_else(|| rustc_provided_target(rustc, target)) + .map(maybe_cpp_version_baseline) .unwrap_or_else(|| { if arch_str == Some("aarch64") { - "11.0" + "11.0".into() } else { - if self.cpp { - "10.9" - } else { - "10.7" - } + maybe_cpp_version_baseline("10.7".into()) } - .into() }), - AppleOs::Ios => env::var("IPHONEOS_DEPLOYMENT_TARGET") - .ok() + + AppleOs::Ios => deployment_from_env("IPHONEOS_DEPLOYMENT_TARGET") .or_else(|| rustc_provided_target(rustc, target)) - .unwrap_or_else(|| "7.0".into()), - AppleOs::WatchOs => env::var("WATCHOS_DEPLOYMENT_TARGET") - .ok() + .map(maybe_cpp_version_baseline) + .unwrap_or_else(|| OLD_IOS_MINIMUM_VERSION.into()), + + AppleOs::WatchOs => deployment_from_env("WATCHOS_DEPLOYMENT_TARGET") .or_else(|| rustc_provided_target(rustc, target)) .unwrap_or_else(|| "5.0".into()), - AppleOs::TvOs => env::var("TVOS_DEPLOYMENT_TARGET") - .ok() + + AppleOs::TvOs => deployment_from_env("TVOS_DEPLOYMENT_TARGET") .or_else(|| rustc_provided_target(rustc, target)) .unwrap_or_else(|| "9.0".into()), } diff --git a/tests/test.rs b/tests/test.rs index fc07e6253..220d2f2f7 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -508,6 +508,45 @@ fn gnu_apple_darwin() { } } +#[cfg(target_os = "macos")] +#[test] +fn macos_cpp_minimums() { + let versions = &[ + // Too low + ("10.7", "10.9"), + // Minimum + ("10.9", "10.9"), + // Higher + ("11.0", "11.0"), + ]; + + let target = "x86_64-apple-darwin"; + for (deployment_target, expected) in versions { + let test = Test::gnu(); + test.gcc() + .target(target) + .host(target) + .cpp(true) + .__set_env("MACOSX_DEPLOYMENT_TARGET", deployment_target) + .file("foo.c") + .compile("foo"); + + test.cmd(0) + .must_have(format!("-mmacosx-version-min={}", expected)); + } + + let test = Test::gnu(); + test.gcc() + .target(target) + .host(target) + .__set_env("MACOSX_DEPLOYMENT_TARGET", "10.7") + .file("foo.c") + .compile("foo"); + + // No C++ leaves it untouched + test.cmd(0).must_have("-mmacosx-version-min=10.7"); +} + #[cfg(target_os = "macos")] #[test] fn clang_apple_tvos() { From bfd68f2d0b19b0872d8bc085290d94d40ca85d60 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 29 Nov 2023 02:49:16 +0200 Subject: [PATCH 138/138] Fix unconditional cargo metadata printing on flag support check (#908) --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 0123bb16c..42278a794 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -527,6 +527,7 @@ impl Build { let host = self.get_host()?; let mut cfg = Build::new(); cfg.flag(flag) + .cargo_metadata(self.cargo_metadata) .target(&target) .opt_level(0) .host(&host)