From 42fdcbcbd64207e4168be43e78d54c4767a815bb Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Tue, 6 Aug 2024 08:10:37 -0700 Subject: [PATCH] feat: docker build fixes (#1258) --- build/src/docker.rs | 37 +++++++++++++++++++++--- build/src/lib.rs | 70 +++++++++++++++++++++++---------------------- 2 files changed, 69 insertions(+), 38 deletions(-) diff --git a/build/src/docker.rs b/build/src/docker.rs index 2047bf9f8c..618b618198 100644 --- a/build/src/docker.rs +++ b/build/src/docker.rs @@ -18,9 +18,15 @@ fn get_docker_image(tag: &str) -> String { pub fn create_docker_command( args: &BuildArgs, program_dir: &Utf8PathBuf, - workspace_root: &Utf8PathBuf, + program_metadata: &cargo_metadata::Metadata, ) -> Result { let image = get_docker_image(&args.tag); + let canonicalized_program_dir: Utf8PathBuf = program_dir + .canonicalize() + .expect("Failed to canonicalize program directory") + .try_into() + .unwrap(); + let workspace_root = &program_metadata.workspace_root; // Check if docker is installed and running. let docker_check = Command::new("docker") @@ -39,10 +45,29 @@ pub fn create_docker_command( let workspace_root_path = format!("{}:/root/program", workspace_root); let program_dir_path = format!( "/root/program/{}", - program_dir.strip_prefix(workspace_root).unwrap() + canonicalized_program_dir + .strip_prefix(workspace_root) + .unwrap() ); - // Add docker-specific arguments. + // Get the target directory for the ELF in the context of the Docker container. + let relative_target_dir = (program_metadata.target_directory) + .strip_prefix(workspace_root) + .unwrap(); + let target_dir = format!( + "/root/program/{}/{}/{}", + relative_target_dir, + crate::HELPER_TARGET_SUBDIR, + "docker" + ); + + // When executing the Docker command: + // 1. Set the target directory to a subdirectory of the program's target directory to avoid build + // conflicts with the parent process. Source: https://github.com/rust-lang/cargo/issues/6412 + // 2. Set the rustup toolchain to succinct. + // 3. Set the encoded rust flags. + // Note: In Docker, you can't use the .env command to set environment variables, you have to use + // the -e flag. let mut docker_args = vec![ "run".to_string(), "--rm".to_string(), @@ -53,6 +78,8 @@ pub fn create_docker_command( "-w".to_string(), program_dir_path, "-e".to_string(), + format!("CARGO_TARGET_DIR={}", target_dir), + "-e".to_string(), "RUSTUP_TOOLCHAIN=succinct".to_string(), "-e".to_string(), format!("CARGO_ENCODED_RUSTFLAGS={}", get_rust_compiler_flags()), @@ -66,6 +93,8 @@ pub fn create_docker_command( docker_args.extend_from_slice(&get_program_build_args(args)); let mut command = Command::new("docker"); - command.current_dir(program_dir.clone()).args(&docker_args); + command + .current_dir(canonicalized_program_dir.clone()) + .args(&docker_args); Ok(command) } diff --git a/build/src/lib.rs b/build/src/lib.rs index 88021ddae0..56de3ad217 100644 --- a/build/src/lib.rs +++ b/build/src/lib.rs @@ -133,7 +133,11 @@ fn get_rust_compiler_flags() -> String { } /// Get the command to build the program locally. -fn create_local_command(args: &BuildArgs, program_dir: &Utf8PathBuf) -> Command { +fn create_local_command( + args: &BuildArgs, + program_dir: &Utf8PathBuf, + program_metadata: &cargo_metadata::Metadata, +) -> Command { let mut command = Command::new("cargo"); let canonicalized_program_dir = program_dir .canonicalize() @@ -153,34 +157,28 @@ fn create_local_command(args: &BuildArgs, program_dir: &Utf8PathBuf) -> Command } } + // When executing the local command: + // 1. Set the target directory to a subdirectory of the program's target directory to avoid build + // conflicts with the parent process. Source: https://github.com/rust-lang/cargo/issues/6412 + // 2. Set the rustup toolchain to succinct. + // 3. Set the encoded rust flags. + // 4. Remove the rustc configuration, otherwise in a build script it will attempt to compile the + // program with the toolchain of the normal build process, rather than the Succinct toolchain. command .current_dir(canonicalized_program_dir) .env("RUSTUP_TOOLCHAIN", "succinct") .env("CARGO_ENCODED_RUSTFLAGS", get_rust_compiler_flags()) + .env_remove("RUSTC") + .env( + "CARGO_TARGET_DIR", + program_metadata.target_directory.join(HELPER_TARGET_SUBDIR), + ) .args(&get_program_build_args(args)); command } /// Execute the command and handle the output depending on the context. -fn execute_command( - mut command: Command, - docker: bool, - program_metadata: &cargo_metadata::Metadata, -) -> Result<()> { - // Strip the rustc configuration, otherwise in the helper it will attempt to compile the SP1 - // program with the toolchain of the normal build process, rather than the Succinct toolchain. - command.env_remove("RUSTC"); - - // Set the target directory to a subdirectory of the program's target directory to avoid - // build conflicts with the parent process. If removed, programs that share the same target - // directory (i.e. same workspace) as the script will hang indefinitely due to a file lock - // when building in the helper. - // Source: https://github.com/rust-lang/cargo/issues/6412 - command.env( - "CARGO_TARGET_DIR", - program_metadata.target_directory.join(HELPER_TARGET_SUBDIR), - ); - +fn execute_command(mut command: Command, docker: bool) -> Result<()> { // Add necessary tags for stdout and stderr from the command. let mut child = command .stdout(Stdio::piped()) @@ -224,10 +222,17 @@ fn copy_elf_to_output_dir( let root_package = program_metadata.root_package(); let root_package_name = root_package.as_ref().map(|p| &p.name); - // The ELF is written to a target folder specified by the program's package. + // The ELF is written to a target folder specified by the program's package. If built with Docker, + // includes /docker after HELPER_TARGET_SUBDIR. + let target_dir_suffix = if args.docker { + format!("{}/{}", HELPER_TARGET_SUBDIR, "docker") + } else { + HELPER_TARGET_SUBDIR.to_string() + }; + let original_elf_path = program_metadata .target_directory - .join(HELPER_TARGET_SUBDIR) + .join(target_dir_suffix) .join(BUILD_TARGET) .join("release") .join(root_package_name.unwrap()); @@ -278,17 +283,7 @@ pub fn build_program(args: &BuildArgs, program_dir: Option) -> Result) -> Result