diff --git a/.cargo/config.toml b/.cargo/config.toml index 93ffc87f1..99863c4ac 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -7,12 +7,15 @@ update-all = "xtask update-all" check-style = "xtask check-style" rootfs = "xtask rootfs" +musl-libs = "xtask musl-libs" opencv = "xtask opencv" ffmpeg = "xtask ffmpeg" libc-test = "xtask libc-test" other-test = "xtask other-test" image = "xtask image" +linux-libos = "xtask linux-libos" + asm = "xtask asm" qemu = "xtask qemu" gdb = "xtask gdb" diff --git a/.github/scripts/install-deps.sh b/.github/scripts/install-deps.sh deleted file mode 100755 index e8334cc56..000000000 --- a/.github/scripts/install-deps.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -sudo apt-get update -sudo apt-get install -y $@ -pip3 install -r tests/requirements.txt \ No newline at end of file diff --git a/.github/scripts/install-qemu.sh b/.github/scripts/make-qemu.sh similarity index 84% rename from .github/scripts/install-qemu.sh rename to .github/scripts/make-qemu.sh index 33de55d91..0be001a4f 100755 --- a/.github/scripts/install-qemu.sh +++ b/.github/scripts/make-qemu.sh @@ -4,4 +4,4 @@ wget https://download.qemu.org/qemu-$1.tar.xz tar -xJf qemu-$1.tar.xz cd qemu-$1 ./configure --target-list=x86_64-softmmu,riscv64-softmmu,aarch64-softmmu -make -j$nproc > /dev/null 2>&1 +make -j > /dev/null 2>&1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 139814381..e32753e5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,41 +46,6 @@ jobs: command: doc args: --all-features --no-deps - build-aarch64: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.rust_toolchain }} - override: true - target: aarch64-unknown-linux-gnu - - - uses: actions-rs/cargo@v1 - with: - command: build - use-cross: true - args: --target aarch64-unknown-linux-gnu --workspace --exclude linux-syscall --exclude zcore-loader --exclude zcore - - build-user: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - name: Pull prebuilt images - run: git lfs pull -I prebuilt/zircon/x64/libc.so,prebuilt/zircon/x64/libfdio.so,prebuilt/zircon/x64/libunwind.so,prebuilt/zircon/x64/libzircon.so,prebuilt/zircon/x64/Scrt1.o - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.rust_toolchain }} - target: x86_64-fuchsia - - - name: Build Zircon user programs - run: cd zircon-user && make build MODE=release - test-libos: runs-on: ubuntu-latest strategy: @@ -153,3 +118,20 @@ jobs: - name: Clippy ${{ matrix.arch }} bare-metal linux if: matrix.arch == 'aarch64' run: cd zCore && make clippy ARCH=${{ matrix.arch }} LINUX=1 + + build-user: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - name: Pull prebuilt images + run: git lfs pull -I prebuilt/zircon/x64/libc.so,prebuilt/zircon/x64/libfdio.so,prebuilt/zircon/x64/libunwind.so,prebuilt/zircon/x64/libzircon.so,prebuilt/zircon/x64/Scrt1.o + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.rust_toolchain }} + target: x86_64-fuchsia + + - name: Build Zircon user programs + run: cd zircon-user && make build MODE=release diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2bd136c7f..28a675844 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,4 @@ -name: Deploy docs +name: Deploy docs on: push: @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - + - uses: actions-rs/toolchain@v1 with: profile: minimal diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5ef56091..c0a2726e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,7 +87,7 @@ jobs: git lfs pull -I prebuilt/zircon/x64/userboot-libos.so - name: Install python dependencies - run: .github/scripts/install-deps.sh + run: pip3 install -r tests/requirements.txt - name: Run fast tests if: github.event_name != 'schedule' @@ -97,8 +97,9 @@ jobs: if: github.event_name == 'schedule' run: cd tests && python3 zircon_core_test.py --libos - linux-libc-test-libos: - name: Linux Libc Test Libos + + zircon-core-test: + name: Zircon Core Test Baremetal runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 @@ -111,22 +112,44 @@ jobs: toolchain: ${{ env.rust_toolchain }} components: rust-src, llvm-tools-preview, rustfmt, clippy - - name: Install dependencies - run: .github/scripts/install-deps.sh musl-tools musl-dev + - name: Pull prebuilt images + run: | + git lfs pull -I prebuilt/zircon/x64/core-tests.zbi + git lfs pull -I prebuilt/zircon/x64/libzircon.so + git lfs pull -I prebuilt/zircon/x64/userboot.so + + - name: Cache QEMU + id: cache-qemu + uses: actions/cache@v3 + with: + path: qemu-${{ env.qemu_version }} + key: qemu-${{ env.qemu_version }} - - name: Prepare rootfs - run: make libc-test + - name: Install ninja-build + run: sudo apt-get update && sudo apt-get install -y ninja-build + + - name: Download and Compile QEMU + if: steps.cache-qemu.outputs.cache-hit != 'true' + run: .github/scripts/make-qemu.sh ${{ env.qemu_version }} + + - name: Install QEMU + run: | + cd qemu-${{ env.qemu_version }} && sudo make install + qemu-system-x86_64 --version + + - name: Install python dependencies + run: pip3 install -r tests/requirements.txt - name: Run fast tests if: github.event_name != 'schedule' - run: cd tests && python3 linux_libc_test.py --libos --fast + run: cd tests && python3 zircon_core_test.py --fast - name: Run full tests if: github.event_name == 'schedule' - run: cd tests && python3 linux_libc_test.py --libos + run: cd tests && python3 zircon_core_test.py - zircon-core-test-baremetal: - name: Zircon Core Test Baremetal + linux-libc-test-libos: + name: Linux Libc Test Libos runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 @@ -139,38 +162,25 @@ jobs: toolchain: ${{ env.rust_toolchain }} components: rust-src, llvm-tools-preview, rustfmt, clippy - - name: Pull prebuilt images - run: | - git lfs pull -I prebuilt/zircon/x64/core-tests.zbi - git lfs pull -I prebuilt/zircon/x64/libzircon.so - git lfs pull -I prebuilt/zircon/x64/userboot.so - - - name: Install dependencies - run: .github/scripts/install-deps.sh ninja-build - - - name: Cache QEMU - id: cache-qemu + - name: Cache musl toolchain uses: actions/cache@v3 with: - path: qemu-${{ env.qemu_version }} - key: qemu-${{ env.qemu_version }}-x86_64-riscv64 + path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz + key: ${{ matrix.arch }}-linux-musl-cross - - name: Download and Compile QEMU - if: steps.cache-qemu.outputs.cache-hit != 'true' - run: .github/scripts/install-qemu.sh ${{ env.qemu_version }} + - name: Prepare rootfs + run: make libc-test - - name: Install QEMU - run: | - cd qemu-${{ env.qemu_version }} && sudo make install - qemu-system-x86_64 --version + - name: Install python dependencies + run: pip3 install -r tests/requirements.txt - name: Run fast tests if: github.event_name != 'schedule' - run: cd tests && python3 zircon_core_test.py --fast + run: cd tests && python3 linux_libc_test.py --libos --fast - name: Run full tests if: github.event_name == 'schedule' - run: cd tests && python3 zircon_core_test.py + run: cd tests && python3 linux_libc_test.py --libos linux-libc-test-baremetal: name: Linux Libc Test Baremetal @@ -196,28 +206,37 @@ jobs: crate: cargo-binutils version: latest - - name: Install dependencies - run: .github/scripts/install-deps.sh musl-tools musl-dev ninja-build - - name: Cache QEMU id: cache-qemu uses: actions/cache@v3 with: path: qemu-${{ env.qemu_version }} - key: qemu-${{ env.qemu_version }}-x64-rv64-aa64 + key: qemu-${{ env.qemu_version }} + + - name: Install ninja-build + run: sudo apt-get update && sudo apt-get install -y ninja-build - name: Download and Compile QEMU if: steps.cache-qemu.outputs.cache-hit != 'true' - run: .github/scripts/install-qemu.sh ${{ env.qemu_version }} + run: .github/scripts/make-qemu.sh ${{ env.qemu_version }} - name: Install QEMU run: | cd qemu-${{ env.qemu_version }} && sudo make install qemu-system-${{ matrix.arch }} --version + - name: Cache musl toolchain + uses: actions/cache@v3 + with: + path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz + key: ${{ matrix.arch }}-linux-musl-cross + - name: Prepare rootfs run: make libc-test ARCH=${{ matrix.arch }} && make image ARCH=${{ matrix.arch }} + - name: Install python dependencies + run: pip3 install -r tests/requirements.txt + - name: Run fast tests if: github.event_name != 'schedule' run: cd tests && python3 linux_libc_test.py --arch ${{ matrix.arch }} --fast @@ -250,28 +269,37 @@ jobs: crate: cargo-binutils version: latest - - name: Install dependencies - run: .github/scripts/install-deps.sh musl-tools musl-dev ninja-build - - name: Cache QEMU id: cache-qemu uses: actions/cache@v3 with: path: qemu-${{ env.qemu_version }} - key: qemu-${{ env.qemu_version }}-x64-rv64-aa64 + key: qemu-${{ env.qemu_version }} + + - name: Install ninja-build + run: sudo apt-get update && sudo apt-get install -y ninja-build - name: Download and Compile QEMU if: steps.cache-qemu.outputs.cache-hit != 'true' - run: .github/scripts/install-qemu.sh ${{ env.qemu_version }} + run: .github/scripts/make-qemu.sh ${{ env.qemu_version }} - name: Install QEMU run: | cd qemu-${{ env.qemu_version }} && sudo make install qemu-system-${{ matrix.arch }} --version + - name: Cache musl toolchain + uses: actions/cache@v3 + with: + path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz + key: ${{ matrix.arch }}-linux-musl-cross + - name: Prepare rootfs run: make other-test ARCH=${{ matrix.arch }} && make image ARCH=${{ matrix.arch }} + - name: Install python dependencies + run: pip3 install -r tests/requirements.txt + - name: Run fast tests if: github.event_name != 'schedule' run: cd tests && python3 linux_other_test.py --arch ${{ matrix.arch }} --fast diff --git a/.github/workflows/third-party.yml b/.github/workflows/third-party.yml index 185fd8b30..81eb81980 100644 --- a/.github/workflows/third-party.yml +++ b/.github/workflows/third-party.yml @@ -22,5 +22,11 @@ jobs: profile: minimal toolchain: ${{ env.rust_toolchain }} - - name: Build riscv64 opencv + - name: Cache musl toolchain + uses: actions/cache@v3 + with: + path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz + key: ${{ matrix.arch }}-linux-musl-cross + + - name: Build riscv64 library run: cargo ${{ matrix.item }} --arch riscv64 diff --git a/xtask/CHANGLOG.md b/xtask/CHANGLOG.md index b724356e5..8803db67f 100644 --- a/xtask/CHANGLOG.md +++ b/xtask/CHANGLOG.md @@ -2,6 +2,19 @@ 最新的更新将出现在最上方。 +## 20220614 (YdrMaster) + +- 增加 `cargo musl-libs --args `，拷贝实际编译使用的 musl 工具链提供的动态库到 rootfs。 + +## 20220613 (YdrMaster) + +- 增加 `cargo linux-libos --args `，以 linux libos 模式启动，并传递指定参数。 + 例如 `cargo linux-libos --args /bin/busybox ls`； + +## 20220612 (YdrMaster) + +- 统一各种架构编译测例的过程（x86_64 现在也下载 musl toolchain）； + ## 20220610 (YdrMaster) - 修改 command/download，使用宏支持多种方式下载； diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 196bee14c..553c98396 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,7 +9,7 @@ build = "build.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -clap = { version = "3.1", features = ["derive"] } +clap = { version = "~3.1", features = ["derive"] } dircpy = "0.3" rand = "0.8" lazy_static = "1.4.0" diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs index aca4bf43d..6e24c5be3 100644 --- a/xtask/src/arch.rs +++ b/xtask/src/arch.rs @@ -1,6 +1,9 @@ //! 支持架构的定义。 -use crate::{LinuxRootfs, XError, ORIGIN, TARGET}; +use crate::{ + command::{dir, download::wget, CommandExt, Tar}, + LinuxRootfs, XError, ORIGIN, TARGET, +}; use std::{path::PathBuf, str::FromStr}; /// 支持的 CPU 架构。 @@ -33,6 +36,23 @@ impl Arch { pub fn target(&self) -> PathBuf { PathBuf::from(TARGET).join(self.name()) } + + /// 下载 musl 工具链，返回工具链路径。 + pub fn linux_musl_cross(&self) -> PathBuf { + let name = format!("{}-linux-musl-cross", self.name().to_lowercase()); + + let origin = self.origin(); + let target = self.target(); + + let tgz = origin.join(format!("{name}.tgz")); + let dir = target.join(&name); + + dir::rm(&dir).unwrap(); + wget(format!("https://musl.cc/{name}.tgz"), &tgz); + Tar::xf(&tgz, Some(target)).invoke(); + + dir + } } impl FromStr for Arch { diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 385a4bbbc..2677998ca 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -182,6 +182,6 @@ fn rustsbi_qemu() -> PathBuf { // Ext::new("unzip").arg("-d").arg(&dir).arg(zip).invoke(); // dir.join("rustsbi-qemu.bin") - // PathBuf::from("default") - PathBuf::from("../rustsbi-qemu/target/riscv64imac-unknown-none-elf/release/rustsbi-qemu.bin") + PathBuf::from("default") + // PathBuf::from("../rustsbi-qemu/target/riscv64imac-unknown-none-elf/release/rustsbi-qemu.bin") } diff --git a/xtask/src/command/cargo.rs b/xtask/src/command/cargo.rs index 0a253c501..a76ab9fbf 100644 --- a/xtask/src/command/cargo.rs +++ b/xtask/src/command/cargo.rs @@ -6,32 +6,44 @@ pub(crate) struct Cargo(Command); ext!(Cargo); impl Cargo { + #[inline] fn new(sub: &(impl AsRef + ?Sized)) -> Self { let mut git = Self(Command::new("cargo")); git.arg(sub); git } + #[inline] pub fn update() -> Self { Self::new("update") } + #[inline] pub fn fmt() -> Self { Self::new("fmt") } + #[inline] pub fn clippy() -> Self { Self::new("clippy") } + #[inline] pub fn doc() -> Self { Self::new("doc") } + #[inline] pub fn build() -> Self { Self::new("build") } + #[inline] + pub fn run() -> Self { + Self::new("run") + } + + #[inline] pub fn all_features(&mut self) -> &mut Self { self.arg("--all-features"); self @@ -60,16 +72,19 @@ impl Cargo { self } + #[inline] pub fn target(&mut self, target: impl AsRef) -> &mut Self { self.arg("--target").arg(target); self } + #[inline] pub fn package(&mut self, package: impl AsRef) -> &mut Self { self.arg("--package").arg(package); self } + #[inline] pub fn release(&mut self) -> &mut Self { self.arg("--release"); self diff --git a/xtask/src/command/dir.rs b/xtask/src/command/dir.rs index 1223af80c..f716e117e 100644 --- a/xtask/src/command/dir.rs +++ b/xtask/src/command/dir.rs @@ -2,6 +2,7 @@ use std::{ fs, + io::ErrorKind, path::{Path, PathBuf}, }; @@ -10,12 +11,16 @@ use std::{ /// 如果返回 `Ok(())`，`path` 将不存在。 pub fn rm(path: impl AsRef) -> std::io::Result<()> { let path = path.as_ref(); - if !path.exists() { - Ok(()) - } else if path.is_dir() { + if path.is_dir() { fs::remove_dir_all(path) + } else if let Err(e) = fs::remove_file(path) { + if matches!(e.kind(), ErrorKind::NotFound) { + Ok(()) + } else { + Err(e) + } } else { - fs::remove_file(path) + Ok(()) } } diff --git a/xtask/src/linux/image.rs b/xtask/src/linux/image.rs index 6344a416d..131c36961 100644 --- a/xtask/src/linux/image.rs +++ b/xtask/src/linux/image.rs @@ -1,9 +1,12 @@ use super::{LinuxRootfs, LIBOS_MUSL_LIBC_PATH}; use crate::{ - command::{CommandExt, Qemu}, + command::{dir, download::wget, CommandExt, Qemu, Tar}, Arch, }; -use std::{fs, path::Path}; +use std::{ + fs, + path::{Path, PathBuf}, +}; impl LinuxRootfs { /// 生成镜像。 @@ -12,6 +15,25 @@ impl LinuxRootfs { self.make(false); // 镜像路径 let image = format!("zCore/{arch}.img", arch = self.0.name()); + // aarch64 还需要下载 firmware + if let Arch::Aarch64 = self.0 { + const URL:&str = "https://github.com/Luchangcheng2333/rayboot/releases/download/2.0.0/aarch64_firmware.tar.gz"; + let aarch64_tar = self.0.origin().join("Aarch64_firmware.zip"); + wget(URL, &aarch64_tar); + + let fw_dir = self.0.target().join("firmware"); + dir::clear(&fw_dir).unwrap(); + Tar::xf(&aarch64_tar, Some(&fw_dir)).invoke(); + + let boot_dir = PathBuf::from("zCore/disk/EFI/Boot"); + dir::clear(&boot_dir).unwrap(); + fs::copy( + fw_dir.join("aarch64_uefi.efi"), + boot_dir.join("bootaa64.efi"), + ) + .unwrap(); + fs::copy(fw_dir.join("Boot.json"), boot_dir.join("Boot.json")).unwrap(); + } // 生成镜像 match self.0 { Arch::Riscv64 | Arch::Aarch64 => fuse(self.path(), &image), diff --git a/xtask/src/linux/mod.rs b/xtask/src/linux/mod.rs index 322c5527b..64fa2c109 100644 --- a/xtask/src/linux/mod.rs +++ b/xtask/src/linux/mod.rs @@ -1,13 +1,13 @@ use crate::{ - command::{dir, download::wget, CommandExt, Tar}, - Arch, ALPINE_ROOTFS_VERSION, ALPINE_WEBSITE, + command::{dir, download::wget, CommandExt, Ext, Tar}, + Arch, }; use std::{ env, - ffi::{OsStr, OsString}, + ffi::OsString, fs, os::unix, - path::PathBuf, + path::{Path, PathBuf}, }; mod image; @@ -68,6 +68,15 @@ impl LinuxRootfs { } } + /// 将 musl 动态库放入 rootfs。 + pub fn put_musl_libs(&self) -> PathBuf { + // 递归 rootfs + self.make(false); + let dir = self.0.linux_musl_cross(); + self.put_libs(&dir, dir.join(format!("{}-linux-musl", self.0.name()))); + dir + } + /// 指定架构的 rootfs 路径。 #[inline] fn path(&self) -> PathBuf { @@ -77,43 +86,20 @@ impl LinuxRootfs { /// 下载并解压 minirootfs。 fn prebuild_rootfs(&self) -> PathBuf { // 构造压缩文件路径 - let file_name = match self.0 { + let tar = self.0.origin().join(match self.0 { Arch::Riscv64 => "minirootfs.tar.xz", Arch::X86_64 | Arch::Aarch64 => "minirootfs.tar.gz", + }); + // 构造下载地址 + let url = match self.0 { + Arch::Riscv64 => + String::from("https://github.com/rcore-os/libc-test-prebuilt/releases/download/0.1/prebuild.tar.xz"), + Arch::X86_64 | Arch::Aarch64 => + // 只能使用 3.12 这个版本 + format!("https://dl-cdn.alpinelinux.org/alpine/v3.12/releases/{arch}/alpine-minirootfs-3.12.0-{arch}.tar.gz", arch = self.0.name()), }; - let tar = self.0.origin().join(file_name); - // 若压缩文件不存在，需要下载 - if !tar.exists() { - let url = match self.0 { - Arch::Riscv64 => String::from("https://github.com/rcore-os/libc-test-prebuilt/releases/download/0.1/prebuild.tar.xz"), - Arch::X86_64 => format!("{ALPINE_WEBSITE}/x86_64/alpine-minirootfs-{ALPINE_ROOTFS_VERSION}-x86_64.tar.gz"), - Arch::Aarch64 => format!("{ALPINE_WEBSITE}/aarch64/alpine-minirootfs-{ALPINE_ROOTFS_VERSION}-aarch64.tar.gz") - }; - wget(url, &tar); - } - match self.0 { - Arch::Aarch64 => { - let aarch64_file = "Aarch64_firmware.zip"; - let aarch64_tar = self.0.origin().join(aarch64_file); - if !aarch64_tar.exists() { - let url = "https://github.com/Luchangcheng2333/rayboot/releases/download/2.0.0/aarch64_firmware.tar.gz"; - wget(url, &aarch64_tar); - } - let fw_dir = self.0.target().join("firmware"); - dir::clear(&fw_dir).unwrap(); - let mut aarch64_tar = Tar::xf(&aarch64_tar, Some(&fw_dir)); - aarch64_tar.invoke(); - let boot_dir = "zCore/disk/EFI/Boot/"; - fs::create_dir_all(boot_dir).ok(); - fs::copy( - fw_dir.join("aarch64_uefi.efi"), - "zCore/disk/EFI/Boot/bootaa64.efi", - ) - .unwrap(); - fs::copy(fw_dir.join("Boot.json"), "zCore/disk/EFI/Boot/Boot.json").ok(); - } - _ => {} - } + // 下载 + wget(url, &tar); // 解压到目标路径 let dir = self.0.target().join("rootfs"); dir::clear(&dir).unwrap(); @@ -124,32 +110,45 @@ impl LinuxRootfs { } dir } -} - -/// 下载 musl 工具链，返回工具链路径。 -fn linux_musl_cross(arch: Arch) -> PathBuf { - let name = format!("{}-linux-musl-cross", arch.name().to_lowercase()); - let name: &str = name.as_str(); - let origin = arch.origin(); - let target = arch.target(); - - let tgz = origin.join(format!("{name}.tgz")); - let dir = target.join(name); - - dir::rm(&dir).unwrap(); - wget(format!("https://musl.cc/{name}.tgz"), &tgz); - Tar::xf(&tgz, Some(target)).invoke(); - - // 将交叉工具链加入 PATH 环境变量 - env::current_dir().unwrap().join(dir).join("bin") + /// 从安装目录拷贝所有 so 和 so 链接到 rootfs + fn put_libs(&self, musl: impl AsRef, dir: impl AsRef) { + let lib = self.path().join("lib"); + let musl_libc_protected = format!("ld-musl-{}.so.1", self.0.name()); + let musl_libc_ignored = "libc.so"; + let strip = musl + .as_ref() + .join("bin") + .join(format!("{}-linux-musl-strip", self.0.name())); + dir.as_ref() + .join("lib") + .read_dir() + .unwrap() + .filter_map(|res| res.map(|e| e.path()).ok()) + .filter(|path| check_so(path)) + .for_each(|source| { + let name = source.file_name().unwrap(); + let target = lib.join(name); + if source.is_symlink() { + if name != musl_libc_protected.as_str() { + dir::rm(&target).unwrap(); + // `fs::copy` 会拷贝文件内容 + unix::fs::symlink(source.read_link().unwrap(), target).unwrap(); + } + } else if name != musl_libc_ignored { + dir::rm(&target).unwrap(); + fs::copy(source, &target).unwrap(); + Ext::new(&strip).arg("-s").arg(target).status(); + } + }); + } } /// 为 PATH 环境变量附加路径。 fn join_path_env(paths: I) -> OsString where I: IntoIterator, - S: AsRef, + S: AsRef, { let mut path = OsString::new(); let mut first = true; @@ -163,7 +162,28 @@ where } else { path.push(":"); } - path.push(item); + path.push(item.as_ref().canonicalize().unwrap().as_os_str()); } path } + +fn check_so>(path: P) -> bool { + let path = path.as_ref(); + // 是符号链接或文件 + // 对于符号链接，`is_file` `exist` 等函数都会针对其指向的真实文件判断 + if !path.is_symlink() && !path.is_file() { + return false; + } + let name = path.file_name().unwrap().to_string_lossy(); + let mut seg = name.split('.'); + // 不能以 . 开头 + if matches!(seg.next(), Some("") | None) { + return false; + } + // 扩展名的第一项是 so + if !matches!(seg.next(), Some("so")) { + return false; + } + // so 之后全是纯十进制数字 + !seg.any(|it| !it.chars().all(|ch| ch.is_ascii_digit())) +} diff --git a/xtask/src/linux/opencv.rs b/xtask/src/linux/opencv.rs index e12ed4e8e..4227f0d9e 100644 --- a/xtask/src/linux/opencv.rs +++ b/xtask/src/linux/opencv.rs @@ -1,6 +1,6 @@ -use super::{join_path_env, linux_musl_cross}; +use super::join_path_env; use crate::{ - command::{download::fetch_online, CommandExt, Ext, Git, Make}, + command::{dir, download::fetch_online, CommandExt, Ext, Git, Make}, Arch, ORIGIN, }; use std::{ @@ -11,7 +11,7 @@ use std::{ impl super::LinuxRootfs { pub fn put_opencv(&self) { // 递归 rootfs - self.make(false); + let musl = self.put_musl_libs(); // 拉 opencv let opencv = PathBuf::from(ORIGIN).join("opencv"); if !opencv.is_dir() { @@ -23,60 +23,64 @@ impl super::LinuxRootfs { .done() }); } - // 构建 - let opencv = opencv.canonicalize().unwrap(); - let build = self.0.target().join("opencv"); - match self.0 { - Arch::Riscv64 => { - // 工具链路径放入 PATH - let path_with_musl_gcc = join_path_env(&[linux_musl_cross(self.0)]); - // ffmpeg 路径 - let ffmpeg = PathBuf::from(ORIGIN) - .join("ffmpeg") - .join("install") - .join("lib"); - // 创建平台相关 cmake - let platform_cmake = self.0.target().join("riscv64-musl-gcc.toolchain.cmake"); - fs::write(&platform_cmake, riscv64_opencv_cmake(&ffmpeg)).unwrap(); - // 创建生成目录 - fs::create_dir_all(&build).unwrap(); - let mut cmake = Ext::new("cmake"); - if ffmpeg.is_dir() { - cmake.env( - "PKG_CONFIG_LIBDIR", - ffmpeg.join("pkgconfig").canonicalize().unwrap(), - ); - } - cmake - .current_dir(&build) - .arg(format!( - "-DCMAKE_TOOLCHAIN_FILE={}", - platform_cmake.canonicalize().unwrap().display() - )) - .arg("-DWITH_FFMPEG=ON") - .arg("-DCMAKE_BUILD_TYPE=Release") - .arg(format!( - "-DCMAKE_INSTALL_PREFIX={}", - build.canonicalize().unwrap().join("install").display(), - )) - .arg(opencv) - .env("PATH", &path_with_musl_gcc) - .invoke(); - Make::install() - .current_dir(&build) - .j(num_cpus::get().min(8)) // 不能用太多线程，以免爆内存 - .env("PATH", path_with_musl_gcc) - .invoke(); + let source = opencv.canonicalize().unwrap(); + let target = self.0.target().join("opencv"); + // 如果 build 目录不存在，需要执行 cmake + let cmake_needed = !target.is_dir(); + // 如果执行了 cmake 或安装目录不存在，需要 make + let install_needed = cmake_needed || !target.join("install").is_dir(); + // 工具链 + let path_with_musl_gcc = join_path_env(&[musl.join("bin")]); + // + if cmake_needed { + dir::clear(&target).unwrap(); + // ffmpeg 路径 + let ffmpeg = PathBuf::from(ORIGIN) + .join("ffmpeg") + .join("install") + .join("lib"); + // 创建平台相关 cmake + let platform_cmake = self.0.target().join("musl-gcc.toolchain.cmake"); + fs::write(&platform_cmake, self.opencv_cmake(&ffmpeg)).unwrap(); + // 执行 + let mut cmake = Ext::new("cmake"); + if ffmpeg.is_dir() { + cmake.env( + "PKG_CONFIG_LIBDIR", + ffmpeg.join("pkgconfig").canonicalize().unwrap(), + ); } - Arch::X86_64 | Arch::Aarch64 => todo!(), + cmake + .current_dir(&target) + .arg(format!( + "-DCMAKE_TOOLCHAIN_FILE={}", + platform_cmake.canonicalize().unwrap().display() + )) + .arg("-DWITH_FFMPEG=ON") + .arg("-DCMAKE_BUILD_TYPE=Release") + .arg(format!( + "-DCMAKE_INSTALL_PREFIX={}", + target.canonicalize().unwrap().join("install").display(), + )) + .arg(source) + .env("PATH", &path_with_musl_gcc) + .invoke(); + } + // + if install_needed { + Make::install() + .current_dir(&target) + .j(num_cpus::get().min(8)) // 不能用太多线程，以免爆内存 + .env("PATH", path_with_musl_gcc) + .invoke(); } // 拷贝 - self.put_libs(build); + self.put_libs(musl, target.join("install")); } pub fn put_ffmpeg(&self) { // 递归 rootfs - self.make(false); + let musl = self.put_musl_libs(); // 拉 ffmpeg let ffmpeg = PathBuf::from(ORIGIN).join("ffmpeg"); if !ffmpeg.is_dir() { @@ -92,8 +96,8 @@ impl super::LinuxRootfs { // 构建 match self.0 { Arch::Riscv64 => { - let path_with_musl_gcc = join_path_env(&[linux_musl_cross(self.0)]); - println!("Configuring ffmpeg, please waiting..."); + let path_with_musl_gcc = join_path_env(&[musl.join("bin")]); + println!("Configuring ffmpeg, please wait..."); Ext::new("./configure") .current_dir(&ffmpeg) .arg("--enable-cross-compile") @@ -118,34 +122,16 @@ impl super::LinuxRootfs { Arch::X86_64 | Arch::Aarch64 => todo!(), } // 拷贝 - self.put_libs(ffmpeg); - } - - /// 从安装目录拷贝所有 so 和 so 链接到 rootfs - fn put_libs(&self, build: impl AsRef) { - let lib = self.path().join("lib"); - build - .as_ref() - .join("install") - .join("lib") - .read_dir() - .unwrap() - .filter_map(|res| res.ok()) - .map(|entry| entry.path()) - .filter(|path| { - (path.is_file() || path.is_symlink()) - && path.file_name().unwrap().to_string_lossy().contains(".so") - }) - .for_each(|so| { - let to = lib.join(so.file_name().unwrap()); - fs::copy(so, to).unwrap(); - }); + self.put_libs(musl, ffmpeg.join("install")); } -} -/// 构造一个用于 riscv64 opencv 构建的 cmake 文件。 -fn riscv64_opencv_cmake(ffmpeg: impl AsRef) -> String { - const HEAD: &str = "\ + /// 构造一个用于 opencv 构建的 cmake 文件。 + fn opencv_cmake(&self, ffmpeg: impl AsRef) -> String { + // 不会写 cmake + if !matches!(self.0, Arch::Riscv64) { + todo!(); + } + const HEAD: &str = "\ set(CMAKE_SYSTEM_NAME \"Linux\") set(CMAKE_SYSTEM_PROCESSOR \"riscv64\") @@ -158,17 +144,18 @@ set(CMAKE_CXX_FLAGS \"\" CACHE STRING \"\") set(CMAKE_C_FLAGS \"-march=rv64gc ${CMAKE_C_FLAGS} ${CMAKE_PASS_TEST_FLAGS}\") set(CMAKE_CXX_FLAGS \"-march=rv64gc ${CMAKE_CXX_FLAGS} ${CMAKE_PASS_TEST_FLAGS}\")"; - let ffmpeg = ffmpeg.as_ref(); - if ffmpeg.is_dir() { - format!( - "\ + let ffmpeg = ffmpeg.as_ref(); + if ffmpeg.is_dir() { + format!( + "\ {HEAD} set(CMAKE_LD_FFMPEG_FLAGS \"-Wl,-rpath-link,{}\") set(CMAKE_EXE_LINKER_FLAGS \"${{CMAKE_EXE_LINKER_FLAGS}} ${{CMAKE_LD_FFMPEG_FLAGS}}\")", - ffmpeg.canonicalize().unwrap().display() - ) - } else { - HEAD.into() + ffmpeg.canonicalize().unwrap().display() + ) + } else { + HEAD.into() + } } } diff --git a/xtask/src/linux/test.rs b/xtask/src/linux/test.rs index 10a7c5d20..3cf8119c0 100644 --- a/xtask/src/linux/test.rs +++ b/xtask/src/linux/test.rs @@ -1,9 +1,9 @@ -use super::{join_path_env, linux_musl_cross}; +use super::join_path_env; use crate::{ command::{dir, CommandExt, Ext, Make}, Arch, }; -use std::{ffi::OsStr, fs, io::Write}; +use std::{ffi::OsStr, fs}; impl super::LinuxRootfs { /// 将 libc-test 放入 rootfs。 @@ -16,41 +16,25 @@ impl super::LinuxRootfs { dircpy::copy_dir("libc-test", &dir).unwrap(); // 编译 fs::copy(dir.join("config.mak.def"), dir.join("config.mak")).unwrap(); - match self.0 { - Arch::Riscv64 => { - Make::new() - .j(usize::MAX) - .env("ARCH", self.0.name()) - .env("CROSS_COMPILE", "riscv64-linux-musl-") - .env("PATH", join_path_env(&[linux_musl_cross(self.0)])) - .current_dir(&dir) - .invoke(); - fs::copy( - self.0 - .target() - .join("rootfs/libc-test/functional/tls_align-static.exe"), - dir.join("src/functional/tls_align-static.exe"), - ) - .unwrap(); - } - Arch::X86_64 => { - fs::OpenOptions::new() - .append(true) - .open(dir.join("config.mak")) - .unwrap() - .write_all(b"CC := musl-gcc\nAR := ar\nRANLIB := ranlib") - .unwrap(); - Make::new().j(usize::MAX).current_dir(dir).invoke(); - } - Arch::Aarch64 => { - Make::new() - .j(usize::MAX) - .env("ARCH", self.0.name()) - .env("CROSS_COMPILE", "aarch64-linux-musl-") - .env("PATH", join_path_env(&[linux_musl_cross(Arch::Aarch64)])) - .current_dir(&dir) - .invoke(); - } + Make::new() + .j(usize::MAX) + .env("ARCH", self.0.name()) + .env("CROSS_COMPILE", &format!("{}-linux-musl-", self.0.name())) + .env( + "PATH", + join_path_env(&[self.0.linux_musl_cross().join("bin")]), + ) + .current_dir(&dir) + .invoke(); + // FIXME 为什么要替换？ + if let Arch::Riscv64 = self.0 { + fs::copy( + self.0 + .target() + .join("rootfs/libc-test/functional/tls_align-static.exe"), + dir.join("src/functional/tls_align-static.exe"), + ) + .unwrap(); } } @@ -58,44 +42,32 @@ impl super::LinuxRootfs { pub fn put_other_test(&self) { // 递归 rootfs self.make(false); - let rootfs = self.path(); - match self.0 { - Arch::Riscv64 => { - dircpy::copy_dir(self.0.target().join("rootfs/oscomp"), rootfs.join("oscomp")) - .unwrap(); - } - Arch::X86_64 => { - let bin = rootfs.join("bin"); - fs::read_dir("linux-syscall/test") - .unwrap() - .filter_map(|res| res.ok()) - .map(|entry| entry.path()) - .filter(|path| path.extension().map_or(false, |ext| ext == OsStr::new("c"))) - .for_each(|c| { - Ext::new("gcc") - .arg(&c) - .arg("-o") - .arg(bin.join(c.file_prefix().unwrap())) - .arg("-Wl,--dynamic-linker=/lib/ld-musl-x86_64.so.1") - .invoke(); - }); - } - Arch::Aarch64 => { - let musl_cross = linux_musl_cross(self.0); - let bin = rootfs.join("bin"); - fs::read_dir("linux-syscall/test") - .unwrap() - .filter_map(|res| res.ok()) - .map(|entry| entry.path()) - .filter(|path| path.extension().map_or(false, |ext| ext == OsStr::new("c"))) - .for_each(|c| { - Ext::new(musl_cross.join("aarch64-linux-musl-gcc")) - .arg(&c) - .arg("-o") - .arg(bin.join(c.file_prefix().unwrap())) - .invoke(); - }); - } + // build linux-syscall/test + let bin = self.path().join("bin"); + let musl_cross = self + .0 + .linux_musl_cross() + .join("bin") + .join(format!("{}-linux-musl-gcc", self.0.name())); + fs::read_dir("linux-syscall/test") + .unwrap() + .filter_map(|res| res.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().map_or(false, |ext| ext == OsStr::new("c"))) + .for_each(|c| { + Ext::new(&musl_cross) + .arg(&c) + .arg("-o") + .arg(bin.join(c.file_prefix().unwrap())) + .invoke() + }); + // 再为 riscv64 添加 oscomp + if let Arch::Riscv64 = self.0 { + dircpy::copy_dir( + self.0.target().join("rootfs/oscomp"), + self.path().join("oscomp"), + ) + .unwrap(); } } } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 27f53e066..50b8af796 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,9 +22,6 @@ use command::{Cargo, CommandExt, Ext, Git, Make}; use errors::XError; use linux::LinuxRootfs; -const ALPINE_WEBSITE: &str = "https://dl-cdn.alpinelinux.org/alpine/v3.12/releases"; -const ALPINE_ROOTFS_VERSION: &str = "3.12.0"; - /// The path to store files from network. const ORIGIN: &str = "ignored/origin"; /// The path to cache generated files durning processes. @@ -59,17 +56,22 @@ enum Commands { /// Build rootfs Rootfs(ArchArg), - /// Put opencv lib into rootfs. + /// Put musl libs into rootfs + MuslLibs(ArchArg), + /// Put opencv libs into rootfs Opencv(ArchArg), - /// Put opencv lib into rootfs. + /// Put ffmpeg libs into rootfs Ffmpeg(ArchArg), - /// Put libc test into rootfs. + /// Put libc test into rootfs LibcTest(ArchArg), - /// Put other test into rootfs. + /// Put other test into rootfs OtherTest(ArchArg), /// Build image Image(ArchArg), + /// Run in Linux libos mode + LinuxLibos(LinuxLibosArg), + /// Dump asm of kernel Asm(AsmArgs), /// Run zCore in qemu @@ -88,9 +90,17 @@ struct ProxyPort { global: bool, } +#[derive(Args)] +struct LinuxLibosArg { + /// Command for busybox. + #[clap(short, long)] + pub args: String, +} + fn main() { + use Commands::*; match Cli::parse().command { - Commands::GitProxy(ProxyPort { port, global }) => { + GitProxy(ProxyPort { port, global }) => { if let Some(port) = port { set_git_proxy(global, port); } else { @@ -98,24 +108,29 @@ fn main() { } } #[cfg(not(target_arch = "riscv64"))] - Commands::Dump => dump::dump_config(), - Commands::Setup => { + Dump => dump::dump_config(), + Setup => { make_git_lfs(); git_submodule_update(true); } - Commands::UpdateAll => update_all(), - Commands::CheckStyle => check_style(), - - Commands::Rootfs(arg) => arg.linux_rootfs().make(true), - Commands::Opencv(arg) => arg.linux_rootfs().put_opencv(), - Commands::Ffmpeg(arg) => arg.linux_rootfs().put_ffmpeg(), - Commands::LibcTest(arg) => arg.linux_rootfs().put_libc_test(), - Commands::OtherTest(arg) => arg.linux_rootfs().put_other_test(), - Commands::Image(arg) => arg.linux_rootfs().image(), - - Commands::Asm(args) => args.asm(), - Commands::Qemu(args) => args.qemu(), - Commands::Gdb(args) => args.gdb(), + UpdateAll => update_all(), + CheckStyle => check_style(), + + Rootfs(arg) => arg.linux_rootfs().make(true), + MuslLibs(arg) => { + arg.linux_rootfs().put_musl_libs(); + } + Opencv(arg) => arg.linux_rootfs().put_opencv(), + Ffmpeg(arg) => arg.linux_rootfs().put_ffmpeg(), + LibcTest(arg) => arg.linux_rootfs().put_libc_test(), + OtherTest(arg) => arg.linux_rootfs().put_other_test(), + Image(arg) => arg.linux_rootfs().image(), + + LinuxLibos(arg) => linux_libos(arg.args), + + Asm(args) => args.asm(), + Qemu(args) => args.qemu(), + Gdb(args) => args.gdb(), } } @@ -202,3 +217,16 @@ fn check_style() { .current_dir("zCore") .invoke(); } + +/// libos 模式执行应用程序。 +fn linux_libos(args: String) { + // 递归 rootfs + LinuxRootfs::new(Arch::X86_64).make(false); + Cargo::run() + .package("zcore") + .release() + .features(true, ["linux"]) + .arg("--") + .args(args.split_whitespace()) + .invoke() +}