Merge pull request rcore-os#322 from YdrMaster/main

refactor(xtask): 统一各种架构编译测例的过程
YdrMaster · Jun 14, 2022 · 0b2ac1e · 0b2ac1e
2 parents fd67f17 + 904644e
commit 0b2ac1e
Show file tree

Hide file tree

Showing 18 changed files with 435 additions and 339 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -7,12 +7,15 @@ update-all = "xtask update-all"
 check-style = "xtask check-style"
 
 rootfs = "xtask rootfs"
+musl-libs = "xtask musl-libs"
 opencv = "xtask opencv"
 ffmpeg = "xtask ffmpeg"
 libc-test = "xtask libc-test"
 other-test = "xtask other-test"
 image = "xtask image"
 
+linux-libos = "xtask linux-libos"
+
 asm = "xtask asm"
 qemu = "xtask qemu"
 gdb = "xtask gdb"
diff --git a/.github/scripts/install-deps.sh b/.github/scripts/install-deps.sh
diff --git a/.github/scripts/install-qemu.sh → .github/scripts/make-qemu.sh b/.github/scripts/install-qemu.sh → .github/scripts/make-qemu.sh
@@ -4,4 +4,4 @@ wget https://download.qemu.org/qemu-$1.tar.xz
 tar -xJf qemu-$1.tar.xz
 cd qemu-$1
 ./configure --target-list=x86_64-softmmu,riscv64-softmmu,aarch64-softmmu
-make -j$nproc > /dev/null 2>&1
+make -j > /dev/null 2>&1
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -46,41 +46,6 @@ jobs:
           command: doc
           args: --all-features --no-deps
 
-  build-aarch64:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ env.rust_toolchain }}
-          override: true
-          target: aarch64-unknown-linux-gnu
-
-      - uses: actions-rs/cargo@v1
-        with:
-          command: build
-          use-cross: true
-          args: --target aarch64-unknown-linux-gnu --workspace --exclude linux-syscall --exclude zcore-loader --exclude zcore
-
-  build-user:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Pull prebuilt images
-        run: git lfs pull -I prebuilt/zircon/x64/libc.so,prebuilt/zircon/x64/libfdio.so,prebuilt/zircon/x64/libunwind.so,prebuilt/zircon/x64/libzircon.so,prebuilt/zircon/x64/Scrt1.o
-
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ env.rust_toolchain }}
-          target: x86_64-fuchsia
-
-      - name: Build Zircon user programs
-        run: cd zircon-user && make build MODE=release
-
   test-libos:
     runs-on: ubuntu-latest
     strategy:
@@ -153,3 +118,20 @@ jobs:
       - name: Clippy ${{ matrix.arch }} bare-metal linux
         if: matrix.arch == 'aarch64'
         run: cd zCore && make clippy ARCH=${{ matrix.arch }} LINUX=1
+
+  build-user:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Pull prebuilt images
+        run: git lfs pull -I prebuilt/zircon/x64/libc.so,prebuilt/zircon/x64/libfdio.so,prebuilt/zircon/x64/libunwind.so,prebuilt/zircon/x64/libzircon.so,prebuilt/zircon/x64/Scrt1.o
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ env.rust_toolchain }}
+          target: x86_64-fuchsia
+
+      - name: Build Zircon user programs
+        run: cd zircon-user && make build MODE=release
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -1,4 +1,4 @@
-name: Deploy docs
+name: Deploy docs
 
 on:
   push:
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
-      
+
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -87,7 +87,7 @@ jobs:
           git lfs pull -I prebuilt/zircon/x64/userboot-libos.so
 
       - name: Install python dependencies
-        run: .github/scripts/install-deps.sh
+        run: pip3 install -r tests/requirements.txt
 
       - name: Run fast tests
         if: github.event_name != 'schedule'
@@ -97,8 +97,9 @@ jobs:
         if: github.event_name == 'schedule'
         run: cd tests && python3 zircon_core_test.py --libos
 
-  linux-libc-test-libos:
-    name: Linux Libc Test Libos
+
+  zircon-core-test:
+    name: Zircon Core Test Baremetal
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
@@ -111,22 +112,44 @@ jobs:
           toolchain: ${{ env.rust_toolchain }}
           components: rust-src, llvm-tools-preview, rustfmt, clippy
 
-      - name: Install dependencies
-        run: .github/scripts/install-deps.sh musl-tools musl-dev
+      - name: Pull prebuilt images
+        run: |
+          git lfs pull -I prebuilt/zircon/x64/core-tests.zbi
+          git lfs pull -I prebuilt/zircon/x64/libzircon.so
+          git lfs pull -I prebuilt/zircon/x64/userboot.so
+
+      - name: Cache QEMU
+        id: cache-qemu
+        uses: actions/cache@v3
+        with:
+          path: qemu-${{ env.qemu_version }}
+          key: qemu-${{ env.qemu_version }}
 
-      - name: Prepare rootfs
-        run: make libc-test
+      - name: Install ninja-build
+        run: sudo apt-get update && sudo apt-get install -y ninja-build
+
+      - name: Download and Compile QEMU
+        if: steps.cache-qemu.outputs.cache-hit != 'true'
+        run: .github/scripts/make-qemu.sh ${{ env.qemu_version }}
+
+      - name: Install QEMU
+        run: |
+          cd qemu-${{ env.qemu_version }} && sudo make install
+          qemu-system-x86_64 --version
+
+      - name: Install python dependencies
+        run: pip3 install -r tests/requirements.txt
 
       - name: Run fast tests
         if: github.event_name != 'schedule'
-        run: cd tests && python3 linux_libc_test.py --libos --fast
+        run: cd tests && python3 zircon_core_test.py --fast
 
       - name: Run full tests
         if: github.event_name == 'schedule'
-        run: cd tests && python3 linux_libc_test.py --libos
+        run: cd tests && python3 zircon_core_test.py
 
-  zircon-core-test-baremetal:
-    name: Zircon Core Test Baremetal
+  linux-libc-test-libos:
+    name: Linux Libc Test Libos
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
@@ -139,38 +162,25 @@ jobs:
           toolchain: ${{ env.rust_toolchain }}
           components: rust-src, llvm-tools-preview, rustfmt, clippy
 
-      - name: Pull prebuilt images
-        run: |
-          git lfs pull -I prebuilt/zircon/x64/core-tests.zbi
-          git lfs pull -I prebuilt/zircon/x64/libzircon.so
-          git lfs pull -I prebuilt/zircon/x64/userboot.so
-
-      - name: Install dependencies
-        run: .github/scripts/install-deps.sh ninja-build
-
-      - name: Cache QEMU
-        id: cache-qemu
+      - name: Cache musl toolchain
         uses: actions/cache@v3
         with:
-          path: qemu-${{ env.qemu_version }}
-          key: qemu-${{ env.qemu_version }}-x86_64-riscv64
+          path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz
+          key: ${{ matrix.arch }}-linux-musl-cross
 
-      - name: Download and Compile QEMU
-        if: steps.cache-qemu.outputs.cache-hit != 'true'
-        run: .github/scripts/install-qemu.sh ${{ env.qemu_version }}
+      - name: Prepare rootfs
+        run: make libc-test
 
-      - name: Install QEMU
-        run: |
-          cd qemu-${{ env.qemu_version }} && sudo make install
-          qemu-system-x86_64 --version
+      - name: Install python dependencies
+        run: pip3 install -r tests/requirements.txt
 
       - name: Run fast tests
         if: github.event_name != 'schedule'
-        run: cd tests && python3 zircon_core_test.py --fast
+        run: cd tests && python3 linux_libc_test.py --libos --fast
 
       - name: Run full tests
         if: github.event_name == 'schedule'
-        run: cd tests && python3 zircon_core_test.py
+        run: cd tests && python3 linux_libc_test.py --libos
 
   linux-libc-test-baremetal:
     name: Linux Libc Test Baremetal
@@ -196,28 +206,37 @@ jobs:
           crate: cargo-binutils
           version: latest
 
-      - name: Install dependencies
-        run: .github/scripts/install-deps.sh musl-tools musl-dev ninja-build
-
       - name: Cache QEMU
         id: cache-qemu
         uses: actions/cache@v3
         with:
           path: qemu-${{ env.qemu_version }}
-          key: qemu-${{ env.qemu_version }}-x64-rv64-aa64
+          key: qemu-${{ env.qemu_version }}
+
+      - name: Install ninja-build
+        run: sudo apt-get update && sudo apt-get install -y ninja-build
 
       - name: Download and Compile QEMU
         if: steps.cache-qemu.outputs.cache-hit != 'true'
-        run: .github/scripts/install-qemu.sh ${{ env.qemu_version }}
+        run: .github/scripts/make-qemu.sh ${{ env.qemu_version }}
 
       - name: Install QEMU
         run: |
           cd qemu-${{ env.qemu_version }} && sudo make install
           qemu-system-${{ matrix.arch }} --version
 
+      - name: Cache musl toolchain
+        uses: actions/cache@v3
+        with:
+          path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz
+          key: ${{ matrix.arch }}-linux-musl-cross
+
       - name: Prepare rootfs
         run: make libc-test ARCH=${{ matrix.arch }} && make image ARCH=${{ matrix.arch }}
 
+      - name: Install python dependencies
+        run: pip3 install -r tests/requirements.txt
+
       - name: Run fast tests
         if: github.event_name != 'schedule'
         run: cd tests && python3 linux_libc_test.py --arch ${{ matrix.arch }} --fast
@@ -250,28 +269,37 @@ jobs:
           crate: cargo-binutils
           version: latest
 
-      - name: Install dependencies
-        run: .github/scripts/install-deps.sh musl-tools musl-dev ninja-build
-
       - name: Cache QEMU
         id: cache-qemu
         uses: actions/cache@v3
         with:
           path: qemu-${{ env.qemu_version }}
-          key: qemu-${{ env.qemu_version }}-x64-rv64-aa64
+          key: qemu-${{ env.qemu_version }}
+
+      - name: Install ninja-build
+        run: sudo apt-get update && sudo apt-get install -y ninja-build
 
       - name: Download and Compile QEMU
         if: steps.cache-qemu.outputs.cache-hit != 'true'
-        run: .github/scripts/install-qemu.sh ${{ env.qemu_version }}
+        run: .github/scripts/make-qemu.sh ${{ env.qemu_version }}
 
       - name: Install QEMU
         run: |
           cd qemu-${{ env.qemu_version }} && sudo make install
           qemu-system-${{ matrix.arch }} --version
 
+      - name: Cache musl toolchain
+        uses: actions/cache@v3
+        with:
+          path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz
+          key: ${{ matrix.arch }}-linux-musl-cross
+
       - name: Prepare rootfs
         run: make other-test ARCH=${{ matrix.arch }} && make image ARCH=${{ matrix.arch }}
 
+      - name: Install python dependencies
+        run: pip3 install -r tests/requirements.txt
+
       - name: Run fast tests
         if: github.event_name != 'schedule'
         run: cd tests && python3 linux_other_test.py --arch ${{ matrix.arch }} --fast

diff --git a/.github/workflows/third-party.yml b/.github/workflows/third-party.yml
@@ -22,5 +22,11 @@ jobs:
           profile: minimal
           toolchain: ${{ env.rust_toolchain }}
 
-      - name: Build riscv64 opencv
+      - name: Cache musl toolchain
+        uses: actions/cache@v3
+        with:
+          path: ignored/origin/${{ matrix.arch }}/${{ matrix.arch }}-linux-musl-cross.tgz
+          key: ${{ matrix.arch }}-linux-musl-cross
+
+      - name: Build riscv64 library
         run: cargo ${{ matrix.item }} --arch riscv64
diff --git a/xtask/CHANGLOG.md b/xtask/CHANGLOG.md
@@ -2,6 +2,19 @@
 
 最新的更新将出现在最上方。
 
+## 20220614 (YdrMaster)
+
+- 增加 `cargo musl-libs --args <args>`，拷贝实际编译使用的 musl 工具链提供的动态库到 rootfs。
+
+## 20220613 (YdrMaster)
+
+- 增加 `cargo linux-libos --args <args>`，以 linux libos 模式启动，并传递指定参数。
+  例如 `cargo linux-libos --args /bin/busybox ls`；
+
+## 20220612 (YdrMaster)
+
+- 统一各种架构编译测例的过程（x86_64 现在也下载 musl toolchain）；
+
 ## 20220610 (YdrMaster)
 
 - 修改 command/download，使用宏支持多种方式下载；

diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
@@ -9,7 +9,7 @@ build = "build.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap = { version = "3.1", features = ["derive"] }
+clap = { version = "~3.1", features = ["derive"] }
 dircpy = "0.3"
 rand = "0.8"
 lazy_static = "1.4.0"

diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs
@@ -1,6 +1,9 @@
 //! 支持架构的定义。
 
-use crate::{LinuxRootfs, XError, ORIGIN, TARGET};
+use crate::{
+    command::{dir, download::wget, CommandExt, Tar},
+    LinuxRootfs, XError, ORIGIN, TARGET,
+};
 use std::{path::PathBuf, str::FromStr};
 
 /// 支持的 CPU 架构。
@@ -33,6 +36,23 @@ impl Arch {
     pub fn target(&self) -> PathBuf {
         PathBuf::from(TARGET).join(self.name())
     }
+
+    /// 下载 musl 工具链，返回工具链路径。
+    pub fn linux_musl_cross(&self) -> PathBuf {
+        let name = format!("{}-linux-musl-cross", self.name().to_lowercase());
+
+        let origin = self.origin();
+        let target = self.target();
+
+        let tgz = origin.join(format!("{name}.tgz"));
+        let dir = target.join(&name);
+
+        dir::rm(&dir).unwrap();
+        wget(format!("https://musl.cc/{name}.tgz"), &tgz);
+        Tar::xf(&tgz, Some(target)).invoke();
+
+        dir
+    }
 }
 
 impl FromStr for Arch {