From d85592d9a9b3e43db94dbbf9358f1359a64a9cc6 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:24:10 +0900 Subject: [PATCH 01/11] chore(cross): Split images --- ...push-cross-images.yml => cross-images.yml} | 19 +-- rust/cubestore/cross/.env | 1 - ...arch64-unknown-linux-gnu-python.Dockerfile | 27 +++++ .../aarch64-unknown-linux-gnu.Dockerfile | 114 ++---------------- rust/cubestore/cross/docker-bake.hcl | 45 +++++++ 5 files changed, 87 insertions(+), 119 deletions(-) rename .github/workflows/{push-cross-images.yml => cross-images.yml} (58%) delete mode 100644 rust/cubestore/cross/.env create mode 100644 rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile create mode 100644 rust/cubestore/cross/docker-bake.hcl diff --git a/.github/workflows/push-cross-images.yml b/.github/workflows/cross-images.yml similarity index 58% rename from .github/workflows/push-cross-images.yml rename to .github/workflows/cross-images.yml index 8c46db34ef87b..bf25937d4d8e1 100644 --- a/.github/workflows/push-cross-images.yml +++ b/.github/workflows/cross-images.yml @@ -3,13 +3,13 @@ name: Cross Images on: push: paths: - - '.github/workflows/push-cross-images.yml' + - '.github/workflows/cross-images.yml' - 'rust/cubestore/cross/**' branches: - 'master' pull_request: paths: - - '.github/workflows/push-cross-images.yml' + - '.github/workflows/cross-images.yml' - 'rust/cubestore/cross/**' jobs: @@ -33,17 +33,8 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Load .cross file - uses: xom9ikk/dotenv@v2.3.0 + - name: Build and push + uses: docker/bake-action@v5 with: - path: rust/cubestore/cross/ - - name: Push to Docker Hub - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./rust/cubestore/cross/${{ matrix.target }}.Dockerfile - platforms: linux/amd64 + workdir: ./rust/cubestore/cross push: ${{ github.ref == 'refs/heads/master' }} - tags: cubejs/rust-cross:${{ matrix.target }},cubejs/rust-cross:${{ matrix.target }}-${{ env.CROSS_VERSION }} diff --git a/rust/cubestore/cross/.env b/rust/cubestore/cross/.env deleted file mode 100644 index 83d9d29c613aa..0000000000000 --- a/rust/cubestore/cross/.env +++ /dev/null @@ -1 +0,0 @@ -CROSS_VERSION=01072024 diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile new file mode 100644 index 0000000000000..189d9479e521b --- /dev/null +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile @@ -0,0 +1,27 @@ +FROM base + +ARG PYTHON_VERSION +ARG PYTHON_RELEASE + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_VERSION_RELEASE} \ + && get autoremove -y; + +RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ + # python is required for cross compiling python :D + && cd Python-${PYTHON_VERSION} \ + && touch config.site-aarch64 \ + && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ + && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \ + && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \ + && CONFIG_SITE=config.site-aarch64 ./configure \ + --enable-shared \ + --enable-optimizations \ + --disable-ipv6 \ + --prefix=/usr/aarch64-linux-gnu \ + --build=aarch64-unknown-linux-gnu \ + --host=x86_64-linux-gnu \ + --with-build-python=/usr/bin/python3.12 \ + && make -j $(nproc) \ + && make install \ + && cd .. && rm -rf Python-${PYTHON_VERSION} \ + && rm -rf /var/lib/apt/lists/*; \ diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile index ee46556c27bda..c1c1ed93e9b2f 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu.Dockerfile @@ -3,28 +3,30 @@ # https://github.com/rust-embedded/cross/blob/master/docker/Dockerfile.aarch64-unknown-linux-gnu FROM cubejs/cross-aarch64-unknown-linux-gnu:31122022 +ARG LLVM_VERSION=18 + RUN apt-get update \ && apt-get -y upgrade \ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget curl apt-transport-https ca-certificates \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" \ + && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-$LLVM_VERSION main" \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y libffi-dev binutils-multiarch binutils-aarch64-linux-gnu gcc-multilib g++-multilib \ # llvm14-dev will install python 3.8 as bin/python3 - && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-18 lld-18 clang-18 libclang-18-dev clang-18 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-$LLVM_VERSION lld-$LLVM_VERSION clang-$LLVM_VERSION libclang-$LLVM_VERSION-dev clang-$LLVM_VERSION \ make cmake libsasl2-dev \ libc6 libc6-dev libc6-arm64-cross libc6-dev-arm64-cross \ gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/*; -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 \ - && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 \ - && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-18 100 \ - && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-18 100 \ - && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-18 100 \ - && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-18 100; +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-$LLVM_VERSION 100; ENV ARCH=arm \ MACHINE=armv8 \ @@ -63,102 +65,6 @@ ENV PYO3_CROSS_PYTHON_VERSION=3.11 \ OPENSSL_LIB_DIR=/usr/aarch64-linux-gnu/lib \ OPENSSL_LIBRARIES=/usr/aarch64-linux-gnu/lib -ENV PYTHON_VERSION=3.12.4 -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - # python is required for cross compiling python :D - && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.12 \ - && cd Python-${PYTHON_VERSION} \ - && touch config.site-aarch64 \ - && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \ - && CONFIG_SITE=config.site-aarch64 ./configure \ - --enable-shared \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr/aarch64-linux-gnu \ - --build=aarch64-unknown-linux-gnu \ - --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python3.12 \ - && make -j $(nproc) \ - && make install \ - && cd .. && rm -rf Python-${PYTHON_VERSION} \ - && apt-get remove -y python3.12 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/*; - -ENV PYTHON_VERSION=3.11.3 -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - # python is required for cross compiling python :D - && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.11 \ - && cd Python-${PYTHON_VERSION} \ - && touch config.site-aarch64 \ - && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \ - && CONFIG_SITE=config.site-aarch64 ./configure \ - --enable-shared \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr/aarch64-linux-gnu \ - --build=aarch64-unknown-linux-gnu \ - --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python3.11 \ - && make -j $(nproc) \ - && make install \ - && cd .. && rm -rf Python-${PYTHON_VERSION} \ - && apt-get remove -y python3.11 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/*; - -ENV PYTHON_VERSION=3.10.11 -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - # python is required for cross compiling python :D - && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.10 \ - && cd Python-${PYTHON_VERSION} \ - && touch config.site-aarch64 \ - && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \ - && CONFIG_SITE=config.site-aarch64 ./configure \ - --enable-shared \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr/aarch64-linux-gnu \ - --build=aarch64-unknown-linux-gnu \ - --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python3.10 \ - && make -j $(nproc) \ - && make install \ - && cd .. && rm -rf Python-${PYTHON_VERSION} \ - && apt-get remove -y python3.10 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/*; - -ENV PYTHON_VERSION=3.9.18 -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - # python is required for cross compiling python :D - && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.9 \ - && cd Python-${PYTHON_VERSION} \ - && touch config.site-aarch64 \ - && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptmx=no" >> config.site-aarch64 \ - && echo "ac_cv_file__dev_ptc=no" >> config.site-aarch64 \ - && CONFIG_SITE=config.site-aarch64 ./configure \ - --enable-shared \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr/aarch64-linux-gnu \ - --build=aarch64-unknown-linux-gnu \ - --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python3.9 \ - && make -j $(nproc) \ - && make install \ - && cd .. && rm -rf Python-${PYTHON_VERSION} \ - && apt-get remove -y python3.9 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/*; - ENV PKG_CONFIG_ALLOW_CROSS=true \ PKG_CONFIG_ALL_STATIC=true \ RUSTFLAGS="-C target-feature=-crt-static" \ diff --git a/rust/cubestore/cross/docker-bake.hcl b/rust/cubestore/cross/docker-bake.hcl new file mode 100644 index 0000000000000..f7b8347fe6548 --- /dev/null +++ b/rust/cubestore/cross/docker-bake.hcl @@ -0,0 +1,45 @@ +variable "CROSS_VERSION" { + default = "15082024" +} + +variable "LLVM_VERSION" { + default = "18" +} + +target "aarch64-unknown-linux-gnu" { + context = "." + dockerfile = "aarch64-unknown-linux-gnu.Dockerfile" + args = { + LLVM_VERSION = LLVM_VERSION + } + tags = ["cubejs/cross-aarch64-unknown-linux-gnu-${CROSS_VERSION}"] + platforms = ["linux/amd64"] +} + +target "aarch64-unknown-linux-gnu-python" { + inherits = ["aarch64-unknown-linux-gnu"] + contexts = { + base = "target:aarch64-unknown-linux-gnu" + } + dockerfile = "aarch64-unknown-linux-gnu-python.Dockerfile" + name = "aarch64-unknown-linux-gnu-python-${replace(item.python_release, ".", "-")}" + matrix = { + item = [ + { + python_version = "3.12.4" + python_release = "3.12" + }, + { + python_version = "3.11.3" + python_release = "3.11" + } + ] + } + args = { + CROSS_VERSION = CROSS_VERSION + PYTHON_VERSION = item.python_version + PYTHON_RELEASE = item.python_release + } + tags = ["cubejs/rust-cross:aarch64-unknown-linux-gnu-${CROSS_VERSION}-python-${item.python_release}"] + platforms = ["linux/amd64"] +} From 708c27382977d85ec622f4c7b808240507e2f24a Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:25:11 +0900 Subject: [PATCH 02/11] chore(cross): Split images --- .github/workflows/cross-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cross-images.yml b/.github/workflows/cross-images.yml index bf25937d4d8e1..6c3ed6bd6cb6f 100644 --- a/.github/workflows/cross-images.yml +++ b/.github/workflows/cross-images.yml @@ -37,4 +37,5 @@ jobs: uses: docker/bake-action@v5 with: workdir: ./rust/cubestore/cross + targets: ${{ matrix.target }} push: ${{ github.ref == 'refs/heads/master' }} From 24317d755f990798c9c3262ee8d40867f492af6b Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:34:21 +0900 Subject: [PATCH 03/11] chore: fixes --- .github/workflows/cross-images.yml | 2 +- rust/cubestore/cross/docker-bake.hcl | 67 ++++++++++++++++++- ...x86_64-unknown-linux-gnu-python.Dockerfile | 22 ++++++ .../cross/x86_64-unknown-linux-gnu.Dockerfile | 18 ++--- .../x86_64-unknown-linux-musl.Dockerfile | 6 +- 5 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 rust/cubestore/cross/x86_64-unknown-linux-gnu-python.Dockerfile diff --git a/.github/workflows/cross-images.yml b/.github/workflows/cross-images.yml index 6c3ed6bd6cb6f..0157a44785818 100644 --- a/.github/workflows/cross-images.yml +++ b/.github/workflows/cross-images.yml @@ -37,5 +37,5 @@ jobs: uses: docker/bake-action@v5 with: workdir: ./rust/cubestore/cross - targets: ${{ matrix.target }} + targets: ${{ matrix.target }}-python push: ${{ github.ref == 'refs/heads/master' }} diff --git a/rust/cubestore/cross/docker-bake.hcl b/rust/cubestore/cross/docker-bake.hcl index f7b8347fe6548..f49860c628981 100644 --- a/rust/cubestore/cross/docker-bake.hcl +++ b/rust/cubestore/cross/docker-bake.hcl @@ -12,7 +12,7 @@ target "aarch64-unknown-linux-gnu" { args = { LLVM_VERSION = LLVM_VERSION } - tags = ["cubejs/cross-aarch64-unknown-linux-gnu-${CROSS_VERSION}"] + tags = ["cubejs/cross:aarch64-unknown-linux-gnu-${CROSS_VERSION}"] platforms = ["linux/amd64"] } @@ -32,6 +32,14 @@ target "aarch64-unknown-linux-gnu-python" { { python_version = "3.11.3" python_release = "3.11" + }, + { + python_version = "3.10.11" + python_release = "3.10" + }, + { + python_version = "3.9.18" + python_release = "3.9" } ] } @@ -43,3 +51,60 @@ target "aarch64-unknown-linux-gnu-python" { tags = ["cubejs/rust-cross:aarch64-unknown-linux-gnu-${CROSS_VERSION}-python-${item.python_release}"] platforms = ["linux/amd64"] } + +# We use -python prefix for ci +target "x86_64-unknown-linux-musl-python" { + context = "." + dockerfile = "x86_64-unknown-linux-musl.Dockerfile" + args = { + LLVM_VERSION = LLVM_VERSION + } + tags = ["cubejs/cross:x86_64-unknown-linux-musl-${CROSS_VERSION}"] + platforms = ["linux/amd64"] +} + +target "x86_64-unknown-linux-gnu" { + context = "." + dockerfile = "x86_64-unknown-linux-gnu.Dockerfile" + args = { + LLVM_VERSION = LLVM_VERSION + } + tags = ["cubejs/cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}"] + platforms = ["linux/amd64"] +} + +target "x86_64-unknown-linux-gnu" { + inherits = ["x86_64-unknown-linux-gnu"] + contexts = { + base = "target:x86_64-unknown-linux-gnu" + } + dockerfile = "x86_64-unknown-linux-gnu-python.Dockerfile" + name = "x86_64-unknown-linux-gnu-python-${replace(item.python_release, ".", "-")}" + matrix = { + item = [ + { + python_version = "3.12.4" + python_release = "3.12" + }, + { + python_version = "3.11.3" + python_release = "3.11" + }, + { + python_version = "3.10.11" + python_release = "3.10" + }, + { + python_version = "3.9.18" + python_release = "3.9" + } + ] + } + args = { + CROSS_VERSION = CROSS_VERSION + PYTHON_VERSION = item.python_version + PYTHON_RELEASE = item.python_release + } + tags = ["cubejs/rust-cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}-python-${item.python_release}"] + platforms = ["linux/amd64"] +} diff --git a/rust/cubestore/cross/x86_64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/x86_64-unknown-linux-gnu-python.Dockerfile new file mode 100644 index 0000000000000..cbdbbf772e83f --- /dev/null +++ b/rust/cubestore/cross/x86_64-unknown-linux-gnu-python.Dockerfile @@ -0,0 +1,22 @@ +FROM base + +ARG PYTHON_VERSION +ARG PYTHON_RELEASE + +RUN cd tmp && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ + && cd Python-${PYTHON_VERSION} && \ + ./configure \ + --enable-shared \ + --with-openssl=/openssl \ + --enable-optimizations \ + --disable-ipv6 \ + --prefix=/usr \ + && make -j $(nproc) \ + && make install \ + && ln -f -s /usr/bin/python${PYTHON_RELEASE} /usr/bin/python3 \ + && cd .. && rm -rf Python-${PYTHON_VERSION}; + +# pyo3 uses python3 to detect version, but there is a bug and it uses python3.9 (system), this force it to use a new python +ENV PYO3_PYTHON=python${PYTHON_RELEASE} + +ENV PATH="/cargo/bin:$PATH" diff --git a/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile b/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile index 1eee90753389f..1bda4931921ff 100644 --- a/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile +++ b/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile @@ -1,21 +1,23 @@ # libc 2.31 python 3.9 FROM debian:bullseye-slim +ARG LLVM_VERSION=18 + RUN apt-get update && apt-get -y upgrade \ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget curl gnupg git apt-transport-https ca-certificates \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main" \ + && add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-$LLVM_VERSION main" \ && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-18 lld-18 clang-18 libclang-18-dev clang-18 make cmake \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-${LLVM_VERSION} lld-${LLVM_VERSION} clang-$LLVM_VERSION libclang-${LLVM_VERSION}-dev clang-$LLVM_VERSION make cmake \ lzma-dev liblzma-dev libpython3-dev \ && rm -rf /var/lib/apt/lists/*; -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 \ - && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 \ - && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-18 100 \ - && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-18 100 \ - && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-18 100 \ - && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-18 100; +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/lld clang-cpp /usr/bin/lld-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-$LLVM_VERSION 100 \ + && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-$LLVM_VERSION 100; # https://www.openssl.org/source/old/1.1.1/ ARG OPENSSL_VERSION=1.1.1w diff --git a/rust/cubestore/cross/x86_64-unknown-linux-musl.Dockerfile b/rust/cubestore/cross/x86_64-unknown-linux-musl.Dockerfile index 9b918e5a21a84..5541d66cf3a6b 100644 --- a/rust/cubestore/cross/x86_64-unknown-linux-musl.Dockerfile +++ b/rust/cubestore/cross/x86_64-unknown-linux-musl.Dockerfile @@ -2,15 +2,17 @@ # https://github.com/rust-embedded/cross/blob/master/docker/Dockerfile.x86_64-unknown-linux-musl FROM rustembedded/cross:x86_64-unknown-linux-musl +ARG LLVM_VERSION=18 + RUN apt-get update \ && apt-get -y upgrade \ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget curl musl-tools libc6-dev apt-transport-https ca-certificates \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" \ + && add-apt-repository "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_VERSION} main" \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt-get update \ # llvm14-dev will install python 3.8 as bin/python3 - && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-18 lld-18 clang-18 libclang-18-dev clang-18 make cmake \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y llvm-${LLVM_VERSION} lld-${LLVM_VERSION} clang-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev clang-${LLVM_VERSION} make cmake \ && rm -rf /var/lib/apt/lists/*; RUN ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \ From b0759d8d64fe07d1efa1699ddbd8f4027a3d56ed Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:35:21 +0900 Subject: [PATCH 04/11] chore: fixes --- ...arch64-unknown-linux-gnu-python.Dockerfile | 2 +- .../cross/x86_64-unknown-linux-gnu.Dockerfile | 44 ------------------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile index 189d9479e521b..2d9db399542db 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile @@ -20,7 +20,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER --prefix=/usr/aarch64-linux-gnu \ --build=aarch64-unknown-linux-gnu \ --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python3.12 \ + --with-build-python=/usr/bin/python${PYTHON_VERSION_RELEASE} \ && make -j $(nproc) \ && make install \ && cd .. && rm -rf Python-${PYTHON_VERSION} \ diff --git a/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile b/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile index 1bda4931921ff..bada80666659d 100644 --- a/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile +++ b/rust/cubestore/cross/x86_64-unknown-linux-gnu.Dockerfile @@ -36,48 +36,4 @@ ENV OPENSSL_DIR=/openssl ENV OPENSSL_ROOT_DIR=/openssl ENV OPENSSL_LIBRARIES=/openssl/lib -ENV PYTHON_VERSION=3.12.4 -RUN cd tmp && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - && cd Python-${PYTHON_VERSION} && \ - ./configure \ - --enable-shared \ - --with-openssl=/openssl \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr \ - && make -j $(nproc) \ - && make install \ - && ln -f -s /usr/bin/python3.12 /usr/bin/python3 \ - && cd .. && rm -rf Python-${PYTHON_VERSION}; - -ENV PYTHON_VERSION=3.11.3 -RUN cd tmp && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - && cd Python-${PYTHON_VERSION} && \ - ./configure \ - --enable-shared \ - --with-openssl=/openssl \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr \ - && make -j $(nproc) \ - && make install \ - && ln -f -s /usr/bin/python3.11 /usr/bin/python3 \ - && cd .. && rm -rf Python-${PYTHON_VERSION}; - -ENV PYTHON_VERSION=3.10.11 -RUN cd tmp && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - && cd Python-${PYTHON_VERSION} && \ - ./configure \ - --enable-shared \ - --with-openssl=/openssl \ - --enable-optimizations \ - --disable-ipv6 \ - --prefix=/usr \ - && make -j $(nproc) \ - && make install \ - && cd .. && rm -rf Python-${PYTHON_VERSION}; - -# pyo3 uses python3 to detect version, but there is a bug and it uses python3.9 (system), this force it to use a new python -ENV PYO3_PYTHON=python3.11 - ENV PATH="/cargo/bin:$PATH" From 5ca9df3c38a03a16966293fcc772cc4ce338a182 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:36:47 +0900 Subject: [PATCH 05/11] chore: fixes --- rust/cubestore/cross/docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cross/docker-bake.hcl b/rust/cubestore/cross/docker-bake.hcl index f49860c628981..6a74f40194d21 100644 --- a/rust/cubestore/cross/docker-bake.hcl +++ b/rust/cubestore/cross/docker-bake.hcl @@ -73,7 +73,7 @@ target "x86_64-unknown-linux-gnu" { platforms = ["linux/amd64"] } -target "x86_64-unknown-linux-gnu" { +target "x86_64-unknown-linux-gnu-python" { inherits = ["x86_64-unknown-linux-gnu"] contexts = { base = "target:x86_64-unknown-linux-gnu" From 47be190e548de290d797f54637edf3ed914383e2 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:42:49 +0900 Subject: [PATCH 06/11] chore: fix build --- .../cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile index 2d9db399542db..4d487441f0891 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile @@ -4,7 +4,7 @@ ARG PYTHON_VERSION ARG PYTHON_RELEASE RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_VERSION_RELEASE} \ - && get autoremove -y; + && rm -rf /var/lib/apt/lists/*; RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ # python is required for cross compiling python :D From 88bbc1834573af08365467a49de9d0bc5ba0b804 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 15:49:48 +0900 Subject: [PATCH 07/11] chore: fix build --- rust/cubestore/cross/README.md | 13 ++++--------- .../aarch64-unknown-linux-gnu-python.Dockerfile | 4 ++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/rust/cubestore/cross/README.md b/rust/cubestore/cross/README.md index 28332e49e4c85..e71102f299aaf 100644 --- a/rust/cubestore/cross/README.md +++ b/rust/cubestore/cross/README.md @@ -19,18 +19,13 @@ Keep in mind: ```sh # dmY -export $(cat .env | xargs) +docker buildx bake x86_64-unknown-linux-gnu-python --push +docker buildx bake aarch64-unknown-linux-gnu-python --push +docker buildx bake x86_64-unknown-linux-musl-python --push -docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION -f x86_64-unknown-linux-gnu.Dockerfile . -docker buildx build --platform linux/amd64 -t cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION -f x86_64-unknown-linux-musl.Dockerfile . -docker buildx build --platform linux/amd64 -t cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION -f aarch64-unknown-linux-gnu.Dockerfile . - -docker push cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION -docker push cubejs/rust-cross:x86_64-unknown-linux-musl-$CROSS_VERSION -docker push cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION +export CROSS_VERSION=15082024 # Verify versions docker run --platform linux/amd64 --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-$CROSS_VERSION cc --version -docker run --platform linux/amd64 --rm -it cubejs/rust-cross:x86_64-unknown-linux-gnu-buster-$CROSS_VERSION cc --version docker run --platform linux/amd64 --rm -it cubejs/rust-cross:aarch64-unknown-linux-gnu-$CROSS_VERSION cc --version ``` diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile index 4d487441f0891..cecd65a2cd35a 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile @@ -3,11 +3,11 @@ FROM base ARG PYTHON_VERSION ARG PYTHON_RELEASE +# python is required for cross compiling python :D RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_VERSION_RELEASE} \ && rm -rf /var/lib/apt/lists/*; RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ - # python is required for cross compiling python :D && cd Python-${PYTHON_VERSION} \ && touch config.site-aarch64 \ && echo "ac_cv_buggy_getaddrinfo=no" >> config.site-aarch64 \ @@ -20,7 +20,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER --prefix=/usr/aarch64-linux-gnu \ --build=aarch64-unknown-linux-gnu \ --host=x86_64-linux-gnu \ - --with-build-python=/usr/bin/python${PYTHON_VERSION_RELEASE} \ + --with-build-python=/usr/bin/python${PYTHON_RELEASE} \ && make -j $(nproc) \ && make install \ && cd .. && rm -rf Python-${PYTHON_VERSION} \ From 3d47c749158e6a44570cbc09bd8964ac6e7a1047 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 16:16:46 +0900 Subject: [PATCH 08/11] chore: fix build --- .../cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile index cecd65a2cd35a..665f2f398468e 100644 --- a/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile +++ b/rust/cubestore/cross/aarch64-unknown-linux-gnu-python.Dockerfile @@ -4,7 +4,7 @@ ARG PYTHON_VERSION ARG PYTHON_RELEASE # python is required for cross compiling python :D -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_VERSION_RELEASE} \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${PYTHON_RELEASE} \ && rm -rf /var/lib/apt/lists/*; RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz \ From 4ba63a35059f167fb771a44c9564e8d13c3542ef Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 16:25:00 +0900 Subject: [PATCH 09/11] chore: fix build --- rust/cubestore/cross/docker-bake.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/cubestore/cross/docker-bake.hcl b/rust/cubestore/cross/docker-bake.hcl index 6a74f40194d21..14b43963aea2c 100644 --- a/rust/cubestore/cross/docker-bake.hcl +++ b/rust/cubestore/cross/docker-bake.hcl @@ -59,7 +59,7 @@ target "x86_64-unknown-linux-musl-python" { args = { LLVM_VERSION = LLVM_VERSION } - tags = ["cubejs/cross:x86_64-unknown-linux-musl-${CROSS_VERSION}"] + tags = ["cubejs/rust-cross:x86_64-unknown-linux-musl-${CROSS_VERSION}"] platforms = ["linux/amd64"] } @@ -69,7 +69,7 @@ target "x86_64-unknown-linux-gnu" { args = { LLVM_VERSION = LLVM_VERSION } - tags = ["cubejs/cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}"] + tags = ["cubejs/rust-cross:x86_64-unknown-linux-gnu-${CROSS_VERSION}"] platforms = ["linux/amd64"] } From df474a3c003aba1dc85bdacdfb09f1fbd84322a1 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 16:34:00 +0900 Subject: [PATCH 10/11] chore: max-parallelism = 1 --- .github/buildkitd.toml | 2 ++ .github/workflows/cross-images.yml | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 .github/buildkitd.toml diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml new file mode 100644 index 0000000000000..f6dc45486dc1d --- /dev/null +++ b/.github/buildkitd.toml @@ -0,0 +1,2 @@ +[worker.oci] + max-parallelism = 1 diff --git a/.github/workflows/cross-images.yml b/.github/workflows/cross-images.yml index 0157a44785818..9dad886645bc6 100644 --- a/.github/workflows/cross-images.yml +++ b/.github/workflows/cross-images.yml @@ -33,6 +33,10 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + config: .github/buildkitd.toml - name: Build and push uses: docker/bake-action@v5 with: From 2bd390c906c40560e232ef882a169bff299931c0 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 15 Jul 2024 16:57:00 +0900 Subject: [PATCH 11/11] chore: fix --- .github/workflows/cross-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cross-images.yml b/.github/workflows/cross-images.yml index 9dad886645bc6..f6fae67b559b3 100644 --- a/.github/workflows/cross-images.yml +++ b/.github/workflows/cross-images.yml @@ -36,7 +36,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: - config: .github/buildkitd.toml + buildkitd-config: .github/buildkitd.toml - name: Build and push uses: docker/bake-action@v5 with: