From 1f5e9c2cb3f779b993d6ae3491d807aa6b8eb72c Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 17:42:54 +0200 Subject: [PATCH 01/37] Fix Checkout action --- .github/workflows/analysis_workflow.yml | 9 +++++++++ .github/workflows/build_steps.yml | 18 ++++++++++++++++++ .github/workflows/build_with_conda.yml | 16 ++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/.github/workflows/analysis_workflow.yml b/.github/workflows/analysis_workflow.yml index 95e1491755..6fa597e3b6 100644 --- a/.github/workflows/analysis_workflow.yml +++ b/.github/workflows/analysis_workflow.yml @@ -124,6 +124,15 @@ jobs: ARCTIC_CMAKE_PRESET: linux-debug ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 + - uses: actions/checkout@v3.3.0 with: submodules: recursive diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 333ea47d63..e1b9c26fc1 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -69,6 +69,15 @@ jobs: defaults: run: {shell: bash} steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 + - name: Checkout uses: actions/checkout@v3.3.0 with: @@ -264,6 +273,15 @@ jobs: distinguishing_name: ${{matrix.os}}-${{needs.compile.outputs.python_impl_name}}-${{matrix.type}}${{matrix.python_deps_id}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 + - name: Checkout uses: actions/checkout@v3.3.0 diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index c4ef35d16e..dcbe06d511 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -31,6 +31,14 @@ jobs: mongodb: image: mongo:4.4 steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 - uses: actions/checkout@v3.3.0 # DONT use recursive submodules checkout to simulate conda feedstock build # with: @@ -116,6 +124,14 @@ jobs: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 - uses: actions/checkout@v3.3.0 # DONT use recursive submodules checkout to simulate conda feedstock build # with: From 5816c2c4d937f3dc3b6d0734cab8a0215bf4278e Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:13:51 +0200 Subject: [PATCH 02/37] Fix Node --- .github/workflows/build_steps.yml | 15 +++++---------- .github/workflows/build_with_conda.yml | 8 -------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index e1b9c26fc1..636e669cb1 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -70,8 +70,12 @@ jobs: run: {shell: bash} steps: - name: Setup Node 16 + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: | - apt update && apt -y install wget + # Replace default repo as centos 7 has reached EOL + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo + sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo + yum update && yum -y install wget touch "$HOME/.bashrc" wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash export NVM_DIR="$HOME/.nvm" @@ -273,15 +277,6 @@ jobs: distinguishing_name: ${{matrix.os}}-${{needs.compile.outputs.python_impl_name}}-${{matrix.type}}${{matrix.python_deps_id}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - name: Setup Node 16 - run: | - apt update && apt -y install wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 - - name: Checkout uses: actions/checkout@v3.3.0 diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index dcbe06d511..954b3c28da 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -124,14 +124,6 @@ jobs: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - name: Setup Node 16 - run: | - apt update && apt -y install wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 - uses: actions/checkout@v3.3.0 # DONT use recursive submodules checkout to simulate conda feedstock build # with: From 7d943b992e8baa2d87334258a91d19de40c22b89 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:16:11 +0200 Subject: [PATCH 03/37] Fix Node --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 636e669cb1..53ec6d02b9 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -75,7 +75,7 @@ jobs: # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - yum update && yum -y install wget + yum update -y && yum install -y wget touch "$HOME/.bashrc" wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash export NVM_DIR="$HOME/.nvm" From edcce7de3abdd2a591870ebc41730848e17535a6 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:31:52 +0200 Subject: [PATCH 04/37] Test with default manylinus iamge --- .github/workflows/build.yml | 2 +- .github/workflows/build_steps.yml | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..f8fbe9323a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: ${{needs.cibw_docker_image.outputs.tag}} + image: quay.io/pypa/manylinux2014_x86_64 volumes: - /:/mnt windows_matrix: diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 53ec6d02b9..b33fa3f8fe 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -69,18 +69,31 @@ jobs: defaults: run: {shell: bash} steps: - - name: Setup Node 16 - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + - name: Prepare Image run: | # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - yum update -y && yum install -y wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 + # TODO Should have a custom Docker image rather than this step + # tools + yum update -y + yum install -y zip flex bison krb5-devel cyrus-sasl-devel openssl-devel \ + devtoolset-11-gdb devtoolset-10-libatomic-devel unzip tar epel-release jq wget libcurl-devel python3 \ + rh-python38-python-devel rh-python38-python-pip perl-IPC-Cmd + + yum install -y mono-complete + + yum clean all + + # sccache + sccache_ver=`curl -sL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/mozilla/sccache/releases/latest | jq -r ".tag_name"` + + wget -nv https://github.com/mozilla/sccache/releases/download/$sccache_ver/sccache-$sccache_ver-x86_64-unknown-linux-musl.tar.gz \ + -O /tmp/sccache.tar.gz + tar -xvf /tmp/sccache.tar.gz -C /tmp/ + cp /tmp/sccache-*/sccache /usr/local/bin/ + rm /tmp/sccache.tar.gz - name: Checkout uses: actions/checkout@v3.3.0 From da4790fefed022cca8e8454e7866691ea7cdae46 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:46:51 +0200 Subject: [PATCH 05/37] Fix node --- .github/workflows/build.yml | 2 +- .github/workflows/build_steps.yml | 31 ++++++++++--------------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8fbe9323a..e7097864ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: quay.io/pypa/manylinux2014_x86_64 + image: ${{needs.cibw_docker_image.outputs.tag}} volumes: - /:/mnt windows_matrix: diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index b33fa3f8fe..05e8cfc734 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -66,34 +66,23 @@ jobs: python_impl_name: ${{inputs.python3 > 0 && format('cp3{0}', inputs.python3) || 'default'}} CIBW_BUILD: ${{format('cp3{0}-{1}', inputs.python3, matrix.cibw_build_suffix)}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 defaults: run: {shell: bash} steps: - - name: Prepare Image + - name: Setup Node 16 + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: | # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - # TODO Should have a custom Docker image rather than this step - # tools - yum update -y - yum install -y zip flex bison krb5-devel cyrus-sasl-devel openssl-devel \ - devtoolset-11-gdb devtoolset-10-libatomic-devel unzip tar epel-release jq wget libcurl-devel python3 \ - rh-python38-python-devel rh-python38-python-pip perl-IPC-Cmd - - yum install -y mono-complete - - yum clean all - - # sccache - sccache_ver=`curl -sL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/mozilla/sccache/releases/latest | jq -r ".tag_name"` - - wget -nv https://github.com/mozilla/sccache/releases/download/$sccache_ver/sccache-$sccache_ver-x86_64-unknown-linux-musl.tar.gz \ - -O /tmp/sccache.tar.gz - tar -xvf /tmp/sccache.tar.gz -C /tmp/ - cp /tmp/sccache-*/sccache /usr/local/bin/ - rm /tmp/sccache.tar.gz + yum update && yum -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 - name: Checkout uses: actions/checkout@v3.3.0 From 7d257e0f46a1d3232f81d1379abc5b7c3b13782a Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:50:40 +0200 Subject: [PATCH 06/37] Fix node --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 05e8cfc734..4a91185d26 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -77,7 +77,7 @@ jobs: # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - yum update && yum -y install wget + yum update -y && yum install -y wget touch "$HOME/.bashrc" wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash export NVM_DIR="$HOME/.nvm" From 3116589339d0792f09d9717f2758e060d9424b7a Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 18:58:33 +0200 Subject: [PATCH 07/37] Try to remove node20 --- .github/workflows/build_steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 4a91185d26..4be25c7b73 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -74,6 +74,7 @@ jobs: - name: Setup Node 16 if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: | + rm -rf /__e/node20 # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo From 6d1d8119d4e1c64d2b7c1dceaf37d26ac86f5052 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 19:53:06 +0200 Subject: [PATCH 08/37] Test node --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 4be25c7b73..4b098a87eb 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -74,7 +74,6 @@ jobs: - name: Setup Node 16 if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: | - rm -rf /__e/node20 # Replace default repo as centos 7 has reached EOL sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo @@ -84,6 +83,7 @@ jobs: export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" nvm install 16 + which node - name: Checkout uses: actions/checkout@v3.3.0 From ef2110a562519888654ef53e1adab5d542c17e01 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 19:55:35 +0200 Subject: [PATCH 09/37] Try to remove node20 --- .github/workflows/build_steps.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 4b098a87eb..cb7a445d52 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -85,10 +85,10 @@ jobs: nvm install 16 which node - - name: Checkout - uses: actions/checkout@v3.3.0 - with: - submodules: recursive # Just in case a dep has its own third-party deps + # - name: Checkout + # uses: actions/checkout@v3.3.0 + # with: + # submodules: recursive # Just in case a dep has its own third-party deps - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.3 From e3a5114b6464fe125ed27c4db472051d382e0fc8 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 20:00:15 +0200 Subject: [PATCH 10/37] Try to remove node20 --- .github/workflows/build_steps.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index cb7a445d52..2b4b646a18 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -84,11 +84,13 @@ jobs: [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" nvm install 16 which node + # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH + echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH - # - name: Checkout - # uses: actions/checkout@v3.3.0 - # with: - # submodules: recursive # Just in case a dep has its own third-party deps + - name: Checkout + uses: actions/checkout@v3.3.0 + with: + submodules: recursive # Just in case a dep has its own third-party deps - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.3 From e414588d688d5f9c9fc910c80f43e638d7ba55a5 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 20:03:45 +0200 Subject: [PATCH 11/37] Try to remove node20 --- .github/workflows/build_steps.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 2b4b646a18..8c1408e6f1 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -71,21 +71,11 @@ jobs: defaults: run: {shell: bash} steps: - - name: Setup Node 16 - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - run: | - # Replace default repo as centos 7 has reached EOL - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo - sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - yum update -y && yum install -y wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 - which node - # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH - echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: + - '16' - name: Checkout uses: actions/checkout@v3.3.0 From bb06097c39f06f05760ae6a59d7cc30f06beb6c7 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 20:05:19 +0200 Subject: [PATCH 12/37] Try to remove node20 --- .github/workflows/build_steps.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 8c1408e6f1..eaf1c0f936 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -71,11 +71,9 @@ jobs: defaults: run: {shell: bash} steps: - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version: - - '16' + - uses: actions/setup-node@v4 + with: + node-version: 16 - name: Checkout uses: actions/checkout@v3.3.0 From b91cb966073bb5ebd6de782ecbdd015390d349ef Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Tue, 26 Nov 2024 20:05:53 +0200 Subject: [PATCH 13/37] Try to remove node20 --- .github/workflows/build_steps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index eaf1c0f936..1ad8afd827 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -72,8 +72,8 @@ jobs: run: {shell: bash} steps: - uses: actions/setup-node@v4 - with: - node-version: 16 + with: + node-version: 16 - name: Checkout uses: actions/checkout@v3.3.0 From 96ce180bf750745232b6d5eb1ba326803f0e75ec Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:16:46 +0200 Subject: [PATCH 14/37] Test manuall cehckout: --- .github/workflows/build_steps.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 1ad8afd827..7357f4b4ab 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -71,19 +71,17 @@ jobs: defaults: run: {shell: bash} steps: - - uses: actions/setup-node@v4 - with: - node-version: 16 - - name: Checkout + if: matrix.os != 'linux' uses: actions/checkout@v3.3.0 with: submodules: recursive # Just in case a dep has its own third-party deps - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.3 - with: - version: "v0.4.0" + - name: Manual checkout + if: matrix.os == 'linux' + run: | + git clone https://github.com/man-group/ArcticDB.git . --branch $GITHUB_REF --depth 1 + git submodule update --init --recursive - name: Windows Pagefile if: matrix.os == 'windows' From 590053f34e37266eb8ce6305111ab06827e8ed3e Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:19:06 +0200 Subject: [PATCH 15/37] Test manuall cehckout: --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 7357f4b4ab..bb3958c96a 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -80,7 +80,7 @@ jobs: - name: Manual checkout if: matrix.os == 'linux' run: | - git clone https://github.com/man-group/ArcticDB.git . --branch $GITHUB_REF --depth 1 + git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 git submodule update --init --recursive - name: Windows Pagefile From 6762cbf4afabcad4d8bd17776c397d2f85a7964f Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:21:43 +0200 Subject: [PATCH 16/37] Fix git folder --- .github/workflows/build_steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index bb3958c96a..19a2db4607 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -80,6 +80,7 @@ jobs: - name: Manual checkout if: matrix.os == 'linux' run: | + git config --global --add safe.directory /__w/ArcticDB/ArcticDB git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 git submodule update --init --recursive From 1998b5c9a30a028d8dab4de293de37bc107c5a0d Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:31:58 +0200 Subject: [PATCH 17/37] try to chckout in github_workspace --- .github/workflows/build_steps.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 19a2db4607..f193c14c6f 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -80,7 +80,8 @@ jobs: - name: Manual checkout if: matrix.os == 'linux' run: | - git config --global --add safe.directory /__w/ArcticDB/ArcticDB + git config --global --add safe.directory $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 git submodule update --init --recursive From 40a787095b4dd56c0a9e12c578693daec24713b0 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:42:20 +0200 Subject: [PATCH 18/37] Wrap the checkout action --- .github/actions/checkout_wrapper/action.yml | 19 +++++++++++++++++++ .github/workflows/build_steps.yml | 15 ++------------- build_tooling/prep_cpp_build.sh | 6 +++--- 3 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 .github/actions/checkout_wrapper/action.yml diff --git a/.github/actions/checkout_wrapper/action.yml b/.github/actions/checkout_wrapper/action.yml new file mode 100644 index 0000000000..5b23e92e47 --- /dev/null +++ b/.github/actions/checkout_wrapper/action.yml @@ -0,0 +1,19 @@ +name: 'Wrapper for actions/checkout' +description: 'Wraps the actions/checkout action to set up the environment for ArcticDB CI tests' + +runs: + using: "composite" + steps: + - name: Checkout + if: matrix.os != 'linux' + uses: actions/checkout@v3.3.0 + with: + submodules: recursive # Just in case a dep has its own third-party deps + + - name: Manual checkout + if: matrix.os == 'linux' + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE + git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 + git submodule update --init --recursive \ No newline at end of file diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index f193c14c6f..d619bc08f8 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -72,18 +72,7 @@ jobs: run: {shell: bash} steps: - name: Checkout - if: matrix.os != 'linux' - uses: actions/checkout@v3.3.0 - with: - submodules: recursive # Just in case a dep has its own third-party deps - - - name: Manual checkout - if: matrix.os == 'linux' - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - cd $GITHUB_WORKSPACE - git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 - git submodule update --init --recursive + uses: ./.github/actions/checkout_wrapper - name: Windows Pagefile if: matrix.os == 'windows' @@ -271,7 +260,7 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - name: Checkout - uses: actions/checkout@v3.3.0 + uses: ./.github/actions/checkout_wrapper - name: Get wheel artifact uses: actions/download-artifact@v3.0.2 diff --git a/build_tooling/prep_cpp_build.sh b/build_tooling/prep_cpp_build.sh index dafbefcaa3..b0c208ecca 100755 --- a/build_tooling/prep_cpp_build.sh +++ b/build_tooling/prep_cpp_build.sh @@ -3,11 +3,11 @@ pushd $(realpath $(dirname $BASH_SOURCE))/../cpp/vcpkg if [[ -e "$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch --unshallow file://$VCPKG_INSTALLATION_ROOT + git fetch file://$VCPKG_INSTALLATION_ROOT elif [[ -n "$VCPKG_INSTALLATION_ROOT" && -e "/host$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch --unshallow file:///host$VCPKG_INSTALLATION_ROOT + git fetch file:///host$VCPKG_INSTALLATION_ROOT else - git fetch --unshallow origin master + git fetch origin master fi cd .. From 9ab74b60f29fdb5c01c7ab2ff114b32301b801cb Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 11:46:32 +0200 Subject: [PATCH 19/37] Remove custom action --- .github/actions/checkout_wrapper/action.yml | 19 --------------- .github/workflows/build_steps.yml | 26 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 21 deletions(-) delete mode 100644 .github/actions/checkout_wrapper/action.yml diff --git a/.github/actions/checkout_wrapper/action.yml b/.github/actions/checkout_wrapper/action.yml deleted file mode 100644 index 5b23e92e47..0000000000 --- a/.github/actions/checkout_wrapper/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: 'Wrapper for actions/checkout' -description: 'Wraps the actions/checkout action to set up the environment for ArcticDB CI tests' - -runs: - using: "composite" - steps: - - name: Checkout - if: matrix.os != 'linux' - uses: actions/checkout@v3.3.0 - with: - submodules: recursive # Just in case a dep has its own third-party deps - - - name: Manual checkout - if: matrix.os == 'linux' - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - cd $GITHUB_WORKSPACE - git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 - git submodule update --init --recursive \ No newline at end of file diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index d619bc08f8..b18ab75ba9 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -72,7 +72,18 @@ jobs: run: {shell: bash} steps: - name: Checkout - uses: ./.github/actions/checkout_wrapper + if: matrix.os != 'linux' + uses: actions/checkout@v3.3.0 + with: + submodules: recursive # Just in case a dep has its own third-party deps + + - name: Manual checkout + if: matrix.os == 'linux' + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE + git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 + git submodule update --init --recursive - name: Windows Pagefile if: matrix.os == 'windows' @@ -260,7 +271,18 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - name: Checkout - uses: ./.github/actions/checkout_wrapper + if: matrix.os != 'linux' + uses: actions/checkout@v3.3.0 + with: + submodules: recursive # Just in case a dep has its own third-party deps + + - name: Manual checkout + if: matrix.os == 'linux' + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE + git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 + git submodule update --init --recursive - name: Get wheel artifact uses: actions/download-artifact@v3.0.2 From 9475d06c760887926c9fa03cc8b6dad3d3234aea Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 12:32:33 +0200 Subject: [PATCH 20/37] Try to remove the node20 --- .github/workflows/build_steps.yml | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index b18ab75ba9..db6ada957e 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -66,24 +66,27 @@ jobs: python_impl_name: ${{inputs.python3 > 0 && format('cp3{0}', inputs.python3) || 'default'}} CIBW_BUILD: ${{format('cp3{0}-{1}', inputs.python3, matrix.cibw_build_suffix)}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 defaults: run: {shell: bash} steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 + sudo rm -rf /__e/node20 - name: Checkout - if: matrix.os != 'linux' uses: actions/checkout@v3.3.0 with: submodules: recursive # Just in case a dep has its own third-party deps - - name: Manual checkout - if: matrix.os == 'linux' - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - cd $GITHUB_WORKSPACE - git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 - git submodule update --init --recursive + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.3 + with: + version: "v0.4.0" - name: Windows Pagefile if: matrix.os == 'windows' @@ -270,19 +273,17 @@ jobs: distinguishing_name: ${{matrix.os}}-${{needs.compile.outputs.python_impl_name}}-${{matrix.type}}${{matrix.python_deps_id}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: + - name: Setup Node 16 + run: | + apt update && apt -y install wget + touch "$HOME/.bashrc" + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nvm install 16 + - name: Checkout - if: matrix.os != 'linux' uses: actions/checkout@v3.3.0 - with: - submodules: recursive # Just in case a dep has its own third-party deps - - - name: Manual checkout - if: matrix.os == 'linux' - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - cd $GITHUB_WORKSPACE - git clone https://github.com/man-group/ArcticDB.git . --branch ${{github.head_ref || github.ref_name}} --depth 1 - git submodule update --init --recursive - name: Get wheel artifact uses: actions/download-artifact@v3.0.2 From 29faa602d8ed60ddea02d319188ef41a9a6d5f5d Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 12:36:01 +0200 Subject: [PATCH 21/37] Fix node --- .github/workflows/build_steps.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index db6ada957e..aae0388718 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -70,14 +70,22 @@ jobs: run: {shell: bash} steps: - name: Setup Node 16 + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: | - apt update && apt -y install wget + # Replace default repo as centos 7 has reached EOL + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo + sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo + yum update -y && yum install -y wget touch "$HOME/.bashrc" wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" nvm install 16 + which node + # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH + echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH sudo rm -rf /__e/node20 + - name: Checkout uses: actions/checkout@v3.3.0 with: From 04976d9010bb3634ea2c7f4d617390cc3a49dcb4 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 12:41:25 +0200 Subject: [PATCH 22/37] Test no volumes --- .github/workflows/build.yml | 2 -- build_tooling/prep_cpp_build.sh | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..70e85c3706 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,8 +53,6 @@ jobs: image: "mongo:4.4" container: image: ${{needs.cibw_docker_image.outputs.tag}} - volumes: - - /:/mnt windows_matrix: - os: windows distro: windows-latest diff --git a/build_tooling/prep_cpp_build.sh b/build_tooling/prep_cpp_build.sh index b0c208ecca..dafbefcaa3 100755 --- a/build_tooling/prep_cpp_build.sh +++ b/build_tooling/prep_cpp_build.sh @@ -3,11 +3,11 @@ pushd $(realpath $(dirname $BASH_SOURCE))/../cpp/vcpkg if [[ -e "$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch file://$VCPKG_INSTALLATION_ROOT + git fetch --unshallow file://$VCPKG_INSTALLATION_ROOT elif [[ -n "$VCPKG_INSTALLATION_ROOT" && -e "/host$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch file:///host$VCPKG_INSTALLATION_ROOT + git fetch --unshallow file:///host$VCPKG_INSTALLATION_ROOT else - git fetch origin master + git fetch --unshallow origin master fi cd .. From 37485802f7a369fe5c166210d986364d99860468 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 12:43:52 +0200 Subject: [PATCH 23/37] Remove sudo --- .github/workflows/build_steps.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index aae0388718..c034beb2fa 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -84,7 +84,6 @@ jobs: which node # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH - sudo rm -rf /__e/node20 - name: Checkout uses: actions/checkout@v3.3.0 From 84b176b44af4527119c3175a6bf396983975384f Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 12:46:32 +0200 Subject: [PATCH 24/37] Remove sudo --- .github/workflows/build_steps.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index c034beb2fa..e418620e96 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -84,6 +84,8 @@ jobs: which node # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH + # try to print the path + echo $GITHUB_PATH - name: Checkout uses: actions/checkout@v3.3.0 From 2efbc5bf8fd8dd7a34048a9d19406c5b2928c84e Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:10:12 +0200 Subject: [PATCH 25/37] test different node --- .github/workflows/build.yml | 4 ++++ .github/workflows/build_steps.yml | 34 +++++++++++++++---------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70e85c3706..304cb7649c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,10 @@ jobs: image: "mongo:4.4" container: image: ${{needs.cibw_docker_image.outputs.tag}} + volumes: + - /:/mnt + - /node20glib217:/node20glib217:rw,rshared + - /node20glib217:/__e/node20:ro,rshared windows_matrix: - os: windows distro: windows-latest diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index e418620e96..e7c428925e 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -69,23 +69,23 @@ jobs: defaults: run: {shell: bash} steps: - - name: Setup Node 16 - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - run: | - # Replace default repo as centos 7 has reached EOL - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo - sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - yum update -y && yum install -y wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 - which node - # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH - echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH - # try to print the path - echo $GITHUB_PATH + # - name: Setup Node 16 + # if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + # run: | + # # Replace default repo as centos 7 has reached EOL + # sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo + # sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo + # yum update -y && yum install -y wget + # touch "$HOME/.bashrc" + # wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + # export NVM_DIR="$HOME/.nvm" + # [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + # nvm install 16 + # which node + # # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH + # echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH + # # try to print the path + # echo $GITHUB_PATH - name: Checkout uses: actions/checkout@v3.3.0 From 9490385463f13e79cfde3040fd1e36381850bc44 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:13:12 +0200 Subject: [PATCH 26/37] Test other checkout --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index e7c428925e..b1f46d48b6 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -88,7 +88,7 @@ jobs: # echo $GITHUB_PATH - name: Checkout - uses: actions/checkout@v3.3.0 + uses: taiki-e/checkout-action@v1 with: submodules: recursive # Just in case a dep has its own third-party deps From 37880d5819cf6eb9d6914218019abe420e44b50b Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:27:52 +0200 Subject: [PATCH 27/37] Fix submodules --- .github/workflows/build.yml | 2 -- .github/workflows/build_steps.yml | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 304cb7649c..e7097864ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,8 +55,6 @@ jobs: image: ${{needs.cibw_docker_image.outputs.tag}} volumes: - /:/mnt - - /node20glib217:/node20glib217:rw,rshared - - /node20glib217:/__e/node20:ro,rshared windows_matrix: - os: windows distro: windows-latest diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index b1f46d48b6..c69161ceab 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -86,13 +86,23 @@ jobs: # echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH # # try to print the path # echo $GITHUB_PATH - - name: Checkout - uses: taiki-e/checkout-action@v1 + if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') + uses: actions/checkout@v3.3.0 with: submodules: recursive # Just in case a dep has its own third-party deps + - name: Checkout + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + uses: taiki-e/checkout-action@v1 + + + - name: Checkout submodules + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + run: git submodule update --init --recursive + - name: Configure sccache + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' uses: mozilla-actions/sccache-action@v0.0.3 with: version: "v0.4.0" From dd88378092b71915be27c319e88b43d573b4ef59 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:30:37 +0200 Subject: [PATCH 28/37] Fix typo --- .github/workflows/build_steps.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index c69161ceab..f077d73044 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -92,7 +92,13 @@ jobs: with: submodules: recursive # Just in case a dep has its own third-party deps - - name: Checkout + - name: Configure sccache + if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') + uses: mozilla-actions/sccache-action@v0.0.3 + with: + version: "v0.4.0" + + - name: Checkout without Node if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' uses: taiki-e/checkout-action@v1 @@ -101,12 +107,6 @@ jobs: if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' run: git submodule update --init --recursive - - name: Configure sccache - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - uses: mozilla-actions/sccache-action@v0.0.3 - with: - version: "v0.4.0" - - name: Windows Pagefile if: matrix.os == 'windows' uses: al-cheb/configure-pagefile-action@v1.3 From 679ff280577f22575db231cd9394da393f0503ad Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:38:40 +0200 Subject: [PATCH 29/37] unshallow --- .github/workflows/build_steps.yml | 1 + build_tooling/prep_cpp_build.sh | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index f077d73044..ea79447f69 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -66,6 +66,7 @@ jobs: python_impl_name: ${{inputs.python3 > 0 && format('cp3{0}', inputs.python3) || 'default'}} CIBW_BUILD: ${{format('cp3{0}-{1}', inputs.python3, matrix.cibw_build_suffix)}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + USE_UNSHALLOW: ${{(matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && '0' || '1'}} defaults: run: {shell: bash} steps: diff --git a/build_tooling/prep_cpp_build.sh b/build_tooling/prep_cpp_build.sh index dafbefcaa3..982d4cc43e 100755 --- a/build_tooling/prep_cpp_build.sh +++ b/build_tooling/prep_cpp_build.sh @@ -2,12 +2,18 @@ pushd $(realpath $(dirname $BASH_SOURCE))/../cpp/vcpkg +if [[ "$USE_UNSHALLOW" == "1" ]]; then + UNSHALLOW_OPTION="--unshallow" +else + UNSHALLOW_OPTION="" +fi + if [[ -e "$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch --unshallow file://$VCPKG_INSTALLATION_ROOT + git fetch $UNSHALLOW_OPTION file://$VCPKG_INSTALLATION_ROOT elif [[ -n "$VCPKG_INSTALLATION_ROOT" && -e "/host$VCPKG_INSTALLATION_ROOT" ]] ; then - git fetch --unshallow file:///host$VCPKG_INSTALLATION_ROOT + git fetch $UNSHALLOW_OPTION file:///host$VCPKG_INSTALLATION_ROOT else - git fetch --unshallow origin master + git fetch $UNSHALLOW_OPTION origin master fi cd .. From 2f1ceb750ae5a9a309459c56aaa231e677f6bd0c Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:54:26 +0200 Subject: [PATCH 30/37] test --- .github/workflows/build_steps.yml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index ea79447f69..2e99626e13 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -66,27 +66,10 @@ jobs: python_impl_name: ${{inputs.python3 > 0 && format('cp3{0}', inputs.python3) || 'default'}} CIBW_BUILD: ${{format('cp3{0}-{1}', inputs.python3, matrix.cibw_build_suffix)}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - USE_UNSHALLOW: ${{(matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && '0' || '1'}} + USE_UNSHALLOW: ${{(matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && '1' || '0'}} defaults: run: {shell: bash} steps: - # - name: Setup Node 16 - # if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - # run: | - # # Replace default repo as centos 7 has reached EOL - # sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g; s|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo - # sed -i '/mirrorlist.centos.org/d' /etc/yum.repos.d/*.repo - # yum update -y && yum install -y wget - # touch "$HOME/.bashrc" - # wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - # export NVM_DIR="$HOME/.nvm" - # [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - # nvm install 16 - # which node - # # Add /github/home/.nvm/versions/node/v16.20.2/bin to GITHUB_PATH - # echo -e "/github/home/.nvm/versions/node/v16.20.2/bin" | tee -a $GITHUB_PATH - # # try to print the path - # echo $GITHUB_PATH - name: Checkout if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') uses: actions/checkout@v3.3.0 @@ -152,7 +135,7 @@ jobs: run: . build_tooling/prep_cpp_build.sh # Also applies to Windows - name: CMake compile - if: inputs.job_type != 'build-python-wheels' + matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container uses: lukka/run-cmake@v10.6 with: @@ -161,6 +144,10 @@ jobs: configurePresetAdditionalArgs: "['-DVCPKG_INSTALL_OPTIONS=--clean-after-build']" buildPreset: ${{env.ARCTIC_CMAKE_PRESET}} + - name: CMake compile without Node + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET + - name: Compile C++ tests if: inputs.job_type == 'cpp-tests' run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install From 6db1a7e5be30d747056559c3ddf9291d6aa85cee Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:56:23 +0200 Subject: [PATCH 31/37] Fix --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 2e99626e13..f0f7425a4f 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -135,7 +135,7 @@ jobs: run: . build_tooling/prep_cpp_build.sh # Also applies to Windows - name: CMake compile - matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') + if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container uses: lukka/run-cmake@v10.6 with: From 6d2e23a267c00f848848a47fe5d7e21a7be99351 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 13:59:16 +0200 Subject: [PATCH 32/37] Fix unshallow --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index f0f7425a4f..d9fd63ba81 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -66,7 +66,7 @@ jobs: python_impl_name: ${{inputs.python3 > 0 && format('cp3{0}', inputs.python3) || 'default'}} CIBW_BUILD: ${{format('cp3{0}-{1}', inputs.python3, matrix.cibw_build_suffix)}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - USE_UNSHALLOW: ${{(matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && '1' || '0'}} + USE_UNSHALLOW: ${{(matrix.os == 'linux' && inputs.job_type != 'build-python-wheels') && '0' || '1'}} defaults: run: {shell: bash} steps: From db286f1e75129df712a4aa487e0c072f2a56d9a5 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 14:08:25 +0200 Subject: [PATCH 33/37] fix --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index d9fd63ba81..b06221e6d9 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -135,7 +135,7 @@ jobs: run: . build_tooling/prep_cpp_build.sh # Also applies to Windows - name: CMake compile - if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') + if: matrix.os == 'windows' && inputs.job_type != 'build-python-wheels' # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container uses: lukka/run-cmake@v10.6 with: From 0fbc8e2350432e7a0e986c2b70fe4ed9e09bd020 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 14:22:06 +0200 Subject: [PATCH 34/37] Fix --- .github/workflows/build_steps.yml | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index b06221e6d9..45a741b4d3 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -150,7 +150,9 @@ jobs: - name: Compile C++ tests if: inputs.job_type == 'cpp-tests' - run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install + run: | + python setup.py develop + cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install - name: C++ Rapidcheck if: inputs.job_type == 'cpp-tests' diff --git a/pyproject.toml b/pyproject.toml index 4a00f801c4..7f0deb6e28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ test-extras = "Testing" before-all = """ [[ -e /etc/arcticdb_deps_installed ]] || { yum update -y && yum install -y zip \ openssl-devel cyrus-sasl-devel devtoolset-10-libatomic-devel libcurl-devel ; } +export USE_UNSHALLOW=1 {project}/build_tooling/prep_cpp_build.sh {project}/build_tooling/vcpkg_caching.sh """ From 424811662b4ceb75e8b42af2a3323b347c94aa6a Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 14:40:05 +0200 Subject: [PATCH 35/37] Try to run tests --- .github/workflows/build_steps.yml | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 45a741b4d3..cbb4b96e78 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -146,13 +146,11 @@ jobs: - name: CMake compile without Node if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET + run: cd cpp; cmake --preset $ARCTIC_CMAKE_PRESET - name: Compile C++ tests if: inputs.job_type == 'cpp-tests' - run: | - python setup.py develop - cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install + run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install - name: C++ Rapidcheck if: inputs.job_type == 'cpp-tests' @@ -273,7 +271,7 @@ jobs: ${{fromJSON(inputs.matrix)}} name: ${{matrix.type}}${{matrix.python_deps_id}} runs-on: ${{matrix.distro}} - container: ${{matrix.os == 'linux' && needs.compile.outputs.manylinux_image || null}} + # container: ${{matrix.os == 'linux' && needs.compile.outputs.manylinux_image || null}} defaults: run: {shell: bash} services: ${{matrix.test_services}} @@ -282,15 +280,6 @@ jobs: distinguishing_name: ${{matrix.os}}-${{needs.compile.outputs.python_impl_name}}-${{matrix.type}}${{matrix.python_deps_id}} ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - name: Setup Node 16 - run: | - apt update && apt -y install wget - touch "$HOME/.bashrc" - wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 16 - - name: Checkout uses: actions/checkout@v3.3.0 @@ -300,12 +289,12 @@ jobs: name: wheel-${{needs.compile.outputs.cibw_build}} path: ${{runner.temp}} - - name: Select Python (Linux) - if: matrix.os == 'linux' - run: echo /opt/python/${{env.python_impl_name}}*/bin >> $GITHUB_PATH + # - name: Select Python (Linux) + # if: matrix.os == 'linux' + # run: echo /opt/python/${{env.python_impl_name}}*/bin >> $GITHUB_PATH - name: Select Python (Windows) - if: matrix.os == 'windows' + # if: matrix.os == 'windows' uses: actions/setup-python@v4.7.1 with: python-version: "3.${{inputs.python3}}" From d6dba000607a8171f3b6af054f788bcd6f9c75dc Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 14:47:24 +0200 Subject: [PATCH 36/37] Fix tests --- .github/workflows/build_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index cbb4b96e78..ad0dcf1e08 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -146,7 +146,7 @@ jobs: - name: CMake compile without Node if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - run: cd cpp; cmake --preset $ARCTIC_CMAKE_PRESET + run: cmake -DTEST=ON --preset $ARCTIC_CMAKE_PRESET cpp - name: Compile C++ tests if: inputs.job_type == 'cpp-tests' From 4f3b0814b24b185dca09bcc34d3278aa58d52512 Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Wed, 27 Nov 2024 15:57:31 +0200 Subject: [PATCH 37/37] Test --- .github/workflows/build.yml | 2 +- .github/workflows/build_steps.yml | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..210a5813eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: ${{needs.cibw_docker_image.outputs.tag}} + image: quay.io/pypa/manylinux_2_28_x86_64 volumes: - /:/mnt windows_matrix: diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index ad0dcf1e08..867ecdae1a 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -71,26 +71,15 @@ jobs: run: {shell: bash} steps: - name: Checkout - if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') uses: actions/checkout@v3.3.0 with: submodules: recursive # Just in case a dep has its own third-party deps - name: Configure sccache - if: matrix.os == 'windows' || (matrix.os == 'linux' && inputs.job_type == 'build-python-wheels') uses: mozilla-actions/sccache-action@v0.0.3 with: version: "v0.4.0" - - name: Checkout without Node - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - uses: taiki-e/checkout-action@v1 - - - - name: Checkout submodules - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - run: git submodule update --init --recursive - - name: Windows Pagefile if: matrix.os == 'windows' uses: al-cheb/configure-pagefile-action@v1.3 @@ -130,6 +119,16 @@ jobs: if: matrix.os == 'windows' run: rm $(which ccache) || true + - name: Install deps + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + run: | + yum update -y + yum install -y zip flex bison krb5-devel cyrus-sasl-devel openssl-devel \ + unzip tar epel-release jq wget libcurl-devel python3 \ + python3-devel python3-pip perl-IPC-Cmd + + yum install -y mono-complete + - name: Prepare C++ compilation env if: inputs.job_type != 'build-python-wheels' run: . build_tooling/prep_cpp_build.sh # Also applies to Windows