From 896d1e58fd8898a935175371e093373192a6e785 Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:34:11 -0700 Subject: [PATCH 1/9] test installing glibc 2.28 layer --- tests/Dockerfile.aws_glue_3 | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/Dockerfile.aws_glue_3 b/tests/Dockerfile.aws_glue_3 index cc0bf7aa..ff5cfa2d 100644 --- a/tests/Dockerfile.aws_glue_3 +++ b/tests/Dockerfile.aws_glue_3 @@ -1,4 +1,26 @@ FROM amazon/aws-glue-libs:glue_libs_3.0.0_image_01 +# Use the AWS Glue image as the base +FROM amazon/aws-glue-libs:glue_libs_3.0.0_image_01 + +# Update GLIBC +RUN yum update -y && yum install -y gcc make wget \ + && wget http://ftp.gnu.org/gnu/libc/glibc-2.28.tar.gz \ + && tar -xzvf glibc-2.28.tar.gz \ + && cd glibc-2.28 \ + && mkdir build \ + && cd build \ + && ../configure --prefix=/opt/glibc-2.28 \ + && make -j4 \ + && make install \ + && echo "/opt/glibc-2.28/lib" > /etc/ld.so.conf.d/glibc-2.28.conf \ + && ldconfig + +# Set the new GLIBC in the environment +ENV LD_LIBRARY_PATH=/opt/glibc-2.28/lib:$LD_LIBRARY_PATH + +# Verify GLIBC version +RUN ldd --version + RUN pip3 install moto~=4.1 datacompy~=0.8 pytest-datadir ecs_logging~=2.0 flask~=2.0 flask-cors~=3.0 ENTRYPOINT ["bash", "-l"] From 042e607f1f7482d5fcc0b429504cc7fed99734c5 Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:43:32 -0700 Subject: [PATCH 2/9] separate cmds --- tests/Dockerfile.aws_glue_3 | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/Dockerfile.aws_glue_3 b/tests/Dockerfile.aws_glue_3 index ff5cfa2d..fdfef6ac 100644 --- a/tests/Dockerfile.aws_glue_3 +++ b/tests/Dockerfile.aws_glue_3 @@ -1,20 +1,23 @@ FROM amazon/aws-glue-libs:glue_libs_3.0.0_image_01 -# Use the AWS Glue image as the base -FROM amazon/aws-glue-libs:glue_libs_3.0.0_image_01 +# Update and install required tools +RUN yum update -y && yum install -y gcc make wget + +# Download GLIBC source +RUN wget http://ftp.gnu.org/gnu/libc/glibc-2.28.tar.gz + +# Extract GLIBC source +RUN tar -xzvf glibc-2.28.tar.gz -# Update GLIBC -RUN yum update -y && yum install -y gcc make wget \ - && wget http://ftp.gnu.org/gnu/libc/glibc-2.28.tar.gz \ - && tar -xzvf glibc-2.28.tar.gz \ - && cd glibc-2.28 \ - && mkdir build \ - && cd build \ +# Configure and install GLIBC +WORKDIR glibc-2.28 +RUN mkdir build && cd build \ && ../configure --prefix=/opt/glibc-2.28 \ && make -j4 \ - && make install \ - && echo "/opt/glibc-2.28/lib" > /etc/ld.so.conf.d/glibc-2.28.conf \ - && ldconfig + && make install + +# Set the new GLIBC in the environment +RUN echo "/opt/glibc-2.28/lib" > /etc/ld.so.conf.d/glibc-2.28.conf && ldconfig # Set the new GLIBC in the environment ENV LD_LIBRARY_PATH=/opt/glibc-2.28/lib:$LD_LIBRARY_PATH From e0f8f4da4242c29512653e308f55fe1e9989ff9d Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Fri, 5 Jul 2024 19:48:25 -0700 Subject: [PATCH 3/9] use older node workaround --- .github/workflows/upload-and-deploy.yaml | 1 + tests/Dockerfile.aws_glue_3 | 25 ------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 55d5d61b..d5653069 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -14,6 +14,7 @@ env: PROD_INPUT_BUCKET: recover-input-data PROD_INTERMEDIATE_BUCKET: recover-intermediate-data INTEGRATION_TEST_NUM_EXPORTS: 28 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: diff --git a/tests/Dockerfile.aws_glue_3 b/tests/Dockerfile.aws_glue_3 index fdfef6ac..cc0bf7aa 100644 --- a/tests/Dockerfile.aws_glue_3 +++ b/tests/Dockerfile.aws_glue_3 @@ -1,29 +1,4 @@ FROM amazon/aws-glue-libs:glue_libs_3.0.0_image_01 -# Update and install required tools -RUN yum update -y && yum install -y gcc make wget - -# Download GLIBC source -RUN wget http://ftp.gnu.org/gnu/libc/glibc-2.28.tar.gz - -# Extract GLIBC source -RUN tar -xzvf glibc-2.28.tar.gz - -# Configure and install GLIBC -WORKDIR glibc-2.28 -RUN mkdir build && cd build \ - && ../configure --prefix=/opt/glibc-2.28 \ - && make -j4 \ - && make install - -# Set the new GLIBC in the environment -RUN echo "/opt/glibc-2.28/lib" > /etc/ld.so.conf.d/glibc-2.28.conf && ldconfig - -# Set the new GLIBC in the environment -ENV LD_LIBRARY_PATH=/opt/glibc-2.28/lib:$LD_LIBRARY_PATH - -# Verify GLIBC version -RUN ldd --version - RUN pip3 install moto~=4.1 datacompy~=0.8 pytest-datadir ecs_logging~=2.0 flask~=2.0 flask-cors~=3.0 ENTRYPOINT ["bash", "-l"] From 2f491d6189f5a3b91b6b9f80a3416071d7f51109 Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Sat, 6 Jul 2024 22:31:15 -0700 Subject: [PATCH 4/9] try to isolate aws configure credentials outside of docker build --- .github/workflows/upload-and-deploy.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index d5653069..95c2222c 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -14,7 +14,7 @@ env: PROD_INPUT_BUCKET: recover-input-data PROD_INTERMEDIATE_BUCKET: recover-intermediate-data INTEGRATION_TEST_NUM_EXPORTS: 28 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + #ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: @@ -118,6 +118,12 @@ jobs: dockerfile: tests/Dockerfile.aws_glue_4 environment: develop steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Assume AWS role uses: aws-actions/configure-aws-credentials@v2 with: @@ -137,10 +143,6 @@ jobs: echo "username-key=$usernameKey" >> $GITHUB_OUTPUT passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) echo "password-key=$passwordKey" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - name: Build and push to ECR id: docker-build-push From f8548cb49bc89ca2ce31089c4cb074f08c050e9c Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:37:45 -0700 Subject: [PATCH 5/9] try to separate out aws configure vs building docker --- .github/workflows/upload-and-deploy.yaml | 61 +++++++++++++++--------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 95c2222c..6f6c9567 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -100,37 +100,24 @@ jobs: --test-sts-permission read_write -v - - pytest-docker: - name: Build and push testing docker images to the pytest ECR repository. - needs: pre-commit + setup: + name: Runs aws credentials configuration before tests runs-on: ubuntu-latest - # These permissions are needed to interact with GitHub's OIDC Token endpoint. + needs: pre-commit + environment: develop permissions: id-token: write contents: read - strategy: - matrix: - include: - - tag_name: aws_glue_3 - dockerfile: tests/Dockerfile.aws_glue_3 - - tag_name: aws_glue_4 - dockerfile: tests/Dockerfile.aws_glue_4 - environment: develop steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Assume AWS role + id: assume-role uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} + role-to-assume: ${{ secrets.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" - # unmasking of the AWS account ID allows the acct id to pass through outputs - mask-aws-account-id: "no" - name: Login to Amazon ECR id: login-ecr @@ -143,20 +130,48 @@ jobs: echo "username-key=$usernameKey" >> $GITHUB_OUTPUT passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) echo "password-key=$passwordKey" >> $GITHUB_OUTPUT + outputs: + ecr-registry: ${{ steps.login-ecr.outputs.registry }} + ecr-username: ${{ steps.get-username.outputs.username-key }} + ecr-password: ${{ steps.get-password.outputs.password-key }} + + + pytest-docker: + name: Build and push testing docker images to the pytest ECR repository. + needs: [setup] + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + strategy: + matrix: + include: + - tag_name: aws_glue_3 + dockerfile: tests/Dockerfile.aws_glue_3 + - tag_name: aws_glue_4 + dockerfile: tests/Dockerfile.aws_glue_4 + environment: develop + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Build and push to ECR id: docker-build-push uses: docker/build-push-action@v4 with: push: true - tags: ${{ steps.login-ecr.outputs.registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }} + tags: ${{ needs.setup.outputs.ecr-registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }} file: ${{ matrix.dockerfile }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache outputs: - ecr-registry: ${{ steps.login-ecr.outputs.registry }} - ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} - ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} + ecr-registry: ${{ needs.setup.outputs.ecr-registry }} + ecr-username: ${{ needs.setup.outputs.ecr-username }} + ecr-password: ${{ needs.setup.outputs.ecr-password }} glue-unit-tests: From 5c2d736b09ba8f56d290d218e024e4fb656ff9f7 Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:58:54 -0700 Subject: [PATCH 6/9] add vars --- .github/workflows/upload-and-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 6f6c9567..b74dfb8a 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -116,7 +116,7 @@ jobs: id: assume-role uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: ${{ secrets.AWS_CREDENTIALS_IAM_ROLE }} + role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" - name: Login to Amazon ECR From b21bb41939d926ea11a397582a12f61377618796 Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:02:18 -0700 Subject: [PATCH 7/9] add back in unmasking --- .github/workflows/upload-and-deploy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index b74dfb8a..8ea3eaf8 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -118,6 +118,8 @@ jobs: with: role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" + # unmasking of the AWS account ID allows the acct id to pass through outputs + mask-aws-account-id: "no" - name: Login to Amazon ECR id: login-ecr From b8d874a7909bf182665de80990b2f02b0027940b Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:43:58 -0700 Subject: [PATCH 8/9] fix credentials access --- .github/workflows/upload-and-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 8ea3eaf8..503001c4 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -134,8 +134,8 @@ jobs: echo "password-key=$passwordKey" >> $GITHUB_OUTPUT outputs: ecr-registry: ${{ steps.login-ecr.outputs.registry }} - ecr-username: ${{ steps.get-username.outputs.username-key }} - ecr-password: ${{ steps.get-password.outputs.password-key }} + ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} + ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} pytest-docker: From 7311bad596be28b0c1ecc02003420b91ead6e11a Mon Sep 17 00:00:00 2001 From: rxu17 <26471741+rxu17@users.noreply.github.com> Date: Tue, 9 Jul 2024 13:16:26 -0700 Subject: [PATCH 9/9] revert --- .github/workflows/upload-and-deploy.yaml | 57 ++++++++---------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 503001c4..d5653069 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -14,7 +14,7 @@ env: PROD_INPUT_BUCKET: recover-input-data PROD_INTERMEDIATE_BUCKET: recover-intermediate-data INTEGRATION_TEST_NUM_EXPORTS: 28 - #ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: @@ -100,20 +100,25 @@ jobs: --test-sts-permission read_write -v - setup: - name: Runs aws credentials configuration before tests - runs-on: ubuntu-latest + + pytest-docker: + name: Build and push testing docker images to the pytest ECR repository. needs: pre-commit - environment: develop + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. permissions: id-token: write contents: read + strategy: + matrix: + include: + - tag_name: aws_glue_3 + dockerfile: tests/Dockerfile.aws_glue_3 + - tag_name: aws_glue_4 + dockerfile: tests/Dockerfile.aws_glue_4 + environment: develop steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Assume AWS role - id: assume-role uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} @@ -132,31 +137,7 @@ jobs: echo "username-key=$usernameKey" >> $GITHUB_OUTPUT passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) echo "password-key=$passwordKey" >> $GITHUB_OUTPUT - outputs: - ecr-registry: ${{ steps.login-ecr.outputs.registry }} - ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} - ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} - - - pytest-docker: - name: Build and push testing docker images to the pytest ECR repository. - needs: [setup] - runs-on: ubuntu-latest - # These permissions are needed to interact with GitHub's OIDC Token endpoint. - permissions: - id-token: write - contents: read - strategy: - matrix: - include: - - tag_name: aws_glue_3 - dockerfile: tests/Dockerfile.aws_glue_3 - - tag_name: aws_glue_4 - dockerfile: tests/Dockerfile.aws_glue_4 - environment: develop - steps: - - name: Checkout repository - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -166,14 +147,14 @@ jobs: uses: docker/build-push-action@v4 with: push: true - tags: ${{ needs.setup.outputs.ecr-registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }} + tags: ${{ steps.login-ecr.outputs.registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }} file: ${{ matrix.dockerfile }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache outputs: - ecr-registry: ${{ needs.setup.outputs.ecr-registry }} - ecr-username: ${{ needs.setup.outputs.ecr-username }} - ecr-password: ${{ needs.setup.outputs.ecr-password }} + ecr-registry: ${{ steps.login-ecr.outputs.registry }} + ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} + ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} glue-unit-tests: