From 02db6499ebfed7e673ee4374f208ed7bef97d2dd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 17:13:26 -0500 Subject: [PATCH 01/78] chore: migrate library generation IT to cloud build --- ...d-library-generation-integration-test.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml new file mode 100644 index 0000000000..8b37987a90 --- /dev/null +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 7200s # 2 hours +substitutions: + _TEST_IMAGE_NAME: "test-image:latest" +steps: +- name: gcr.io/cloud-builders/docker + args: [ + "build", + "-t", "${_TEST_IMAGE_NAME}", + "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", + "." + ] + id: library-generation-build + waitFor: ["-"] + env: + - 'DOCKER_BUILDKIT=1' + +- name: test-image:latest + waitFor: ["library-generation-build"] + id: library-generation-integration-test + +options: + logging: CLOUD_LOGGING_ONLY From 7d2ed278d8a98da7fa7c3d1f246481414cf57f9c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 17:23:41 -0500 Subject: [PATCH 02/78] run python test --- ...d-library-generation-integration-test.yaml | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 8b37987a90..3d044f0010 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -13,24 +13,16 @@ # limitations under the License. timeout: 7200s # 2 hours -substitutions: - _TEST_IMAGE_NAME: "test-image:latest" steps: -- name: gcr.io/cloud-builders/docker +- name: python + id: install-python-deps + entrypoint: python + waitFor: ["-"] args: [ - "build", - "-t", "${_TEST_IMAGE_NAME}", - "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", - "." + "-m", + "unittest", + "hermetic_build/library_generation/tests/integration_tests.py" ] - id: library-generation-build - waitFor: ["-"] - env: - - 'DOCKER_BUILDKIT=1' - -- name: test-image:latest - waitFor: ["library-generation-build"] - id: library-generation-integration-test options: logging: CLOUD_LOGGING_ONLY From f7c462f461d7ca762032b121b6879d69a344a9f3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 19:54:44 -0500 Subject: [PATCH 03/78] trigger ci From b96c7716613ba2ef647ec4665c02e51792d9c3f9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 19:57:24 -0500 Subject: [PATCH 04/78] install deps --- ...loudbuild-library-generation-integration-test.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3d044f0010..cba716aab5 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -16,8 +16,17 @@ timeout: 7200s # 2 hours steps: - name: python id: install-python-deps - entrypoint: python + entrypoint: pip waitFor: ["-"] + args: [ + "install", + "GitPython" + ] + +- name: python + id: run-integration-test + entrypoint: python + waitFor: ["install-python-deps"] args: [ "-m", "unittest", From ef67e37f08e042df5eb471115cd93a93401be73e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 20:05:03 -0500 Subject: [PATCH 05/78] install venv --- ...d-library-generation-integration-test.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index cba716aab5..3b99975819 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -14,10 +14,29 @@ timeout: 7200s # 2 hours steps: +- name: python + id: install-venv + entrypoint: python + waitFor: ["-"] + args: [ + "-m", + "venv", + ".venv" + ] + +- name: python + id: init-venv + entrypoint: bash + waitFor: ["install-venv"] + args: [ + "source", + ".venv/bin/activate" + ] + - name: python id: install-python-deps entrypoint: pip - waitFor: ["-"] + waitFor: ["init-venv"] args: [ "install", "GitPython" From 2a918b07a5686f22c590cac7ab46c1fbb6900e33 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 20:20:28 -0500 Subject: [PATCH 06/78] build image --- ...d-library-generation-integration-test.yaml | 44 +++++++------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3b99975819..2eee9ddb91 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -13,44 +13,32 @@ # limitations under the License. timeout: 7200s # 2 hours +substitutions: + _TEST_IMAGE_ID: "test-image:latest" steps: -- name: python - id: install-venv - entrypoint: python - waitFor: ["-"] - args: [ - "-m", - "venv", - ".venv" - ] - -- name: python - id: init-venv - entrypoint: bash - waitFor: ["install-venv"] +- name: gcr.io/cloud-builders/docker args: [ - "source", - ".venv/bin/activate" - ] - -- name: python - id: install-python-deps - entrypoint: pip - waitFor: ["init-venv"] - args: [ - "install", - "GitPython" + "build", + "-t", + "${_TEST_IMAGE_ID}", + "--file", + ".cloudbuild/library_generation/library_generation.Dockerfile", + "." ] + env: + - "DOCKER_BUILDKIT=1" + id: library-generation-build + waitFor: ["-"] -- name: python - id: run-integration-test +- name: ${_TEST_IMAGE_ID} entrypoint: python - waitFor: ["install-python-deps"] args: [ "-m", "unittest", "hermetic_build/library_generation/tests/integration_tests.py" ] + waitFor: [ "library-generation-build" ] + id: integration-test options: logging: CLOUD_LOGGING_ONLY From fd9b98679f90a68224940cc68286efeeca2c29cd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 20:50:53 -0500 Subject: [PATCH 07/78] install additional deps --- ...ild-library-generation-integration-test.yaml | 17 ++++++++++++++++- .../tests/integration_tests.py | 11 ----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 2eee9ddb91..14ff6f3db0 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -14,6 +14,7 @@ timeout: 7200s # 2 hours substitutions: + _TEMP_TEST_IMAGE_ID: "temp-test-image:latest" _TEST_IMAGE_ID: "test-image:latest" steps: - name: gcr.io/cloud-builders/docker @@ -30,6 +31,20 @@ steps: id: library-generation-build waitFor: ["-"] +- name: gcr.io/cloud-builders/docker + entrypoint: bash + args: + - "c" + - | + docker build -t ${_TEST_IMAGE_ID} - << EOF + FROM ${_TEMP_TEST_IMAGE_ID} + RUN apk update && apk maven + EOF + env: + - "DOCKER_BUILDKIT=1" + id: additional-deps + waitFor: ["library-generation-build"] + - name: ${_TEST_IMAGE_ID} entrypoint: python args: [ @@ -37,7 +52,7 @@ steps: "unittest", "hermetic_build/library_generation/tests/integration_tests.py" ] - waitFor: [ "library-generation-build" ] + waitFor: ["additional-deps"] id: integration-test options: diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index f738a92093..fd692962f1 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -60,7 +60,6 @@ class IntegrationTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) - cls.__build_image(docker_file=build_file, cwd=repo_root_dir) @classmethod def setUp(cls) -> None: @@ -195,16 +194,6 @@ def __copy_api_definition(cls, committish: str) -> str: shutil.rmtree(repo_dest) return api_temp_dir - @classmethod - def __build_image(cls, docker_file: str, cwd: str): - # we build the docker image without removing intermediate containers, so - # we can re-test more quickly - subprocess.check_call( - ["docker", "build", "-f", docker_file, "-t", image_tag, "."], - cwd=cwd, - env=dict(os.environ, DOCKER_BUILDKIT="1"), - ) - @classmethod def __download_generator_jar(cls, coordinates_file: str) -> None: """ From 72ad45c67ad978467fdf4863501d201ac544d007 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 20:57:43 -0500 Subject: [PATCH 08/78] install additional deps --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 14ff6f3db0..77b62bfd2d 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -34,7 +34,7 @@ steps: - name: gcr.io/cloud-builders/docker entrypoint: bash args: - - "c" + - "-c" - | docker build -t ${_TEST_IMAGE_ID} - << EOF FROM ${_TEMP_TEST_IMAGE_ID} From ee7c20e9a8e28d3d341ffe2ecebc0a9e43e28804 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 21:32:35 -0500 Subject: [PATCH 09/78] build add image --- ...cloudbuild-library-generation-integration-test.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 77b62bfd2d..7e709a0861 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -31,17 +31,13 @@ steps: id: library-generation-build waitFor: ["-"] -- name: gcr.io/cloud-builders/docker +- name: ${_TEMP_TEST_IMAGE_ID} entrypoint: bash args: - "-c" - | - docker build -t ${_TEST_IMAGE_ID} - << EOF - FROM ${_TEMP_TEST_IMAGE_ID} - RUN apk update && apk maven - EOF - env: - - "DOCKER_BUILDKIT=1" + apk update && apk maven + DOCKER_BUILDKIT=1 docker build -t "${_TEST_IMAGE_ID}" . id: additional-deps waitFor: ["library-generation-build"] From f1b3a7e4d7b8cf25a036c4d07dfa8c09b8961ef0 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 21:41:33 -0500 Subject: [PATCH 10/78] add image name --- .../cloudbuild-library-generation-integration-test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 7e709a0861..ee228e3492 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -14,8 +14,9 @@ timeout: 7200s # 2 hours substitutions: - _TEMP_TEST_IMAGE_ID: "temp-test-image:latest" - _TEST_IMAGE_ID: "test-image:latest" + _IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation" + _TEMP_TEST_IMAGE_ID: "${_IMAGE_NAME}:temp-test-image" + _TEST_IMAGE_ID: "${_IMAGE_NAME}:test-image" steps: - name: gcr.io/cloud-builders/docker args: [ From 4384936733f8e9bdee53daffd60480a60f0cded8 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 16 Dec 2024 21:53:21 -0500 Subject: [PATCH 11/78] change image id --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index ee228e3492..b333c7e085 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -22,7 +22,7 @@ steps: args: [ "build", "-t", - "${_TEST_IMAGE_ID}", + "${_TEMP_TEST_IMAGE_ID}", "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "." From 37b5fe0dfc04ae746886451c327826ba0b68bdf2 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 07:53:16 -0500 Subject: [PATCH 12/78] update apk cmd --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index b333c7e085..f89659b66a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -37,7 +37,7 @@ steps: args: - "-c" - | - apk update && apk maven + apk update && apk add maven DOCKER_BUILDKIT=1 docker build -t "${_TEST_IMAGE_ID}" . id: additional-deps waitFor: ["library-generation-build"] From 9caf9d9b2d9f0ac35108792f59e7445d9addf8d8 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 08:09:10 -0500 Subject: [PATCH 13/78] add a dockerfile --- ...d-library-generation-integration-test.yaml | 21 ++++++++++--------- .../library_generation_it.Dockerfile | 2 ++ 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 .cloudbuild/library_generation/library_generation_it.Dockerfile diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index f89659b66a..f8da03ce3b 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -14,9 +14,8 @@ timeout: 7200s # 2 hours substitutions: - _IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation" - _TEMP_TEST_IMAGE_ID: "${_IMAGE_NAME}:temp-test-image" - _TEST_IMAGE_ID: "${_IMAGE_NAME}:test-image" + _TEMP_TEST_IMAGE_ID: "intermediate-test-image:latest" + _TEST_IMAGE_ID: "test-image:latest" steps: - name: gcr.io/cloud-builders/docker args: [ @@ -32,13 +31,15 @@ steps: id: library-generation-build waitFor: ["-"] -- name: ${_TEMP_TEST_IMAGE_ID} - entrypoint: bash - args: - - "-c" - - | - apk update && apk add maven - DOCKER_BUILDKIT=1 docker build -t "${_TEST_IMAGE_ID}" . +- name: gcr.io/cloud-builders/docker + args: [ + "build", + "-t", + "${_TEST_IMAGE_ID}", + "--file", + ".cloudbuild/library_generation/library_generation_it.Dockerfile", + "." + ] id: additional-deps waitFor: ["library-generation-build"] diff --git a/.cloudbuild/library_generation/library_generation_it.Dockerfile b/.cloudbuild/library_generation/library_generation_it.Dockerfile new file mode 100644 index 0000000000..d1547f5931 --- /dev/null +++ b/.cloudbuild/library_generation/library_generation_it.Dockerfile @@ -0,0 +1,2 @@ +FROM intermediate-test-image:latest +RUN apk update && apk add maven \ No newline at end of file From 1ec3d8d2bbb9386e7a3e138c74e302acb6d6bd7a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 08:29:56 -0500 Subject: [PATCH 14/78] change entrypoint --- .../cloudbuild-library-generation-integration-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index f8da03ce3b..30cb606974 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -44,8 +44,9 @@ steps: waitFor: ["library-generation-build"] - name: ${_TEST_IMAGE_ID} - entrypoint: python + entrypoint: bash args: [ + "python", "-m", "unittest", "hermetic_build/library_generation/tests/integration_tests.py" From 286b2be27cb2670b12cf60ced77ec28b1475e0fb Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 08:52:23 -0500 Subject: [PATCH 15/78] use python --- .../cloudbuild-library-generation-integration-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 30cb606974..3b931e5e7d 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -43,9 +43,10 @@ steps: id: additional-deps waitFor: ["library-generation-build"] -- name: ${_TEST_IMAGE_ID} +- name: python3.12 entrypoint: bash args: [ + "-c", "python", "-m", "unittest", From 86693889989c71f65f91263a865de00a9126a261 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:05:10 -0500 Subject: [PATCH 16/78] change python tag --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3b931e5e7d..ec18be2bf1 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -43,7 +43,7 @@ steps: id: additional-deps waitFor: ["library-generation-build"] -- name: python3.12 +- name: python:3.13.1 entrypoint: bash args: [ "-c", From 7d3f86c54f5d144281e79620e9234ad24cc0b6d7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:22:35 -0500 Subject: [PATCH 17/78] change cmd --- ...loudbuild-library-generation-integration-test.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index ec18be2bf1..16e5b9fc8b 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -45,13 +45,10 @@ steps: - name: python:3.13.1 entrypoint: bash - args: [ - "-c", - "python", - "-m", - "unittest", - "hermetic_build/library_generation/tests/integration_tests.py" - ] + args: + - "-c" + - | + python -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["additional-deps"] id: integration-test From cc09441bad7a78e174ea1e72c02fc0def265d075 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:33:08 -0500 Subject: [PATCH 18/78] install deps --- ...ld-library-generation-integration-test.yaml | 2 ++ .cloudbuild/library_generation/requirements.in | 1 + .../library_generation/requirements.txt | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 .cloudbuild/library_generation/requirements.in create mode 100644 .cloudbuild/library_generation/requirements.txt diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 16e5b9fc8b..a7896472a8 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,6 +48,8 @@ steps: args: - "-c" - | + python -m pip install --upgrade pip + python -m pip install --require-hashes -r .cloudbuild/library_generation/requirements.txt python -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["additional-deps"] id: integration-test diff --git a/.cloudbuild/library_generation/requirements.in b/.cloudbuild/library_generation/requirements.in new file mode 100644 index 0000000000..04040f6ff4 --- /dev/null +++ b/.cloudbuild/library_generation/requirements.in @@ -0,0 +1 @@ +GitPython==3.1.43 \ No newline at end of file diff --git a/.cloudbuild/library_generation/requirements.txt b/.cloudbuild/library_generation/requirements.txt new file mode 100644 index 0000000000..5c93f54dea --- /dev/null +++ b/.cloudbuild/library_generation/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes .cloudbuild/library_generation/requirements.in +# +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff + # via -r .cloudbuild/library_generation/requirements.in +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb From 03d7518969bd2da0f211fccb269d9d161af0131d Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:42:45 -0500 Subject: [PATCH 19/78] install deps --- ...ld-library-generation-integration-test.yaml | 7 +++++-- .cloudbuild/library_generation/requirements.in | 1 - .../library_generation/requirements.txt | 18 ------------------ 3 files changed, 5 insertions(+), 21 deletions(-) delete mode 100644 .cloudbuild/library_generation/requirements.in delete mode 100644 .cloudbuild/library_generation/requirements.txt diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index a7896472a8..67f45f0535 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,8 +48,11 @@ steps: args: - "-c" - | - python -m pip install --upgrade pip - python -m pip install --require-hashes -r .cloudbuild/library_generation/requirements.txt + pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt + pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt + pip install hermetic_build/library_generation python -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["additional-deps"] id: integration-test diff --git a/.cloudbuild/library_generation/requirements.in b/.cloudbuild/library_generation/requirements.in deleted file mode 100644 index 04040f6ff4..0000000000 --- a/.cloudbuild/library_generation/requirements.in +++ /dev/null @@ -1 +0,0 @@ -GitPython==3.1.43 \ No newline at end of file diff --git a/.cloudbuild/library_generation/requirements.txt b/.cloudbuild/library_generation/requirements.txt deleted file mode 100644 index 5c93f54dea..0000000000 --- a/.cloudbuild/library_generation/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes .cloudbuild/library_generation/requirements.in -# -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via gitpython -gitpython==3.1.43 \ - --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ - --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff - # via -r .cloudbuild/library_generation/requirements.in -smmap==5.0.1 \ - --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ - --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da - # via gitdb From 14fda4dec4f41b9b50c7d0251586dc418879efd4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:45:49 -0500 Subject: [PATCH 20/78] restart ci From 06487f8cefeac609bf36461d18fbdc5e9e0593af Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 09:55:35 -0500 Subject: [PATCH 21/78] install maven --- ...d-library-generation-integration-test.yaml | 19 ++++--------------- .../library_generation_it.Dockerfile | 2 -- 2 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 .cloudbuild/library_generation/library_generation_it.Dockerfile diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 67f45f0535..c92756a625 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -14,14 +14,13 @@ timeout: 7200s # 2 hours substitutions: - _TEMP_TEST_IMAGE_ID: "intermediate-test-image:latest" _TEST_IMAGE_ID: "test-image:latest" steps: - name: gcr.io/cloud-builders/docker args: [ "build", "-t", - "${_TEMP_TEST_IMAGE_ID}", + "${_TEST_IMAGE_ID}", "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "." @@ -31,30 +30,20 @@ steps: id: library-generation-build waitFor: ["-"] -- name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", - "${_TEST_IMAGE_ID}", - "--file", - ".cloudbuild/library_generation/library_generation_it.Dockerfile", - "." - ] - id: additional-deps - waitFor: ["library-generation-build"] - - name: python:3.13.1 entrypoint: bash args: - "-c" - | + sudo apt-get update + sudo apt-get install maven pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation python -m unittest hermetic_build/library_generation/tests/integration_tests.py - waitFor: ["additional-deps"] + waitFor: ["library-generation-build"] id: integration-test options: diff --git a/.cloudbuild/library_generation/library_generation_it.Dockerfile b/.cloudbuild/library_generation/library_generation_it.Dockerfile deleted file mode 100644 index d1547f5931..0000000000 --- a/.cloudbuild/library_generation/library_generation_it.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM intermediate-test-image:latest -RUN apk update && apk add maven \ No newline at end of file From 3c795f4bc20203da287e388f3c126c5071f17c1c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:02:05 -0500 Subject: [PATCH 22/78] install maven --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c92756a625..c335b90500 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -35,8 +35,8 @@ steps: args: - "-c" - | - sudo apt-get update - sudo apt-get install maven + apt-get update + apt-get install maven pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common From a5b14a04565e47dcf889b8dd6f52e6b4a26a82e7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:12:13 -0500 Subject: [PATCH 23/78] assume yes --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c335b90500..93b4d1b636 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -36,7 +36,7 @@ steps: - "-c" - | apt-get update - apt-get install maven + apt-get -y install maven pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common From 427ddace953073a8e703e5cbbd502b6a77d770e3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:27:34 -0500 Subject: [PATCH 24/78] install docker --- .../cloudbuild-library-generation-integration-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 93b4d1b636..06c756649f 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -37,6 +37,7 @@ steps: - | apt-get update apt-get -y install maven + apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common From e9f8036e3e66ee875ba589e83684bfbf43665b24 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:42:29 -0500 Subject: [PATCH 25/78] install docker --- ...dbuild-library-generation-integration-test.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 06c756649f..7c73fbe358 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -30,14 +30,24 @@ steps: id: library-generation-build waitFor: ["-"] -- name: python:3.13.1 +- name: ubuntu:25.04 entrypoint: bash args: - "-c" - | apt-get update - apt-get -y install maven + apt-get -y install maven sudo ca-certificates curl + # Install docker, see https://docs.docker.com/engine/install/debian/#install-using-the-repository + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + # Install python dependencies pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common From 410333595524f71ea1a65fb7b443bfbc0a7dd2da Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:43:48 -0500 Subject: [PATCH 26/78] restore --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 7c73fbe358..32d8c07a9e 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -30,7 +30,7 @@ steps: id: library-generation-build waitFor: ["-"] -- name: ubuntu:25.04 +- name: python:3.13.1 entrypoint: bash args: - "-c" @@ -45,7 +45,7 @@ steps: echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # Install python dependencies pip install --upgrade pip From 2642f7468a99d5ac03cb841ac0bfde3b1c31696a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:46:29 -0500 Subject: [PATCH 27/78] add env --- .../cloudbuild-library-generation-integration-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 32d8c07a9e..150b1e0298 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -54,6 +54,8 @@ steps: pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation python -m unittest hermetic_build/library_generation/tests/integration_tests.py + env: + - "VERSION_CODENAME=bookworm" waitFor: ["library-generation-build"] id: integration-test From af6edb38b8b82ab61bd2ae36a8342c3e76cd2a48 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:47:40 -0500 Subject: [PATCH 28/78] add env --- .../cloudbuild-library-generation-integration-test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 150b1e0298..03efffc3d8 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -15,6 +15,7 @@ timeout: 7200s # 2 hours substitutions: _TEST_IMAGE_ID: "test-image:latest" + VERSION_CODENAME: "bookworm" steps: - name: gcr.io/cloud-builders/docker args: [ @@ -44,7 +45,7 @@ steps: # Add the repository to Apt sources: echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + $(. /etc/os-release && echo "${VERSION_CODENAME}") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # Install python dependencies @@ -54,8 +55,6 @@ steps: pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation python -m unittest hermetic_build/library_generation/tests/integration_tests.py - env: - - "VERSION_CODENAME=bookworm" waitFor: ["library-generation-build"] id: integration-test From 1732a09da6cc043cd845ba73079aaaced4efa12c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 10:48:15 -0500 Subject: [PATCH 29/78] add env --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 03efffc3d8..29a5b2c949 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _TEST_IMAGE_ID: "test-image:latest" - VERSION_CODENAME: "bookworm" + _VERSION_CODENAME: "bookworm" steps: - name: gcr.io/cloud-builders/docker args: [ @@ -45,7 +45,7 @@ steps: # Add the repository to Apt sources: echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "${VERSION_CODENAME}") stable" | \ + $(. /etc/os-release && echo "${_VERSION_CODENAME}") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # Install python dependencies From 11efa79b9a598a700888afc046654715b5cd899e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:28:29 -0500 Subject: [PATCH 30/78] change base image --- ...d-library-generation-integration-test.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 29a5b2c949..12b387310a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -31,23 +31,25 @@ steps: id: library-generation-build waitFor: ["-"] -- name: python:3.13.1 +- name: ubuntu:25.04 entrypoint: bash args: - "-c" - | - apt-get update - apt-get -y install maven sudo ca-certificates curl - # Install docker, see https://docs.docker.com/engine/install/debian/#install-using-the-repository + # Install docker, see https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository + sudo apt-get update + sudo apt-get install ca-certificates curl sudo install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc - chmod a+r /etc/apt/keyrings/docker.asc - # Add the repository to Apt sources: + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "${_VERSION_CODENAME}") stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo apt-get update + sudo apt-get install \ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ + maven python # Install python dependencies pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt From 96e4aa7f024ebc1059936d0e5b70157599d4e1fb Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:37:13 -0500 Subject: [PATCH 31/78] rm sudo --- ...d-library-generation-integration-test.yaml | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 12b387310a..239ee64c3a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -17,19 +17,19 @@ substitutions: _TEST_IMAGE_ID: "test-image:latest" _VERSION_CODENAME: "bookworm" steps: -- name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", - "${_TEST_IMAGE_ID}", - "--file", - ".cloudbuild/library_generation/library_generation.Dockerfile", - "." - ] - env: - - "DOCKER_BUILDKIT=1" - id: library-generation-build - waitFor: ["-"] +#- name: gcr.io/cloud-builders/docker +# args: [ +# "build", +# "-t", +# "${_TEST_IMAGE_ID}", +# "--file", +# ".cloudbuild/library_generation/library_generation.Dockerfile", +# "." +# ] +# env: +# - "DOCKER_BUILDKIT=1" +# id: library-generation-build +# waitFor: ["-"] - name: ubuntu:25.04 entrypoint: bash @@ -37,17 +37,17 @@ steps: - "-c" - | # Install docker, see https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc + apt-get update + apt-get install ca-certificates curl + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "${_VERSION_CODENAME}") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt-get install \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-get install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ maven python # Install python dependencies @@ -57,7 +57,7 @@ steps: pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation python -m unittest hermetic_build/library_generation/tests/integration_tests.py - waitFor: ["library-generation-build"] + waitFor: ["-"] id: integration-test options: From 50af6248ae9f60f2b84f7d6d84f7d9992101a93d Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:39:58 -0500 Subject: [PATCH 32/78] assume yes --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 239ee64c3a..269815c9c2 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -38,7 +38,7 @@ steps: - | # Install docker, see https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository apt-get update - apt-get install ca-certificates curl + apt-get install -y ca-certificates curl install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc @@ -47,7 +47,7 @@ steps: $(. /etc/os-release && echo "${_VERSION_CODENAME}") stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get update - apt-get install \ + apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ maven python # Install python dependencies From b2014579f060d5b335182ab99f6520e0e63a7e3b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:45:47 -0500 Subject: [PATCH 33/78] codename --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 269815c9c2..dcae10dac5 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _TEST_IMAGE_ID: "test-image:latest" - _VERSION_CODENAME: "bookworm" + _VERSION_CODENAME: "Noble Numbat" steps: #- name: gcr.io/cloud-builders/docker # args: [ @@ -31,7 +31,7 @@ steps: # id: library-generation-build # waitFor: ["-"] -- name: ubuntu:25.04 +- name: ubuntu:24.04 entrypoint: bash args: - "-c" From 05f53c9313615f098b338d814854778c9feaf307 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:53:37 -0500 Subject: [PATCH 34/78] change command --- ...build-library-generation-integration-test.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index dcae10dac5..d6647085e4 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -15,7 +15,6 @@ timeout: 7200s # 2 hours substitutions: _TEST_IMAGE_ID: "test-image:latest" - _VERSION_CODENAME: "Noble Numbat" steps: #- name: gcr.io/cloud-builders/docker # args: [ @@ -36,15 +35,15 @@ steps: args: - "-c" - | - # Install docker, see https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository + # Install docker apt-get update - apt-get install -y ca-certificates curl - install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - chmod a+r /etc/apt/keyrings/docker.asc + apt-get install -y ca-certificates curl gnupg lsb-release + mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ + gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "${_VERSION_CODENAME}") stable" | \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get update apt-get -y install \ From c93ec40a72207299ea073e637c3f788c7f14219a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 13:57:37 -0500 Subject: [PATCH 35/78] install python --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index d6647085e4..f57e346c5a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,7 +48,7 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python + maven python3 python3-pip # Install python dependencies pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt From d0924432e3843926aa5ec28da931108ef3019d9f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:08:52 -0500 Subject: [PATCH 36/78] restore it --- .../library_generation/tests/integration_tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index fd692962f1..f738a92093 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -60,6 +60,7 @@ class IntegrationTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) + cls.__build_image(docker_file=build_file, cwd=repo_root_dir) @classmethod def setUp(cls) -> None: @@ -194,6 +195,16 @@ def __copy_api_definition(cls, committish: str) -> str: shutil.rmtree(repo_dest) return api_temp_dir + @classmethod + def __build_image(cls, docker_file: str, cwd: str): + # we build the docker image without removing intermediate containers, so + # we can re-test more quickly + subprocess.check_call( + ["docker", "build", "-f", docker_file, "-t", image_tag, "."], + cwd=cwd, + env=dict(os.environ, DOCKER_BUILDKIT="1"), + ) + @classmethod def __download_generator_jar(cls, coordinates_file: str) -> None: """ From ed00803fc2f16ac040533b9f0f838c606d134375 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:09:37 -0500 Subject: [PATCH 37/78] rm python --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index f57e346c5a..c8d93ceb5a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,7 +48,7 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python3 python3-pip + maven python3-pip # Install python dependencies pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt From 5119e325d29b04f2caeaa6e1076cae1a16d1261a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:13:10 -0500 Subject: [PATCH 38/78] rm python --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c8d93ceb5a..b2ac28113f 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,14 +48,14 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python3-pip + maven # Install python dependencies pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation - python -m unittest hermetic_build/library_generation/tests/integration_tests.py + python3 -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["-"] id: integration-test From a38bad285e7db147f6bd14698430e8bc9d8b2b73 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:15:12 -0500 Subject: [PATCH 39/78] use python3 --- ...cloudbuild-library-generation-integration-test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index b2ac28113f..47faa2be06 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -50,11 +50,11 @@ steps: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ maven # Install python dependencies - pip install --upgrade pip - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation + python3 -mpip install --upgrade pip + python3 -mpip install --require-hashes -r hermetic_build/common/requirements.txt + python3 -mpip install hermetic_build/common + python3 -mpip install --require-hashes -r hermetic_build/library_generation/requirements.txt + python3 -mpip install hermetic_build/library_generation python3 -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["-"] id: integration-test From 817e140272e51585121e784fbfde75e75816c4ae Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:18:46 -0500 Subject: [PATCH 40/78] install pip3 --- ...oudbuild-library-generation-integration-test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 47faa2be06..ba40fc4478 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,13 +48,13 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven + maven python3-pip # Install python dependencies - python3 -mpip install --upgrade pip - python3 -mpip install --require-hashes -r hermetic_build/common/requirements.txt - python3 -mpip install hermetic_build/common - python3 -mpip install --require-hashes -r hermetic_build/library_generation/requirements.txt - python3 -mpip install hermetic_build/library_generation + pip3 install --upgrade pip + pip3 install --require-hashes -r hermetic_build/common/requirements.txt + pip3 install hermetic_build/common + pip3 install --require-hashes -r hermetic_build/library_generation/requirements.txt + pip3 install hermetic_build/library_generation python3 -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["-"] id: integration-test From 7dcc870b86be94b03e4234c6d3376d1e1efc90ab Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:22:29 -0500 Subject: [PATCH 41/78] install venv --- ...dbuild-library-generation-integration-test.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index ba40fc4478..fe044e1378 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,13 +48,15 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python3-pip + maven # Install python dependencies - pip3 install --upgrade pip - pip3 install --require-hashes -r hermetic_build/common/requirements.txt - pip3 install hermetic_build/common - pip3 install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip3 install hermetic_build/library_generation + python3 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt + pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt + pip install hermetic_build/library_generation python3 -m unittest hermetic_build/library_generation/tests/integration_tests.py waitFor: ["-"] id: integration-test From 68e91a5d8b83a2eebc63f755e32ca8346dfae9e3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:25:09 -0500 Subject: [PATCH 42/78] install python3-virtualenv --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index fe044e1378..e109cff72d 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,7 +48,7 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven + maven python3-virtualenv # Install python dependencies python3 -m venv .venv source .venv/bin/activate From 91f5c28be4d35d79afaca322ee4e949677a0098b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 15:27:49 -0500 Subject: [PATCH 43/78] install python3.12-venv --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index e109cff72d..3e28054c19 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -48,7 +48,7 @@ steps: apt-get update apt-get -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python3-virtualenv + maven python3.12-venv # Install python dependencies python3 -m venv .venv source .venv/bin/activate From dce4c4c47e480d39917554bb9b70c90ca1cd03a3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 17 Dec 2024 16:55:44 -0500 Subject: [PATCH 44/78] remove unused param --- ...uild-library-generation-integration-test.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3e28054c19..af92856f02 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -13,23 +13,7 @@ # limitations under the License. timeout: 7200s # 2 hours -substitutions: - _TEST_IMAGE_ID: "test-image:latest" steps: -#- name: gcr.io/cloud-builders/docker -# args: [ -# "build", -# "-t", -# "${_TEST_IMAGE_ID}", -# "--file", -# ".cloudbuild/library_generation/library_generation.Dockerfile", -# "." -# ] -# env: -# - "DOCKER_BUILDKIT=1" -# id: library-generation-build -# waitFor: ["-"] - - name: ubuntu:24.04 entrypoint: bash args: From 27119081e618ce390d12c21d4acda0e513cdbf03 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 12:36:10 -0500 Subject: [PATCH 45/78] change dockerfile --- hermetic_build/library_generation/tests/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index f738a92093..72d1949d27 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -33,7 +33,7 @@ generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") repo_root_dir = os.path.join(script_dir, "..", "..", "..") build_file = os.path.join( - repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" + repo_root_dir, ".cloudbuild", "library_generation", "library_generation_airlock.Dockerfile" ) image_tag = "test-image:latest" repo_prefix = "https://github.com/googleapis" From e02b4a8adfb5b7f5c45ec3b787b7d443cc5a4e55 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 13:27:14 -0500 Subject: [PATCH 46/78] debug --- hermetic_build/library_generation/generate_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 2e233acb7b..0c69dc8410 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import shutil from typing import Optional import library_generation.utils.utilities as util @@ -49,7 +50,7 @@ def generate_from_yaml( ) # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - + print(f"Contents in output: {os.listdir(repo_config.output_folder)}") for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") generate_composed_library( From 779bf703954e20edf7b022b5adc0218f724ebc86 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 13:42:07 -0500 Subject: [PATCH 47/78] debug --- hermetic_build/library_generation/generate_repo.py | 1 + hermetic_build/library_generation/tests/integration_tests.py | 1 + 2 files changed, 2 insertions(+) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 0c69dc8410..8bdc0e9782 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -48,6 +48,7 @@ def generate_from_yaml( repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) + print(f"Contents in api definition: {os.listdir(api_definitions_path)}") # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) print(f"Contents in output: {os.listdir(repo_config.output_folder)}") diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 72d1949d27..94162eb9ed 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -69,6 +69,7 @@ def setUp(cls) -> None: def test_entry_point_running_in_container(self): api_definitions_path = self.__copy_api_definition(googleapis_commitish) + print(f"Contents in host: {os.listdir(api_definitions_path)}") config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: config = from_yaml(config_file) From 401902b57115500a21be5df3fb32b9744504b32a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 13:43:07 -0500 Subject: [PATCH 48/78] show python version --- .../cloudbuild-library-generation-integration-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index af92856f02..6960545970 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -36,6 +36,7 @@ steps: # Install python dependencies python3 -m venv .venv source .venv/bin/activate + python --version pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common From a50bdf3f685a50fce598bdd434c9656548ccb913 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 13:53:23 -0500 Subject: [PATCH 49/78] debug --- hermetic_build/library_generation/generate_repo.py | 2 ++ hermetic_build/library_generation/tests/integration_tests.py | 1 + 2 files changed, 3 insertions(+) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 8bdc0e9782..0cb6e721ff 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -48,9 +48,11 @@ def generate_from_yaml( repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) + print(f"api definition in docker: {api_definitions_path}") print(f"Contents in api definition: {os.listdir(api_definitions_path)}") # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) + print(f"output folder: {repo_config.output_folder}") print(f"Contents in output: {os.listdir(repo_config.output_folder)}") for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 94162eb9ed..f189068a6a 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -69,6 +69,7 @@ def setUp(cls) -> None: def test_entry_point_running_in_container(self): api_definitions_path = self.__copy_api_definition(googleapis_commitish) + print(f"api definition path in host: {api_definitions_path}") print(f"Contents in host: {os.listdir(api_definitions_path)}") config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: From d241b86a2e8df5bd82556fe8da0557bc95b16ed7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 13:59:09 -0500 Subject: [PATCH 50/78] separate it --- ...d-library-generation-integration-test.yaml | 2 +- .../tests/integration_tests.py | 2 +- .../tests/integration_tests_cloud_build.py | 373 ++++++++++++++++++ 3 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 hermetic_build/library_generation/tests/integration_tests_cloud_build.py diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 6960545970..722d321991 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -42,7 +42,7 @@ steps: pip install hermetic_build/common pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation - python3 -m unittest hermetic_build/library_generation/tests/integration_tests.py + python3 -m unittest hermetic_build/library_generation/tests/integration_tests_cloud_build.py waitFor: ["-"] id: integration-test diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index f189068a6a..8c5ee67b19 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -33,7 +33,7 @@ generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") repo_root_dir = os.path.join(script_dir, "..", "..", "..") build_file = os.path.join( - repo_root_dir, ".cloudbuild", "library_generation", "library_generation_airlock.Dockerfile" + repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" ) image_tag = "test-image:latest" repo_prefix = "https://github.com/googleapis" diff --git a/hermetic_build/library_generation/tests/integration_tests_cloud_build.py b/hermetic_build/library_generation/tests/integration_tests_cloud_build.py new file mode 100644 index 0000000000..f189068a6a --- /dev/null +++ b/hermetic_build/library_generation/tests/integration_tests_cloud_build.py @@ -0,0 +1,373 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import difflib +import json +import tempfile +from filecmp import dircmp +from git import Repo +import os +import shutil +import subprocess +import unittest +from pathlib import Path +from common.model.generation_config import GenerationConfig +from common.model.generation_config import from_yaml +from library_generation.tests.compare_poms import compare_xml +from library_generation.utils.utilities import sh_util as shell_call + + +script_dir = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(script_dir, "resources", "integration") +golden_dir = os.path.join(config_dir, "golden") +generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") +repo_root_dir = os.path.join(script_dir, "..", "..", "..") +build_file = os.path.join( + repo_root_dir, ".cloudbuild", "library_generation", "library_generation_airlock.Dockerfile" +) +image_tag = "test-image:latest" +repo_prefix = "https://github.com/googleapis" +output_dir = shell_call("get_output_folder") +# this map tells which branch of each repo should we use for our diff tests +commitish_map = { + "google-cloud-java": "chore/test-hermetic-build", + "java-bigtable": "chore/test-hermetic-build", +} +generation_config_name = "generation_config.yaml" +googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" +# This variable is used to override the jar created by building the image +# with our own downloaded jar in order to lock the integration test to use +# a constant version specified in +# library_generation/test/resources/integration/test_generator_coordinates +# This allows us to decouple the generation workflow testing with what the +# generator jar will actually generate. +# See library_generation/DEVELOPMENT.md ("The Hermetic Build's +# well-known folder"). +WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" + + +class IntegrationTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) + cls.__build_image(docker_file=build_file, cwd=repo_root_dir) + + @classmethod + def setUp(cls) -> None: + cls.__remove_generated_files() + os.makedirs(f"{golden_dir}", exist_ok=True) + + def test_entry_point_running_in_container(self): + api_definitions_path = self.__copy_api_definition(googleapis_commitish) + print(f"api definition path in host: {api_definitions_path}") + print(f"Contents in host: {os.listdir(api_definitions_path)}") + config_files = self.__get_config_files(config_dir) + for repo, config_file in config_files: + config = from_yaml(config_file) + repo_location = f"{output_dir}/{repo}" + config_location = f"{golden_dir}/../{repo}" + # 1. pull repository + repo_dest = self.__pull_repo_to( + Path(repo_location), repo, commitish_map[repo] + ) + # 2. prepare golden files + library_names = self.__get_library_names_from_config(config) + self.__prepare_golden_files( + config=config, library_names=library_names, repo_dest=repo_dest + ) + # 3. run entry_point.py in docker container + self.__run_entry_point_in_docker_container( + repo_location=repo_location, + config_location=config_location, + generation_config=generation_config_name, + api_definition=api_definitions_path, + ) + # 4. compare generation result with golden files + print( + "Generation finished successfully. " + "Will now compare differences between generated and existing " + "libraries" + ) + for library_name in library_names: + actual_library = ( + f"{repo_dest}/{library_name}" if config.is_monorepo() else repo_dest + ) + print("*" * 50) + print(f"Checking for differences in '{library_name}'.") + print(f" The expected library is in {golden_dir}/{library_name}.") + print(f" The actual library is in {actual_library}. ") + compare_result = dircmp( + f"{golden_dir}/{library_name}", + actual_library, + ignore=[".repo-metadata.json"], + ) + diff_files = [] + golden_only = [] + generated_only = [] + # compare source code + self.__recursive_diff_files( + compare_result, diff_files, golden_only, generated_only + ) + + # print all found differences for inspection + print_file = lambda f: print(f" - {f}") + if len(diff_files) > 0: + print(" Some files (found in both folders) are differing:") + for diff_file in diff_files: + print(f"Difference in {diff_file}:") + with open( + f"{golden_dir}/{library_name}/{diff_file}" + ) as expected_file: + with open(f"{actual_library}/{diff_file}") as actual_file: + [ + print(line) + for line in difflib.unified_diff( + expected_file.readlines(), + actual_file.readlines(), + ) + ] + if len(golden_only) > 0: + print(" There were files found only in the golden dir:") + [print_file(f) for f in golden_only] + if len(generated_only) > 0: + print(" There were files found only in the generated dir:") + [print_file(f) for f in generated_only] + + self.assertTrue(len(golden_only) == 0) + self.assertTrue(len(generated_only) == 0) + self.assertTrue(len(diff_files) == 0) + + print(f" No differences found in {library_name}") + # compare .repo-metadata.json + self.assertTrue( + self.__compare_json_files( + f"{golden_dir}/{library_name}/.repo-metadata.json", + f"{actual_library}/.repo-metadata.json", + ), + msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", + ) + print(" .repo-metadata.json comparison succeed.") + + if not config.is_monorepo(): + continue + + # compare gapic-libraries-bom/pom.xml and pom.xml + self.assertFalse( + compare_xml( + f"{golden_dir}/gapic-libraries-bom/pom.xml", + f"{repo_dest}/gapic-libraries-bom/pom.xml", + False, + ) + ) + print(" gapic-libraries-bom/pom.xml comparison succeed.") + self.assertFalse( + compare_xml( + f"{golden_dir}/pom.xml", + f"{repo_dest}/pom.xml", + False, + ) + ) + print(" pom.xml comparison succeed.") + self.__remove_generated_files() + shutil.rmtree(api_definitions_path) + + @classmethod + def __copy_api_definition(cls, committish: str) -> str: + repo_dest = cls.__pull_repo_to( + dest=tempfile.mkdtemp(), repo="googleapis", committish=committish + ) + api_temp_dir = tempfile.mkdtemp() + print(f"Copying api definition to {api_temp_dir}...") + shutil.copytree( + f"{repo_dest}/google", f"{api_temp_dir}/google", dirs_exist_ok=True + ) + shutil.copytree( + f"{repo_dest}/grafeas", f"{api_temp_dir}/grafeas", dirs_exist_ok=True + ) + shutil.rmtree(repo_dest) + return api_temp_dir + + @classmethod + def __build_image(cls, docker_file: str, cwd: str): + # we build the docker image without removing intermediate containers, so + # we can re-test more quickly + subprocess.check_call( + ["docker", "build", "-f", docker_file, "-t", image_tag, "."], + cwd=cwd, + env=dict(os.environ, DOCKER_BUILDKIT="1"), + ) + + @classmethod + def __download_generator_jar(cls, coordinates_file: str) -> None: + """ + Downloads the jar at the version specified in the + coordinates file + :param coordinates_file: path to the file containing the coordinates + """ + with open(coordinates_file, "r") as coordinates_file_handle: + # make this var available in the function scope + # nonlocal coordinates + coordinates = coordinates_file_handle.read() + # download the jar + subprocess.check_call( + [ + "mvn", + "dependency:copy", + "-B", + "-ntp", + f"-Dartifact={coordinates}", + f"-DoutputDirectory={config_dir}", + ] + ) + + # compute the filename of the downloaded jar + split_coordinates = coordinates.split(":") + artifact_id = split_coordinates[1] + version = split_coordinates[2] + jar_filename = f"{artifact_id}-{version}.jar" + + # rename the jar to its well-known filename defined at the top of this + # script file + source_jar_path = os.path.join(config_dir, jar_filename) + destination_jar_path = os.path.join( + config_dir, WELL_KNOWN_GENERATOR_JAR_FILENAME + ) + shutil.move(source_jar_path, destination_jar_path) + + @classmethod + def __remove_generated_files(cls): + shutil.rmtree(f"{output_dir}", ignore_errors=True) + if os.path.isdir(f"{golden_dir}"): + shutil.rmtree(f"{golden_dir}") + + @classmethod + def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: + shutil.rmtree(dest, ignore_errors=True) + repo_url = f"{repo_prefix}/{repo}" + print(f"Cloning repository {repo_url}") + repo = Repo.clone_from(repo_url, dest) + repo.git.checkout(committish) + return str(dest) + + @classmethod + def __get_library_names_from_config(cls, config: GenerationConfig) -> list[str]: + library_names = [] + for library in config.libraries: + library_names.append(f"java-{library.get_library_name()}") + + return library_names + + @classmethod + def __prepare_golden_files( + cls, config: GenerationConfig, library_names: list[str], repo_dest: str + ): + for library_name in library_names: + if config.is_monorepo(): + shutil.copytree( + f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}" + ) + shutil.copytree( + f"{repo_dest}/gapic-libraries-bom", + f"{golden_dir}/gapic-libraries-bom", + dirs_exist_ok=True, + ) + shutil.copyfile(f"{repo_dest}/pom.xml", f"{golden_dir}/pom.xml") + else: + shutil.copytree(f"{repo_dest}", f"{golden_dir}/{library_name}") + + @classmethod + def __run_entry_point_in_docker_container( + cls, + repo_location: str, + config_location: str, + generation_config: str, + api_definition: str, + ): + # we use the calling user to prevent the mapped volumes from changing + # owners + user_id = shell_call("id -u") + group_id = shell_call("id -g") + subprocess.check_call( + [ + "docker", + "run", + "-u", + f"{user_id}:{group_id}", + "--rm", + "--quiet", + "--network", + "none", + "-v", + f"{repo_location}:/workspace/repo", + "-v", + f"{config_location}:/workspace/config", + "-v", + f"{api_definition}:/workspace/api", + "-v", + f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", + "-w", + "/workspace/repo", + image_tag, + f"--generation-config-path=/workspace/config/{generation_config}", + f"--api-definitions-path=/workspace/api", + ], + ) + + @classmethod + def __get_config_files(cls, path: str) -> list[tuple[str, str]]: + config_files = [] + for sub_dir in Path(path).resolve().iterdir(): + if sub_dir.is_file(): + continue + repo = sub_dir.name + if repo in ["golden", "java-bigtable"]: + continue + config = f"{sub_dir}/{generation_config_name}" + config_files.append((repo, config)) + return config_files + + @classmethod + def __compare_json_files(cls, expected: str, actual: str) -> bool: + return cls.__load_json_to_sorted_list( + expected + ) == cls.__load_json_to_sorted_list(actual) + + @classmethod + def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: + with open(path) as f: + data = json.load(f) + res = [(key, value) for key, value in data.items()] + + return sorted(res, key=lambda x: x[0]) + + @classmethod + def __recursive_diff_files( + cls, + dcmp: dircmp, + diff_files: list[str], + left_only: list[str], + right_only: list[str], + dirname: str = "", + ): + """ + Recursively compares two subdirectories. The found differences are + passed to three expected list references. + """ + append_dirname = lambda d: dirname + d + diff_files.extend(map(append_dirname, dcmp.diff_files)) + left_only.extend(map(append_dirname, dcmp.left_only)) + right_only.extend(map(append_dirname, dcmp.right_only)) + for sub_dirname, sub_dcmp in dcmp.subdirs.items(): + cls.__recursive_diff_files( + sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" + ) From e58cf09c3c6336d47637186df4b3a4cf7a630aee Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 14:00:19 -0500 Subject: [PATCH 51/78] format --- .../tests/integration_tests_cloud_build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests_cloud_build.py b/hermetic_build/library_generation/tests/integration_tests_cloud_build.py index f189068a6a..bf69596911 100644 --- a/hermetic_build/library_generation/tests/integration_tests_cloud_build.py +++ b/hermetic_build/library_generation/tests/integration_tests_cloud_build.py @@ -33,7 +33,10 @@ generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") repo_root_dir = os.path.join(script_dir, "..", "..", "..") build_file = os.path.join( - repo_root_dir, ".cloudbuild", "library_generation", "library_generation_airlock.Dockerfile" + repo_root_dir, + ".cloudbuild", + "library_generation", + "library_generation_airlock.Dockerfile", ) image_tag = "test-image:latest" repo_prefix = "https://github.com/googleapis" From ce51ccb25d9c6f9a8ad83f05a7f080b96b4723fc Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 18:48:27 -0500 Subject: [PATCH 52/78] move steps to config --- ...d-library-generation-integration-test.yaml | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 722d321991..54a17bd95c 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -13,38 +13,51 @@ # limitations under the License. timeout: 7200s # 2 hours +substitutions: + _TEST_IMAGE: "test-image:latest" steps: +- name: gcr.io/cloud-builders/docker + args: [ + "build", + "-t", "${_TEST_IMAGE}", + "-f", ".cloudbuild/library_generation/library_generation_airlock.Dockerfile", + "." + ] + id: build-image + waitFor: ["-"] + env: + - "DOCKER_BUILDKIT=1" +- name: ubuntu:24.04 + entrypoint: bash + args: + - "-c" + - | + cd /workspace + git clone https://github.com/googleapis/googleapis + cd googleapis + git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 + id: download-api-definitions + waitFor: ["-"] - name: ubuntu:24.04 entrypoint: bash args: - - "-c" - - | - # Install docker - apt-get update - apt-get install -y ca-certificates curl gnupg lsb-release - mkdir -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ - gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - apt-get update - apt-get -y install \ - docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - maven python3.12-venv - # Install python dependencies - python3 -m venv .venv - source .venv/bin/activate - python --version - pip install --upgrade pip - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation - python3 -m unittest hermetic_build/library_generation/tests/integration_tests_cloud_build.py + - "-c" + - | + cd /workspace + git clone https://github.com/googleapis/google-cloud-java + cd google-cloud-java + git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 + git checkout chore/test-hermetic-build + mkdir ../golden + cd ../golden + cp -r ../google-cloud-java/java-apigee-connect . + cp -r ../google-cloud-java/java-alloydb . + cp -r ../google-cloud-java/java-alloydb-connectors . + cp -r ../google-cloud-java/java-cloudcontrolspartner . + cp -r ../google-cloud-java/gapic-libraries-bom . + cp -r ../google-cloud-java/pom.xml . + id: prepare-golden waitFor: ["-"] - id: integration-test options: logging: CLOUD_LOGGING_ONLY From e3001755a48606535a0c8a269cef5fd001f466eb Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 18:52:20 -0500 Subject: [PATCH 53/78] change base image --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 54a17bd95c..c9aeeb1936 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -27,7 +27,7 @@ steps: waitFor: ["-"] env: - "DOCKER_BUILDKIT=1" -- name: ubuntu:24.04 +- name: alpine/git:latest entrypoint: bash args: - "-c" @@ -38,7 +38,7 @@ steps: git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 id: download-api-definitions waitFor: ["-"] -- name: ubuntu:24.04 +- name: alpine/git:latest entrypoint: bash args: - "-c" From 9885e45d5d30701a410bc532ae407576053c1c84 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 18:55:18 -0500 Subject: [PATCH 54/78] change base image --- .../cloudbuild-library-generation-integration-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c9aeeb1936..c04d40a6dd 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -27,7 +27,7 @@ steps: waitFor: ["-"] env: - "DOCKER_BUILDKIT=1" -- name: alpine/git:latest +- name: alpine/git@sha256:0528136dcb1c3fc461556c5796d6214efa530166cba922c28e9ae146d9acfb82 entrypoint: bash args: - "-c" @@ -38,7 +38,7 @@ steps: git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 id: download-api-definitions waitFor: ["-"] -- name: alpine/git:latest +- name: alpine/git@sha256:0528136dcb1c3fc461556c5796d6214efa530166cba922c28e9ae146d9acfb82 entrypoint: bash args: - "-c" From f7fe0a67f44e6a959eb102156b63c536cb4c2301 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 18:58:08 -0500 Subject: [PATCH 55/78] install git --- .../cloudbuild-library-generation-integration-test.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c04d40a6dd..60e2fde8f3 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -27,22 +27,24 @@ steps: waitFor: ["-"] env: - "DOCKER_BUILDKIT=1" -- name: alpine/git@sha256:0528136dcb1c3fc461556c5796d6214efa530166cba922c28e9ae146d9acfb82 +- name: ubuntu:24.04 entrypoint: bash args: - "-c" - | + apt-get update && apt-get install -y git cd /workspace git clone https://github.com/googleapis/googleapis cd googleapis git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 id: download-api-definitions waitFor: ["-"] -- name: alpine/git@sha256:0528136dcb1c3fc461556c5796d6214efa530166cba922c28e9ae146d9acfb82 +- name: ubuntu:24.04 entrypoint: bash args: - "-c" - | + apt-get update && apt-get install -y git cd /workspace git clone https://github.com/googleapis/google-cloud-java cd google-cloud-java From 50e12c79672e382064357bb3553ee5f47dc1b753 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 19:14:18 -0500 Subject: [PATCH 56/78] generate libraries --- ...d-library-generation-integration-test.yaml | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 60e2fde8f3..0ee036513c 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -27,6 +27,7 @@ steps: waitFor: ["-"] env: - "DOCKER_BUILDKIT=1" + - name: ubuntu:24.04 entrypoint: bash args: @@ -36,9 +37,10 @@ steps: cd /workspace git clone https://github.com/googleapis/googleapis cd googleapis - git switch 113a378d5aad5018876ec0a8cbfd4d6a4f746809 + git checkout 113a378d5aad5018876ec0a8cbfd4d6a4f746809 id: download-api-definitions waitFor: ["-"] + - name: ubuntu:24.04 entrypoint: bash args: @@ -61,5 +63,57 @@ steps: id: prepare-golden waitFor: ["-"] +- name: ubuntu:24.04 + entrypoint: bash + args: + - "-c" + - | + # Install docker + apt-get update + apt-get install -y ca-certificates curl gnupg lsb-release + mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ + gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-get -y install \ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "/workspace/google-cloud-java:/workspace" \ + -v "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config" \ + -v "/workspace/googleapis:/workspace/apis" \ + "${_TEST_IMAGE}" \ + --generation-config-path="/workspace/config/generation_config.yaml" \ + --api-definitions-path="/workspace/apis" + id: generate-libraries + waitFor: [ + "build-image", + "download-api-definitions", + "prepare-golden" + ] options: logging: CLOUD_LOGGING_ONLY + + + + + + + + + + + + + + + + + + From b5556138c821e46e70eaae48c1a57beaa38322a4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 20:05:03 -0500 Subject: [PATCH 57/78] verify generation --- ...d-library-generation-integration-test.yaml | 31 +- .../scripts/integration_tests_cloud_build.py | 253 ++++++++++++ .../scripts/requirements.in | 1 + .../scripts/requirements.txt | 146 +++++++ .../tests/integration_tests_cloud_build.py | 376 ------------------ 5 files changed, 413 insertions(+), 394 deletions(-) create mode 100644 .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py create mode 100644 .cloudbuild/library_generation/scripts/requirements.in create mode 100644 .cloudbuild/library_generation/scripts/requirements.txt delete mode 100644 hermetic_build/library_generation/tests/integration_tests_cloud_build.py diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 0ee036513c..15296fd53a 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -97,23 +97,18 @@ steps: "download-api-definitions", "prepare-golden" ] + +- name: ubuntu:24.04 + entrypoint: bash + args: + - "-c" + - | + apt-get update && apt-get -y install python3.12-venv + python3 -m venv .venv + source .venv/bin/activate + pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt + python3 -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py + id: verify-generation + waitFor: ["generate-libraries"] options: logging: CLOUD_LOGGING_ONLY - - - - - - - - - - - - - - - - - - diff --git a/.cloudbuild/library_generation/scripts/integration_tests_cloud_build.py b/.cloudbuild/library_generation/scripts/integration_tests_cloud_build.py new file mode 100644 index 0000000000..708ef58651 --- /dev/null +++ b/.cloudbuild/library_generation/scripts/integration_tests_cloud_build.py @@ -0,0 +1,253 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import difflib +import json +import os +import sys +import unittest +import xml.etree.ElementTree as tree +from collections import Counter +from filecmp import dircmp + +script_dir = os.path.dirname(os.path.realpath(__file__)) +# generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") +# This variable is used to override the jar created by building the image +# with our own downloaded jar in order to lock the integration test to use +# a constant version specified in +# library_generation/test/resources/integration/test_generator_coordinates +# This allows us to decouple the generation workflow testing with what the +# generator jar will actually generate. +# See library_generation/DEVELOPMENT.md ("The Hermetic Build's +# well-known folder"). +WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" + + +class CloudBuildIntegrationTest(unittest.TestCase): + + def test_monorepo_generation(self): + repo_dest = "/workspace/google-cloud-java" + golden_dir = "/workspace/golden" + for library_name in [ + "apigee-connect", + "alloydb", + "alloydb-connectors", + "cloudcontrolspartner", + ]: + actual_library = f"{repo_dest}/{library_name}" + print("*" * 50) + print(f"Checking for differences in '{library_name}'.") + print(f" The expected library is in {golden_dir}/{library_name}.") + print(f" The actual library is in {actual_library}.") + compare_result = dircmp( + f"{golden_dir}/{library_name}", + actual_library, + ignore=[".repo-metadata.json"], + ) + diff_files = [] + golden_only = [] + generated_only = [] + # compare source code + self.__recursive_diff_files( + compare_result, diff_files, golden_only, generated_only + ) + + # print all found differences for inspection + def print_file(f: str) -> None: + return print(f" - {f}") + + if len(diff_files) > 0: + print(" Some files (found in both folders) are differing:") + for diff_file in diff_files: + print(f"Difference in {diff_file}:") + with open( + f"{golden_dir}/{library_name}/{diff_file}" + ) as expected_file: + with open(f"{actual_library}/{diff_file}") as actual_file: + [ + print(line) + for line in difflib.unified_diff( + expected_file.readlines(), + actual_file.readlines(), + ) + ] + if len(golden_only) > 0: + print(" There were files found only in the golden dir:") + [print_file(f) for f in golden_only] + if len(generated_only) > 0: + print(" There were files found only in the generated dir:") + [print_file(f) for f in generated_only] + + self.assertTrue(len(golden_only) == 0) + self.assertTrue(len(generated_only) == 0) + self.assertTrue(len(diff_files) == 0) + + print(f" No differences found in {library_name}") + # compare .repo-metadata.json + self.assertTrue( + self.__compare_json_files( + f"{golden_dir}/{library_name}/.repo-metadata.json", + f"{actual_library}/.repo-metadata.json", + ), + msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", + ) + print(" .repo-metadata.json comparison succeed.") + # compare gapic-libraries-bom/pom.xml and pom.xml + self.assertFalse( + self.compare_xml( + f"{golden_dir}/gapic-libraries-bom/pom.xml", + f"{repo_dest}/gapic-libraries-bom/pom.xml", + False, + ) + ) + print(" gapic-libraries-bom/pom.xml comparison succeed.") + self.assertFalse( + self.compare_xml( + f"{golden_dir}/pom.xml", + f"{repo_dest}/pom.xml", + False, + ) + ) + print(" pom.xml comparison succeed.") + + @classmethod + def __compare_json_files(cls, expected: str, actual: str) -> bool: + return cls.__load_json_to_sorted_list( + expected + ) == cls.__load_json_to_sorted_list(actual) + + @classmethod + def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: + with open(path) as f: + data = json.load(f) + res = [(key, value) for key, value in data.items()] + + return sorted(res, key=lambda x: x[0]) + + @classmethod + def __recursive_diff_files( + cls, + dcmp: dircmp, + diff_files: list[str], + left_only: list[str], + right_only: list[str], + dirname: str = "", + ): + """ + Recursively compares two subdirectories. The found differences are + passed to three expected list references. + """ + + def append_dirname(d: str) -> str: + return dirname + d + + diff_files.extend(map(append_dirname, dcmp.diff_files)) + left_only.extend(map(append_dirname, dcmp.left_only)) + right_only.extend(map(append_dirname, dcmp.right_only)) + for sub_dirname, sub_dcmp in dcmp.subdirs.items(): + cls.__recursive_diff_files( + sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" + ) + + @classmethod + def compare_xml(cls, expected, actual, print_trees): + """ + compares two XMLs for content differences + the argument print_whole_trees determines if both trees should be printed + """ + try: + expected_tree = tree.parse(expected) + actual_tree = tree.parse(actual) + except tree.ParseError as e: + cls.eprint(f"Error parsing XML") + raise e + except FileNotFoundError as e: + cls.eprint(f"Error reading file") + raise e + + expected_elements = [] + actual_elements = [] + + cls.append_to_element_list(expected_tree.getroot(), "/", expected_elements) + cls.append_to_element_list(actual_tree.getroot(), "/", actual_elements) + + expected_counter = Counter(expected_elements) + actual_counter = Counter(actual_elements) + intersection = expected_counter & actual_counter + only_in_expected = expected_counter - intersection + only_in_actual = actual_counter - intersection + if print_trees: + cls.eprint("expected") + cls.print_counter(actual_counter) + cls.eprint("actual") + cls.print_counter(expected_counter) + if len(only_in_expected) > 0 or len(only_in_actual) > 0: + cls.eprint("only in " + expected) + cls.print_counter(only_in_expected) + cls.eprint("only in " + actual) + cls.print_counter(only_in_actual) + return True + return False + + @classmethod + def append_to_element_list(cls, node, path, elements): + """ + Recursively traverses a node tree and appends element text to a given + `elements` array. If the element tag is `dependency` + then the maven coordinates for its children will be computed as well + """ + namespace_start, namespace_end, tag_name = node.tag.rpartition("}") + namespace = namespace_start + namespace_end + if tag_name == "dependency": + group_id = cls.get_text_from_element(node, "groupId", namespace) + artifact_id = cls.get_text_from_element(node, "artifactId", namespace) + artifact_str = "" + artifact_str += group_id + artifact_str += ":" + artifact_id + elements.append(path + "/" + tag_name + "=" + artifact_str) + if node.text and len(node.text.strip()) > 0: + elements.append(path + "/" + tag_name + "=" + node.text) + + if tag_name == "version": + # versions may be yet to be processed, we disregard them + return elements + + for child in node: + child_path = path + "/" + tag_name + cls.append_to_element_list(child, child_path, elements) + + return elements + + @classmethod + def get_text_from_element(cls, node, element_name, namespace): + """ + Convenience method to access a node's child elements via path and get + its text. + """ + child = node.find(namespace + element_name) + return child.text if child is not None else "" + + @classmethod + def eprint(cls, *args, **kwargs): + """ + prints to stderr + """ + print(*args, file=sys.stderr, **kwargs) + + @classmethod + def print_counter(cls, counter): + """ + Convenience method to pretty print the contents of a Counter (or dict) + """ + for key, value in counter.items(): + cls.eprint(f"{key}: {value}") diff --git a/.cloudbuild/library_generation/scripts/requirements.in b/.cloudbuild/library_generation/scripts/requirements.in new file mode 100644 index 0000000000..59f0563fba --- /dev/null +++ b/.cloudbuild/library_generation/scripts/requirements.in @@ -0,0 +1 @@ +lxml==5.3.0 \ No newline at end of file diff --git a/.cloudbuild/library_generation/scripts/requirements.txt b/.cloudbuild/library_generation/scripts/requirements.txt new file mode 100644 index 0000000000..d54b3df119 --- /dev/null +++ b/.cloudbuild/library_generation/scripts/requirements.txt @@ -0,0 +1,146 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes .cloudbuild/library_generation/scripts/requirements.in +# +lxml==5.3.0 \ + --hash=sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e \ + --hash=sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229 \ + --hash=sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3 \ + --hash=sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5 \ + --hash=sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70 \ + --hash=sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15 \ + --hash=sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002 \ + --hash=sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd \ + --hash=sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22 \ + --hash=sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf \ + --hash=sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22 \ + --hash=sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832 \ + --hash=sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727 \ + --hash=sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e \ + --hash=sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30 \ + --hash=sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f \ + --hash=sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f \ + --hash=sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51 \ + --hash=sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4 \ + --hash=sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de \ + --hash=sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875 \ + --hash=sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42 \ + --hash=sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e \ + --hash=sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6 \ + --hash=sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391 \ + --hash=sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc \ + --hash=sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b \ + --hash=sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237 \ + --hash=sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4 \ + --hash=sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86 \ + --hash=sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f \ + --hash=sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a \ + --hash=sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8 \ + --hash=sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f \ + --hash=sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903 \ + --hash=sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03 \ + --hash=sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e \ + --hash=sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99 \ + --hash=sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7 \ + --hash=sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab \ + --hash=sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d \ + --hash=sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22 \ + --hash=sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492 \ + --hash=sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b \ + --hash=sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3 \ + --hash=sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be \ + --hash=sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469 \ + --hash=sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f \ + --hash=sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a \ + --hash=sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c \ + --hash=sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a \ + --hash=sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4 \ + --hash=sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94 \ + --hash=sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442 \ + --hash=sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b \ + --hash=sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84 \ + --hash=sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c \ + --hash=sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9 \ + --hash=sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1 \ + --hash=sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be \ + --hash=sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367 \ + --hash=sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e \ + --hash=sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21 \ + --hash=sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa \ + --hash=sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16 \ + --hash=sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d \ + --hash=sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe \ + --hash=sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83 \ + --hash=sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba \ + --hash=sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040 \ + --hash=sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763 \ + --hash=sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8 \ + --hash=sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff \ + --hash=sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2 \ + --hash=sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a \ + --hash=sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b \ + --hash=sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce \ + --hash=sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c \ + --hash=sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577 \ + --hash=sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8 \ + --hash=sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71 \ + --hash=sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512 \ + --hash=sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540 \ + --hash=sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f \ + --hash=sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2 \ + --hash=sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a \ + --hash=sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce \ + --hash=sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e \ + --hash=sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2 \ + --hash=sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27 \ + --hash=sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1 \ + --hash=sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d \ + --hash=sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1 \ + --hash=sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330 \ + --hash=sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920 \ + --hash=sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99 \ + --hash=sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff \ + --hash=sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18 \ + --hash=sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff \ + --hash=sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c \ + --hash=sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179 \ + --hash=sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080 \ + --hash=sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19 \ + --hash=sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d \ + --hash=sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70 \ + --hash=sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32 \ + --hash=sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a \ + --hash=sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2 \ + --hash=sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79 \ + --hash=sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3 \ + --hash=sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5 \ + --hash=sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f \ + --hash=sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d \ + --hash=sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3 \ + --hash=sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b \ + --hash=sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753 \ + --hash=sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9 \ + --hash=sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957 \ + --hash=sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033 \ + --hash=sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb \ + --hash=sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656 \ + --hash=sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab \ + --hash=sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b \ + --hash=sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d \ + --hash=sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd \ + --hash=sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859 \ + --hash=sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11 \ + --hash=sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c \ + --hash=sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a \ + --hash=sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005 \ + --hash=sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654 \ + --hash=sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80 \ + --hash=sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e \ + --hash=sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec \ + --hash=sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7 \ + --hash=sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965 \ + --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \ + --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8 + # via -r .cloudbuild/library_generation/scripts/requirements.in diff --git a/hermetic_build/library_generation/tests/integration_tests_cloud_build.py b/hermetic_build/library_generation/tests/integration_tests_cloud_build.py deleted file mode 100644 index bf69596911..0000000000 --- a/hermetic_build/library_generation/tests/integration_tests_cloud_build.py +++ /dev/null @@ -1,376 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import difflib -import json -import tempfile -from filecmp import dircmp -from git import Repo -import os -import shutil -import subprocess -import unittest -from pathlib import Path -from common.model.generation_config import GenerationConfig -from common.model.generation_config import from_yaml -from library_generation.tests.compare_poms import compare_xml -from library_generation.utils.utilities import sh_util as shell_call - - -script_dir = os.path.dirname(os.path.realpath(__file__)) -config_dir = os.path.join(script_dir, "resources", "integration") -golden_dir = os.path.join(config_dir, "golden") -generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") -repo_root_dir = os.path.join(script_dir, "..", "..", "..") -build_file = os.path.join( - repo_root_dir, - ".cloudbuild", - "library_generation", - "library_generation_airlock.Dockerfile", -) -image_tag = "test-image:latest" -repo_prefix = "https://github.com/googleapis" -output_dir = shell_call("get_output_folder") -# this map tells which branch of each repo should we use for our diff tests -commitish_map = { - "google-cloud-java": "chore/test-hermetic-build", - "java-bigtable": "chore/test-hermetic-build", -} -generation_config_name = "generation_config.yaml" -googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" -# This variable is used to override the jar created by building the image -# with our own downloaded jar in order to lock the integration test to use -# a constant version specified in -# library_generation/test/resources/integration/test_generator_coordinates -# This allows us to decouple the generation workflow testing with what the -# generator jar will actually generate. -# See library_generation/DEVELOPMENT.md ("The Hermetic Build's -# well-known folder"). -WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" - - -class IntegrationTest(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) - cls.__build_image(docker_file=build_file, cwd=repo_root_dir) - - @classmethod - def setUp(cls) -> None: - cls.__remove_generated_files() - os.makedirs(f"{golden_dir}", exist_ok=True) - - def test_entry_point_running_in_container(self): - api_definitions_path = self.__copy_api_definition(googleapis_commitish) - print(f"api definition path in host: {api_definitions_path}") - print(f"Contents in host: {os.listdir(api_definitions_path)}") - config_files = self.__get_config_files(config_dir) - for repo, config_file in config_files: - config = from_yaml(config_file) - repo_location = f"{output_dir}/{repo}" - config_location = f"{golden_dir}/../{repo}" - # 1. pull repository - repo_dest = self.__pull_repo_to( - Path(repo_location), repo, commitish_map[repo] - ) - # 2. prepare golden files - library_names = self.__get_library_names_from_config(config) - self.__prepare_golden_files( - config=config, library_names=library_names, repo_dest=repo_dest - ) - # 3. run entry_point.py in docker container - self.__run_entry_point_in_docker_container( - repo_location=repo_location, - config_location=config_location, - generation_config=generation_config_name, - api_definition=api_definitions_path, - ) - # 4. compare generation result with golden files - print( - "Generation finished successfully. " - "Will now compare differences between generated and existing " - "libraries" - ) - for library_name in library_names: - actual_library = ( - f"{repo_dest}/{library_name}" if config.is_monorepo() else repo_dest - ) - print("*" * 50) - print(f"Checking for differences in '{library_name}'.") - print(f" The expected library is in {golden_dir}/{library_name}.") - print(f" The actual library is in {actual_library}. ") - compare_result = dircmp( - f"{golden_dir}/{library_name}", - actual_library, - ignore=[".repo-metadata.json"], - ) - diff_files = [] - golden_only = [] - generated_only = [] - # compare source code - self.__recursive_diff_files( - compare_result, diff_files, golden_only, generated_only - ) - - # print all found differences for inspection - print_file = lambda f: print(f" - {f}") - if len(diff_files) > 0: - print(" Some files (found in both folders) are differing:") - for diff_file in diff_files: - print(f"Difference in {diff_file}:") - with open( - f"{golden_dir}/{library_name}/{diff_file}" - ) as expected_file: - with open(f"{actual_library}/{diff_file}") as actual_file: - [ - print(line) - for line in difflib.unified_diff( - expected_file.readlines(), - actual_file.readlines(), - ) - ] - if len(golden_only) > 0: - print(" There were files found only in the golden dir:") - [print_file(f) for f in golden_only] - if len(generated_only) > 0: - print(" There were files found only in the generated dir:") - [print_file(f) for f in generated_only] - - self.assertTrue(len(golden_only) == 0) - self.assertTrue(len(generated_only) == 0) - self.assertTrue(len(diff_files) == 0) - - print(f" No differences found in {library_name}") - # compare .repo-metadata.json - self.assertTrue( - self.__compare_json_files( - f"{golden_dir}/{library_name}/.repo-metadata.json", - f"{actual_library}/.repo-metadata.json", - ), - msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", - ) - print(" .repo-metadata.json comparison succeed.") - - if not config.is_monorepo(): - continue - - # compare gapic-libraries-bom/pom.xml and pom.xml - self.assertFalse( - compare_xml( - f"{golden_dir}/gapic-libraries-bom/pom.xml", - f"{repo_dest}/gapic-libraries-bom/pom.xml", - False, - ) - ) - print(" gapic-libraries-bom/pom.xml comparison succeed.") - self.assertFalse( - compare_xml( - f"{golden_dir}/pom.xml", - f"{repo_dest}/pom.xml", - False, - ) - ) - print(" pom.xml comparison succeed.") - self.__remove_generated_files() - shutil.rmtree(api_definitions_path) - - @classmethod - def __copy_api_definition(cls, committish: str) -> str: - repo_dest = cls.__pull_repo_to( - dest=tempfile.mkdtemp(), repo="googleapis", committish=committish - ) - api_temp_dir = tempfile.mkdtemp() - print(f"Copying api definition to {api_temp_dir}...") - shutil.copytree( - f"{repo_dest}/google", f"{api_temp_dir}/google", dirs_exist_ok=True - ) - shutil.copytree( - f"{repo_dest}/grafeas", f"{api_temp_dir}/grafeas", dirs_exist_ok=True - ) - shutil.rmtree(repo_dest) - return api_temp_dir - - @classmethod - def __build_image(cls, docker_file: str, cwd: str): - # we build the docker image without removing intermediate containers, so - # we can re-test more quickly - subprocess.check_call( - ["docker", "build", "-f", docker_file, "-t", image_tag, "."], - cwd=cwd, - env=dict(os.environ, DOCKER_BUILDKIT="1"), - ) - - @classmethod - def __download_generator_jar(cls, coordinates_file: str) -> None: - """ - Downloads the jar at the version specified in the - coordinates file - :param coordinates_file: path to the file containing the coordinates - """ - with open(coordinates_file, "r") as coordinates_file_handle: - # make this var available in the function scope - # nonlocal coordinates - coordinates = coordinates_file_handle.read() - # download the jar - subprocess.check_call( - [ - "mvn", - "dependency:copy", - "-B", - "-ntp", - f"-Dartifact={coordinates}", - f"-DoutputDirectory={config_dir}", - ] - ) - - # compute the filename of the downloaded jar - split_coordinates = coordinates.split(":") - artifact_id = split_coordinates[1] - version = split_coordinates[2] - jar_filename = f"{artifact_id}-{version}.jar" - - # rename the jar to its well-known filename defined at the top of this - # script file - source_jar_path = os.path.join(config_dir, jar_filename) - destination_jar_path = os.path.join( - config_dir, WELL_KNOWN_GENERATOR_JAR_FILENAME - ) - shutil.move(source_jar_path, destination_jar_path) - - @classmethod - def __remove_generated_files(cls): - shutil.rmtree(f"{output_dir}", ignore_errors=True) - if os.path.isdir(f"{golden_dir}"): - shutil.rmtree(f"{golden_dir}") - - @classmethod - def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: - shutil.rmtree(dest, ignore_errors=True) - repo_url = f"{repo_prefix}/{repo}" - print(f"Cloning repository {repo_url}") - repo = Repo.clone_from(repo_url, dest) - repo.git.checkout(committish) - return str(dest) - - @classmethod - def __get_library_names_from_config(cls, config: GenerationConfig) -> list[str]: - library_names = [] - for library in config.libraries: - library_names.append(f"java-{library.get_library_name()}") - - return library_names - - @classmethod - def __prepare_golden_files( - cls, config: GenerationConfig, library_names: list[str], repo_dest: str - ): - for library_name in library_names: - if config.is_monorepo(): - shutil.copytree( - f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}" - ) - shutil.copytree( - f"{repo_dest}/gapic-libraries-bom", - f"{golden_dir}/gapic-libraries-bom", - dirs_exist_ok=True, - ) - shutil.copyfile(f"{repo_dest}/pom.xml", f"{golden_dir}/pom.xml") - else: - shutil.copytree(f"{repo_dest}", f"{golden_dir}/{library_name}") - - @classmethod - def __run_entry_point_in_docker_container( - cls, - repo_location: str, - config_location: str, - generation_config: str, - api_definition: str, - ): - # we use the calling user to prevent the mapped volumes from changing - # owners - user_id = shell_call("id -u") - group_id = shell_call("id -g") - subprocess.check_call( - [ - "docker", - "run", - "-u", - f"{user_id}:{group_id}", - "--rm", - "--quiet", - "--network", - "none", - "-v", - f"{repo_location}:/workspace/repo", - "-v", - f"{config_location}:/workspace/config", - "-v", - f"{api_definition}:/workspace/api", - "-v", - f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", - "-w", - "/workspace/repo", - image_tag, - f"--generation-config-path=/workspace/config/{generation_config}", - f"--api-definitions-path=/workspace/api", - ], - ) - - @classmethod - def __get_config_files(cls, path: str) -> list[tuple[str, str]]: - config_files = [] - for sub_dir in Path(path).resolve().iterdir(): - if sub_dir.is_file(): - continue - repo = sub_dir.name - if repo in ["golden", "java-bigtable"]: - continue - config = f"{sub_dir}/{generation_config_name}" - config_files.append((repo, config)) - return config_files - - @classmethod - def __compare_json_files(cls, expected: str, actual: str) -> bool: - return cls.__load_json_to_sorted_list( - expected - ) == cls.__load_json_to_sorted_list(actual) - - @classmethod - def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: - with open(path) as f: - data = json.load(f) - res = [(key, value) for key, value in data.items()] - - return sorted(res, key=lambda x: x[0]) - - @classmethod - def __recursive_diff_files( - cls, - dcmp: dircmp, - diff_files: list[str], - left_only: list[str], - right_only: list[str], - dirname: str = "", - ): - """ - Recursively compares two subdirectories. The found differences are - passed to three expected list references. - """ - append_dirname = lambda d: dirname + d - diff_files.extend(map(append_dirname, dcmp.diff_files)) - left_only.extend(map(append_dirname, dcmp.left_only)) - right_only.extend(map(append_dirname, dcmp.right_only)) - for sub_dirname, sub_dcmp in dcmp.subdirs.items(): - cls.__recursive_diff_files( - sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" - ) From b72bfdb5fe0771a59cd3afb8017e87ea235c7688 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 20:31:59 -0500 Subject: [PATCH 58/78] change to python image --- .../cloudbuild-library-generation-integration-test.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 15296fd53a..375fb319ee 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -98,14 +98,11 @@ steps: "prepare-golden" ] -- name: ubuntu:24.04 - entrypoint: bash +- name: python:3.13.1-alpine3.21 + entrypoint: python args: - - "-c" + - "-m" - | - apt-get update && apt-get -y install python3.12-venv - python3 -m venv .venv - source .venv/bin/activate pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt python3 -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py id: verify-generation From 65334f073bc339477c8786942fb1876c2d8b8859 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 20:32:19 -0500 Subject: [PATCH 59/78] change to python image --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 375fb319ee..b27bdb4908 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -104,7 +104,7 @@ steps: - "-m" - | pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt - python3 -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py + python -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py id: verify-generation waitFor: ["generate-libraries"] options: From 63d597a0a0d7c46d4ea9b5a7c8987938f1750f09 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 21:05:39 -0500 Subject: [PATCH 60/78] use ubuntu --- .../cloudbuild-library-generation-integration-test.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index b27bdb4908..88512a4340 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -98,11 +98,14 @@ steps: "prepare-golden" ] -- name: python:3.13.1-alpine3.21 - entrypoint: python +- name: ubuntu:24.04 + entrypoint: bash args: - - "-m" + - "-c" - | + apt-get update && apt-get install -y python3 python3.12-venv + python3 -m venv .venv + source .venv/bin/activate pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt python -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py id: verify-generation From 8deab901f0ccd0e24e2d721d610a94d67ccd200b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 21:15:56 -0500 Subject: [PATCH 61/78] use apt --- ...dbuild-library-generation-integration-test.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 88512a4340..8135d9c346 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -33,7 +33,7 @@ steps: args: - "-c" - | - apt-get update && apt-get install -y git + apt update && apt install -y git cd /workspace git clone https://github.com/googleapis/googleapis cd googleapis @@ -46,7 +46,7 @@ steps: args: - "-c" - | - apt-get update && apt-get install -y git + apt update && apt install -y git cd /workspace git clone https://github.com/googleapis/google-cloud-java cd google-cloud-java @@ -69,8 +69,8 @@ steps: - "-c" - | # Install docker - apt-get update - apt-get install -y ca-certificates curl gnupg lsb-release + apt update + apt install -y ca-certificates curl gnupg lsb-release mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ gpg --dearmor -o /etc/apt/keyrings/docker.gpg @@ -78,8 +78,8 @@ steps: "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null - apt-get update - apt-get -y install \ + apt update + apt -y install \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker run \ --rm \ @@ -103,7 +103,7 @@ steps: args: - "-c" - | - apt-get update && apt-get install -y python3 python3.12-venv + apt update && apt install -y python3 python3.12-venv python3 -m venv .venv source .venv/bin/activate pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt From ba98d928bdd4f1e0658332a0ba3cc34d7996b508 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 18 Dec 2024 21:33:35 -0500 Subject: [PATCH 62/78] rename --- .../cloudbuild-library-generation-integration-test.yaml | 2 +- .../{integration_tests_cloud_build.py => integration_tests.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .cloudbuild/library_generation/scripts/{integration_tests_cloud_build.py => integration_tests.py} (100%) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 8135d9c346..8994950a45 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -107,7 +107,7 @@ steps: python3 -m venv .venv source .venv/bin/activate pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt - python -m unittest .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py + python -m unittest .cloudbuild/library_generation/scripts/integration_tests.py id: verify-generation waitFor: ["generate-libraries"] options: diff --git a/.cloudbuild/library_generation/scripts/integration_tests_cloud_build.py b/.cloudbuild/library_generation/scripts/integration_tests.py similarity index 100% rename from .cloudbuild/library_generation/scripts/integration_tests_cloud_build.py rename to .cloudbuild/library_generation/scripts/integration_tests.py From dbdfae7ae0e30825582b9c865c81bf18c592c697 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 07:51:50 -0500 Subject: [PATCH 63/78] change class name --- .cloudbuild/library_generation/scripts/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/scripts/integration_tests.py b/.cloudbuild/library_generation/scripts/integration_tests.py index 708ef58651..881f83a47d 100644 --- a/.cloudbuild/library_generation/scripts/integration_tests.py +++ b/.cloudbuild/library_generation/scripts/integration_tests.py @@ -33,7 +33,7 @@ WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" -class CloudBuildIntegrationTest(unittest.TestCase): +class IntegrationTest(unittest.TestCase): def test_monorepo_generation(self): repo_dest = "/workspace/google-cloud-java" From abd07eb282418b7e11518c04902116b5a2e607b4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 08:01:43 -0500 Subject: [PATCH 64/78] use docker image --- ...d-library-generation-integration-test.yaml | 76 ++++++++++++------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 8994950a45..b1c4d3e610 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -63,34 +63,54 @@ steps: id: prepare-golden waitFor: ["-"] -- name: ubuntu:24.04 - entrypoint: bash - args: - - "-c" - - | - # Install docker - apt update - apt install -y ca-certificates curl gnupg lsb-release - mkdir -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ - gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - apt update - apt -y install \ - docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - docker run \ - --rm \ - --quiet \ - -u "$(id -u):$(id -g)" \ - -v "/workspace/google-cloud-java:/workspace" \ - -v "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config" \ - -v "/workspace/googleapis:/workspace/apis" \ - "${_TEST_IMAGE}" \ - --generation-config-path="/workspace/config/generation_config.yaml" \ - --api-definitions-path="/workspace/apis" +#- name: ubuntu:24.04 +# entrypoint: bash +# args: +# - "-c" +# - | +# # Install docker +# apt update +# apt install -y ca-certificates curl gnupg lsb-release +# mkdir -p /etc/apt/keyrings +# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ +# gpg --dearmor -o /etc/apt/keyrings/docker.gpg +# echo \ +# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ +# $(lsb_release -cs) stable" | \ +# tee /etc/apt/sources.list.d/docker.list > /dev/null +# apt update +# apt -y install \ +# docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +# docker run \ +# --rm \ +# --quiet \ +# -u "$(id -u):$(id -g)" \ +# -v "/workspace/google-cloud-java:/workspace" \ +# -v "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config" \ +# -v "/workspace/googleapis:/workspace/apis" \ +# "${_TEST_IMAGE}" \ +# --generation-config-path="/workspace/config/generation_config.yaml" \ +# --api-definitions-path="/workspace/apis" +# id: generate-libraries +# waitFor: [ +# "build-image", +# "download-api-definitions", +# "prepare-golden" +# ] + +- name: gcr.io/cloud-builders/docker + args: [ + "run", + "--rm", + "-v", "/workspace/google-cloud-java:/workspace", + "-v", "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config", + "-v", "/workspace/googleapis:/workspace/apis", + "${_TEST_IMAGE}", + "--generation-config-path=/workspace/config/generation_config.yaml", + "--api-definitions-path=/workspace/apis" + ] + env: + - "DOCKER_BUILDKIT=1" id: generate-libraries waitFor: [ "build-image", From 965e201ee2361239177b157fcb5a72757c148c07 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 08:13:31 -0500 Subject: [PATCH 65/78] add module --- .cloudbuild/library_generation/scripts/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .cloudbuild/library_generation/scripts/__init__.py diff --git a/.cloudbuild/library_generation/scripts/__init__.py b/.cloudbuild/library_generation/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From b95e02a9692a2a71bc29fb5c81704d2772221fd6 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 08:28:46 -0500 Subject: [PATCH 66/78] add another module --- .cloudbuild/library_generation/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .cloudbuild/library_generation/__init__.py diff --git a/.cloudbuild/library_generation/__init__.py b/.cloudbuild/library_generation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 5cb7817751d2de77aacf8a09970333db4fda87bf Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 09:42:10 -0500 Subject: [PATCH 67/78] change dir --- .../cloudbuild-library-generation-integration-test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index b1c4d3e610..e2f64715f8 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -126,8 +126,9 @@ steps: apt update && apt install -y python3 python3.12-venv python3 -m venv .venv source .venv/bin/activate - pip install --require-hashes -r .cloudbuild/library_generation/scripts/requirements.txt - python -m unittest .cloudbuild/library_generation/scripts/integration_tests.py + pip install --require-hashes -r requirements.txt + python -m unittest integration_tests.py + dir: ".cloudbuild/library_generation/scripts" id: verify-generation waitFor: ["generate-libraries"] options: From 4e34ae272381f388f3d1d020a8bdf49b58574947 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 10:08:47 -0500 Subject: [PATCH 68/78] add java --- .../library_generation/scripts/integration_tests.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.cloudbuild/library_generation/scripts/integration_tests.py b/.cloudbuild/library_generation/scripts/integration_tests.py index 881f83a47d..cb3166e58b 100644 --- a/.cloudbuild/library_generation/scripts/integration_tests.py +++ b/.cloudbuild/library_generation/scripts/integration_tests.py @@ -38,12 +38,13 @@ class IntegrationTest(unittest.TestCase): def test_monorepo_generation(self): repo_dest = "/workspace/google-cloud-java" golden_dir = "/workspace/golden" - for library_name in [ - "apigee-connect", - "alloydb", - "alloydb-connectors", - "cloudcontrolspartner", - ]: + library_names = [ + "java-apigee-connect", + "java-alloydb", + "java-alloydb-connectors", + "java-cloudcontrolspartner", + ] + for library_name in library_names: actual_library = f"{repo_dest}/{library_name}" print("*" * 50) print(f"Checking for differences in '{library_name}'.") From 3e04119cfe073cb1da7ef93424dd4337d07516fd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 10:12:54 -0500 Subject: [PATCH 69/78] restore files --- hermetic_build/library_generation/generate_repo.py | 4 ---- hermetic_build/library_generation/tests/integration_tests.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 0cb6e721ff..0692f2bcf6 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -48,12 +48,8 @@ def generate_from_yaml( repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) - print(f"api definition in docker: {api_definitions_path}") - print(f"Contents in api definition: {os.listdir(api_definitions_path)}") # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - print(f"output folder: {repo_config.output_folder}") - print(f"Contents in output: {os.listdir(repo_config.output_folder)}") for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") generate_composed_library( diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 8c5ee67b19..f738a92093 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -69,8 +69,6 @@ def setUp(cls) -> None: def test_entry_point_running_in_container(self): api_definitions_path = self.__copy_api_definition(googleapis_commitish) - print(f"api definition path in host: {api_definitions_path}") - print(f"Contents in host: {os.listdir(api_definitions_path)}") config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: config = from_yaml(config_file) From 3272b148f6cf52d8a0ccd8d3150ec9912b6a98a0 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 10:13:56 -0500 Subject: [PATCH 70/78] restore files --- hermetic_build/library_generation/generate_repo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 0692f2bcf6..b97e5ca48f 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -12,13 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import shutil from typing import Optional import library_generation.utils.utilities as util -from library_generation.generate_composed_library import generate_composed_library from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig +from library_generation.generate_composed_library import generate_composed_library from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing From d7ae12a41c43bb127afc4924837fc8bedbf3d775 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 10:47:46 -0500 Subject: [PATCH 71/78] fix generator jar --- ...d-library-generation-integration-test.yaml | 54 +++++++------------ 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index e2f64715f8..fbd63988dc 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -63,40 +63,20 @@ steps: id: prepare-golden waitFor: ["-"] -#- name: ubuntu:24.04 -# entrypoint: bash -# args: -# - "-c" -# - | -# # Install docker -# apt update -# apt install -y ca-certificates curl gnupg lsb-release -# mkdir -p /etc/apt/keyrings -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ -# gpg --dearmor -o /etc/apt/keyrings/docker.gpg -# echo \ -# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ -# $(lsb_release -cs) stable" | \ -# tee /etc/apt/sources.list.d/docker.list > /dev/null -# apt update -# apt -y install \ -# docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -# docker run \ -# --rm \ -# --quiet \ -# -u "$(id -u):$(id -g)" \ -# -v "/workspace/google-cloud-java:/workspace" \ -# -v "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config" \ -# -v "/workspace/googleapis:/workspace/apis" \ -# "${_TEST_IMAGE}" \ -# --generation-config-path="/workspace/config/generation_config.yaml" \ -# --api-definitions-path="/workspace/apis" -# id: generate-libraries -# waitFor: [ -# "build-image", -# "download-api-definitions", -# "prepare-golden" -# ] +- name: ubuntu:24.04 + entrypoint: bash + args: + - "-c" + - | + apt update && apt install -y maven + mvn dependency:copy \ + -B -ntp \ + -Dartifact=com.google.api:gapic-generator-java:2.38.1 \ + -DoutputDirectory=/workspace + cd /workspace + mv gapic-generator-java-2.38.1.jar gapic-generator-java.jar + id: prepare-generator-jar + waitFor: [ "-" ] - name: gcr.io/cloud-builders/docker args: [ @@ -105,6 +85,9 @@ steps: "-v", "/workspace/google-cloud-java:/workspace", "-v", "/workspace/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config", "-v", "/workspace/googleapis:/workspace/apis", + # Fix gapic-generator-java so that the generation result stays + # the same. + "-v", "/workspace/gapic-generator-java.jar:/home/.library_generation/gapic-generator-java.jar", "${_TEST_IMAGE}", "--generation-config-path=/workspace/config/generation_config.yaml", "--api-definitions-path=/workspace/apis" @@ -115,7 +98,8 @@ steps: waitFor: [ "build-image", "download-api-definitions", - "prepare-golden" + "prepare-golden", + "prepare-generator-jar" ] - name: ubuntu:24.04 From c4bab31324f371b1451873f912799ae74f8bbd0b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:01:45 -0500 Subject: [PATCH 72/78] use e2_highcpu_8 --- .../cloudbuild-library-generation-integration-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index fbd63988dc..ab98b6f3e9 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -117,3 +117,4 @@ steps: waitFor: ["generate-libraries"] options: logging: CLOUD_LOGGING_ONLY + machineType: E2_HIGHCPU_8 From e1e65c12a836536d7c738e6fd7d87eaa8309d980 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:03:36 -0500 Subject: [PATCH 73/78] delete module --- .cloudbuild/library_generation/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .cloudbuild/library_generation/__init__.py diff --git a/.cloudbuild/library_generation/__init__.py b/.cloudbuild/library_generation/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From e3b60f817b7e39db668d1ff05f465f90cb94508b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:13:17 -0500 Subject: [PATCH 74/78] change image --- .../cloudbuild-library-generation-integration-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index ab98b6f3e9..f115fc61e9 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -63,12 +63,12 @@ steps: id: prepare-golden waitFor: ["-"] -- name: ubuntu:24.04 - entrypoint: bash +- name: maven:3.9.9-eclipse-temurin-11-alpine + entrypoint: /bin/sh args: - "-c" - | - apt update && apt install -y maven +# apt update && apt install -y maven mvn dependency:copy \ -B -ntp \ -Dartifact=com.google.api:gapic-generator-java:2.38.1 \ From 75dfc192c5a4a892cc87b8ec25682312f53aab25 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:14:03 -0500 Subject: [PATCH 75/78] change image --- .../cloudbuild-library-generation-integration-test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index f115fc61e9..83b801489e 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -68,7 +68,6 @@ steps: args: - "-c" - | -# apt update && apt install -y maven mvn dependency:copy \ -B -ntp \ -Dartifact=com.google.api:gapic-generator-java:2.38.1 \ From 2e1e8aba762e84f34ec3eed1346822e7bf8b3c9c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:17:39 -0500 Subject: [PATCH 76/78] usage python image --- .../cloudbuild-library-generation-integration-test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 83b801489e..c50c85fdb4 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -101,12 +101,11 @@ steps: "prepare-generator-jar" ] -- name: ubuntu:24.04 - entrypoint: bash +- name: python:3.12.7-alpine3.20 + entrypoint: /bin/sh args: - "-c" - | - apt update && apt install -y python3 python3.12-venv python3 -m venv .venv source .venv/bin/activate pip install --require-hashes -r requirements.txt From 9e5229a48563e64de4d6241b980bd881e35e4ede Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:18:58 -0500 Subject: [PATCH 77/78] usage git image --- ...cloudbuild-library-generation-integration-test.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index c50c85fdb4..49469e56b5 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -28,12 +28,11 @@ steps: env: - "DOCKER_BUILDKIT=1" -- name: ubuntu:24.04 - entrypoint: bash +- name: alpine/git:latest + entrypoint: /bin/sh args: - "-c" - | - apt update && apt install -y git cd /workspace git clone https://github.com/googleapis/googleapis cd googleapis @@ -41,12 +40,11 @@ steps: id: download-api-definitions waitFor: ["-"] -- name: ubuntu:24.04 - entrypoint: bash +- name: alpine/git:latest + entrypoint: /bin/sh args: - "-c" - | - apt update && apt install -y git cd /workspace git clone https://github.com/googleapis/google-cloud-java cd google-cloud-java From c91d264bb8c97ca0f4cf2a896d271095a42a45ef Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 19 Dec 2024 11:22:21 -0500 Subject: [PATCH 78/78] rm unused var --- .../library_generation/scripts/integration_tests.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.cloudbuild/library_generation/scripts/integration_tests.py b/.cloudbuild/library_generation/scripts/integration_tests.py index cb3166e58b..14f77725d7 100644 --- a/.cloudbuild/library_generation/scripts/integration_tests.py +++ b/.cloudbuild/library_generation/scripts/integration_tests.py @@ -21,16 +21,6 @@ from filecmp import dircmp script_dir = os.path.dirname(os.path.realpath(__file__)) -# generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") -# This variable is used to override the jar created by building the image -# with our own downloaded jar in order to lock the integration test to use -# a constant version specified in -# library_generation/test/resources/integration/test_generator_coordinates -# This allows us to decouple the generation workflow testing with what the -# generator jar will actually generate. -# See library_generation/DEVELOPMENT.md ("The Hermetic Build's -# well-known folder"). -WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" class IntegrationTest(unittest.TestCase):