Skip to content

Commit

Permalink
chore: bake protoc into the hermetic build docker image (#2707)
Browse files Browse the repository at this point in the history 
### Does the config work with renovate bot?
Yes. After a local run using `LOG_LEVEL=debug npx --yes --package
renovate -- renovate --platform=local &> renovate.out`, we obtained a
dependency update in the logs:
```
           {
             "deps": [
               {     
                 "depName": "protocolbuffers/protobuf",
                 "currentValue": "25.2",
                 "datasource": "github-releases",
                 "extractVersion": "^v(?<version>.*)$",
                 "replaceString": "ARG PROTOC_VERSION=25.2\n",
                 "updates": [
                   {     
                     "bucket": "non-major",
                     "newVersion": "25.3",
                     "newValue": "25.3",
                     "releaseTimestamp": "2024-02-15T23:20:43.000Z",
                     "newMajor": 25,
                     "newMinor": 3,
                     "updateType": "minor",
                     "branchName": "renovate/protocolbuffers-protobuf-25.x"
                   },    
                   {     
                     "bucket": "major",
                     "newVersion": "26.1",
                     "newValue": "26.1",
                     "releaseTimestamp": "2024-03-27T20:28:47.000Z",
                     "newMajor": 26,
                     "newMinor": 1,
                     "updateType": "major",
                     "branchName": "renovate/protocolbuffers-protobuf-26.x"
                   }
                 ],
                 "packageName": "protocolbuffers/protobuf",
                 "versioning": "semver-coerced",
                 "warnings": [],
                 "sourceUrl": "https://github.com/protocolbuffers/protobuf",
                 "registryUrl": "https://github.com",
                 "currentVersion": "25.2",
                 "currentVersionTimestamp": "2024-01-09T23:52:55.000Z",
                 "isSingleVersion": true, 
                 "fixedVersion": "25.2"
               }
             ],

```

However, we disabled the setting for now. This log entry confirms it.
`DEBUG: Dependency: protocolbuffers/protobuf, is disabled
(repository=local)`

---------

Co-authored-by: Joe Wang <[email protected]>
  • Loading branch information
@diegomarquezp @JoeWang1127
diegomarquezp and JoeWang1127 authored May 23, 2024
1 parent dfedafc commit cf44b07
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 20 deletions.
19 changes: 15 additions & 4 deletions .cloudbuild/library_generation/library_generation.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,35 @@
# build from the root of this repo:
FROM gcr.io/cloud-devrel-public-resources/python

ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834
SHELL [ "/bin/bash", "-c" ]

ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940
ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
ARG PROTOC_VERSION=25.3
ENV HOME=/home

# install OS tools
RUN apt-get update && apt-get install -y \
unzip openjdk-17-jdk rsync maven jq \
&& apt-get clean

# copy source code
COPY library_generation /src

# install protoc
WORKDIR /protoc
RUN source /src/utils/utilities.sh \
&& download_protoc "${PROTOC_VERSION}" "linux-x86_64"
# we indicate protoc is available in the container via env vars
ENV DOCKER_PROTOC_LOCATION=/protoc
ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"

# use python 3.11 (the base image has several python versions; here we define the default one)
RUN rm $(which python3)
RUN ln -s $(which python3.11) /usr/local/bin/python
RUN ln -s $(which python3.11) /usr/local/bin/python3
RUN python -m pip install --upgrade pip

# copy source code
COPY library_generation /src

# install scripts as a python package
WORKDIR /src
RUN python -m pip install -r requirements.txt
Expand Down
39 changes: 39 additions & 0 deletions library_generation/test/generate_library_unit_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ get_grpc_version_failed_with_invalid_generator_version_test() {
assertEquals 1 $((res))
}

get_protoc_version_succeed_docker_env_var_test() {
local version_with_docker
local version_without_docker
export DOCKER_PROTOC_VERSION="9.9.9"
version_with_docker=$(get_protoc_version "2.24.0")
assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}"
unset DOCKER_PROTOC_VERSION
}

get_protoc_version_succeed_with_valid_generator_version_test() {
local actual_version
actual_version=$(get_protoc_version "2.24.0")
Expand Down Expand Up @@ -134,6 +143,34 @@ download_protoc_failed_with_invalid_arch_test() {
assertEquals 1 $((res))
}

download_tools_succeed_with_baked_protoc() {
# This mimics a docker container scenario.
# This test consists of creating an empty /tmp/.../protoc-99.99/bin folder and map
# it to the DOCKER_PROTOC_LOCATION env var (which is treated specially in the
# `download_tools` function). If `DOCKER_PROTOC_VERSION` matches exactly as
# the version passed to `download_protoc`, then we will not download protoc
# but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION
# (which we manually created in this test)
local test_dir=$(mktemp -d)
pushd "${test_dir}"
export DOCKER_PROTOC_LOCATION=$(mktemp -d)
export DOCKER_PROTOC_VERSION="99.99"
export output_folder=$(get_output_folder)
mkdir "${output_folder}"
local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin"
mkdir -p "${protoc_bin_folder}"

local test_ggj_version="2.40.0"
local test_grpc_version="1.64.0"
download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64"
assertEquals "${protoc_bin_folder}" "${protoc_path}"

rm -rdf "${output_folder}"
unset DOCKER_PROTOC_LOCATION
unset DOCKER_PROTOC_VERSION
unset output_folder
}

download_grpc_plugin_succeed_with_valid_version_linux_test() {
download_grpc_plugin "1.55.1" "linux-x86_64"
assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe"
Expand Down Expand Up @@ -256,6 +293,7 @@ test_list=(
extract_folder_name_test
get_grpc_version_succeed_with_valid_generator_version_test
get_grpc_version_failed_with_invalid_generator_version_test
get_protoc_version_succeed_docker_env_var_test
get_protoc_version_succeed_with_valid_generator_version_test
get_protoc_version_failed_with_invalid_generator_version_test
get_gapic_opts_with_rest_test
Expand All @@ -268,6 +306,7 @@ test_list=(
download_protoc_succeed_with_valid_version_macos_test
download_protoc_failed_with_invalid_version_linux_test
download_protoc_failed_with_invalid_arch_test
download_tools_succeed_with_baked_protoc
download_grpc_plugin_succeed_with_valid_version_linux_test
download_grpc_plugin_succeed_with_valid_version_macos_test
download_grpc_plugin_failed_with_invalid_version_linux_test
Expand Down
8 changes: 0 additions & 8 deletions library_generation/test/integration_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ class IntegrationTest(unittest.TestCase):
def setUpClass(cls) -> None:
IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir)

@classmethod
def tearDownClass(cls) -> None:
cls.__remove_docker_image()

@classmethod
def setUp(cls) -> None:
cls.__remove_generated_files()
Expand Down Expand Up @@ -307,7 +303,3 @@ def __recursive_diff_files(
cls.__recursive_diff_files(
sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/"
)

@classmethod
def __remove_docker_image(cls):
subprocess.check_call(["docker", "image", "rmi", image_tag])
2 changes: 1 addition & 1 deletion library_generation/test/model/generation_config_unit_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def test_from_yaml_succeeds(self):
self.assertEqual(
"1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
)
self.assertEqual("26.37.0", config.libraris_bom_version)
self.assertEqual("26.37.0", config.libraries_bom_version)
self.assertEqual(
[
".github/*",
Expand Down
6 changes: 3 additions & 3 deletions library_generation/test/utils/generation_config_comparator_unit_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ def test_compare_config_generator_update(self):
self.assertEqual("1.2.4", config_change.current_value)

def test_compare_config_libraries_bom_update(self):
self.baseline_config.libraris_bom_version = "26.36.0"
self.current_config.libraris_bom_version = "26.37.0"
self.baseline_config.libraries_bom_version = "26.36.0"
self.current_config.libraries_bom_version = "26.37.0"
result = compare_config(
baseline_config=self.baseline_config,
current_config=self.current_config,
Expand All @@ -100,7 +100,7 @@ def test_compare_config_libraries_bom_update(self):
len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1
)
config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0]
self.assertEqual("libraris_bom_version", config_change.changed_param)
self.assertEqual("libraries_bom_version", config_change.changed_param)
self.assertEqual("26.37.0", config_change.current_value)

def test_compare_protobuf_update(self):
Expand Down
24 changes: 21 additions & 3 deletions library_generation/utils/utilities.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ get_grpc_version() {
get_protoc_version() {
local gapic_generator_version=$1
local protoc_version
if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then
>&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}"
echo "${DOCKER_PROTOC_VERSION}"
return
fi
pushd "${output_folder}" > /dev/null
# get protobuf version from gapic-generator-java-pom-parent/pom.xml
download_gapic_generator_pom_parent "${gapic_generator_version}"
Expand All @@ -131,7 +136,17 @@ download_tools() {
local os_architecture=$4
pushd "${output_folder}"
download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar"
download_protoc "${protoc_version}" "${os_architecture}"

# the variable protoc_path is used in generate_library.sh. It is explicitly
# exported to make clear that it is used outside this utilities file.
if [[ "${DOCKER_PROTOC_VERSION}" == "${protoc_version}" ]]; then
# if the specified protoc_version matches the one baked in the docker
# container, we just point protoc_path to its location.
export protoc_path="${DOCKER_PROTOC_LOCATION}/protoc-${protoc_version}/bin"
else
export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}")
fi

download_grpc_plugin "${grpc_version}" "${os_architecture}"
popd
}
Expand Down Expand Up @@ -162,7 +177,10 @@ download_generator_artifact() {
download_protoc() {
local protoc_version=$1
local os_architecture=$2
if [ ! -d "protoc-${protoc_version}" ]; then

local protoc_path="${output_folder}/protoc-${protoc_version}/bin"

if [ ! -d "${protoc_path}" ]; then
# pull proto files and protoc from protobuf repository as maven central
# doesn't have proto files
download_from \
Expand All @@ -173,8 +191,8 @@ download_protoc() {
cp -r "protoc-${protoc_version}/include/google" .
rm "protoc-${protoc_version}.zip"
fi
echo "${protoc_path}"

protoc_path="${output_folder}/protoc-${protoc_version}/bin"
}

download_grpc_plugin() {
Expand Down
15 changes: 14 additions & 1 deletion renovate.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,18 @@
"depNameTemplate": "com.google.protobuf:protobuf-java",
"datasourceTemplate": "maven"
},
{
"customType": "regex",
"fileMatch": [
"^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$"
],
"matchStrings": [
"ARG PROTOC_VERSION=[\"']?(?<currentValue>.+?)[\"']?\\s+"
],
"datasourceTemplate": "github-releases",
"depNameTemplate": "protocolbuffers/protobuf",
"extractVersionTemplate": "^v(?<version>.*)$"
},
{
"customType": "regex",
"fileMatch": [
Expand Down Expand Up @@ -141,7 +153,8 @@
},
{
"matchPackagePatterns": [
"^com.google.protobuf"
"^com.google.protobuf",
"^protocolbuffers/protobuf"
],
"groupName": "Protobuf dependencies",
"enabled": false
Expand Down

0 comments on commit cf44b07

Please sign in to comment.