Faster and streamlined PHP distribtests (for linux and macos) (grpc#2…

…7101)

* faster and streamlined PHP distribtests

* add kokoro job for PHP distribtests

* php distribtests job for macos

* add trailing newline

tools/internal_ci/linux/grpc_distribtests_php.cfg

@@ -0,0 +1,26 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_php.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_distribtests_php.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+# Build all PHP linux artifacts
+tools/run_tests/task_runner.py -f artifact linux php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+rm -rf artifacts_from_build_artifacts_step
+mv artifacts artifacts_from_build_artifacts_step || true
+
+# This step mostly just copies artifacts from input_artifacts (but it also does some wheel stripping)
+tools/run_tests/task_runner.py -f package linux php -x build_packages/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+
+# Run all PHP linux distribtests
+# We run the distribtests even if some of the artifacts have failed to build, since that gives
+# a better signal about which distribtest are affected by the currently broken artifact builds.
+tools/run_tests/task_runner.py -f distribtest linux php -j 4 -x distribtests/sponge_log.xml || FAILED="true"
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/internal_ci/macos/grpc_distribtests_php.cfg

@@ -0,0 +1,27 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_php.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/macos/grpc_distribtests_php.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+export PREPARE_BUILD_INSTALL_DEPS_PHP=true
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# Build all PHP macos artifacts
+tools/run_tests/task_runner.py -f artifact macos php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# PHP's "build_package" step is basically just a passthough, so the build artifacts can be used
+# directly by the "distribtests" step (and we skip the "build_package" phase entirely on mac).
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/php_pecl_package_macos_*/* input_artifacts/ || true
+
+# Run all PHP linux distribtests
+# We run the distribtests even if some of the artifacts have failed to build, since that gives
+# a better signal about which distribtest are affected by the currently broken artifact builds.
+tools/run_tests/task_runner.py -f distribtest macos php -j 4 -x distribtests/sponge_log.xml || FAILED="true"
+
+tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/run_tests/artifacts/artifact_targets.py

@@ -275,10 +275,15 @@ def pre_build_jobspecs(self):
         return []
 
     def build_jobspec(self):
-        return create_docker_jobspec(
-            self.name,
-            'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch),
-            'tools/run_tests/artifacts/build_artifact_php.sh')
+        if self.platform == 'linux':
+            return create_docker_jobspec(
+                self.name,
+                'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch),
+                'tools/run_tests/artifacts/build_artifact_php.sh')
+        else:
+            return create_jobspec(
+                self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'],
+                use_workspace=True)
 
 
 class ProtocArtifact:
@@ -393,5 +398,6 @@ def targets():
         PythonArtifact('windows', 'x64', 'Python39'),
         RubyArtifact('linux', 'x64'),
         RubyArtifact('macos', 'x64'),
-        PHPArtifact('linux', 'x64')
+        PHPArtifact('linux', 'x64'),
+        PHPArtifact('macos', 'x64')
     ]

tools/run_tests/artifacts/build_artifact_php.sh

@@ -21,12 +21,5 @@ mkdir -p "${ARTIFACTS_OUT}"
 
 # Build the PHP extension archive (this just zips all the files up)
 pear package
-# Note: the extension compiled by this step is not being used in any
-# way, i.e. they are not the pacakge being distributed.
-# This is done here to get an early signal for compiling the PHP
-# extension in some form.
-find . -name "grpc-*.tgz" | cut -b3- | xargs pecl install
-# Verified that the grpc extension is built properly.
-php -d extension=grpc.so --re grpc | head -1
 
 cp -r grpc-*.tgz "${ARTIFACTS_OUT}"/

tools/run_tests/artifacts/build_package_php.sh

@@ -20,4 +20,4 @@ cd "$(dirname "$0")/../../.."
 # All the PHP packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
 mkdir -p artifacts/
-cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true
+cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_pecl_package_linux_*/* artifacts/ || true

tools/run_tests/artifacts/distribtest_targets.py

@@ -238,7 +238,7 @@ def __init__(self, platform, arch, docker_suffix=None):
         self.platform = platform
         self.arch = arch
         self.docker_suffix = docker_suffix
-        self.labels = ['distribtest', 'php7', platform, arch]
+        self.labels = ['distribtest', 'php', 'php7', platform, arch]
         if docker_suffix:
             self.labels.append(docker_suffix)