cloudbuild.yaml

# Google Cloud Build script for bqutils
#
# This build script is used to unit test the BigQuery UDFs for every
# change pushed to the udfs/ directory.
#
# Manual Execution:
# Use the below command to invoke the build manually. Note the substitutions for
# BRANCH_NAME and REVISION_ID. These variables are normally populated when the
# build is executed via build triggers but will be empty during manual
# execution. Dummy branch and revisions can be passed during manual execution so
# the artifacts can be uploaded upon build completion.
#
# gcloud builds submit . --config=cloudbuild.yaml
#

steps:
  ############################################################
  # Dynamically create the package.json file based off the libs
  # specified in the js_libs/js_libs.yaml file.
  ############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: generate_js_libs_package_json
  entrypoint: python3
  args:
    - tests/udf_test_utils.py
    - --generate-js-libs-package-json
  ###########################################################
  # Install npm packages based off the package.json file
  # created in the previous step.
  ###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: install_npm_packages
  entrypoint: npm
  args:
    - install
  ############################################################
  # Dynamically create webpack config files needed by webpack
  # to build npm packages into single .js files which will be
  # hosted on GCS and used by BigQuery UDFs.
  ############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: generate_webpack_configs
  entrypoint: python3
  args:
    - tests/udf_test_utils.py
    - --generate-webpack-configs
  ###########################################################
  # Build (via webpack) all js libraries for BigQuery UDFs
  ###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: build_bq_js_libs
  entrypoint: npm
  args:
    - run-script
    - build-all-libs
  #########################################################################
  # Clone datasketches-cpp github repo as dependency for building WASM libraries
  # Build (via emscripten) all WASM libraries for BigQuery Sketch UD(A)Fs
  #########################################################################
- name: docker.io/emscripten/emsdk
  id: build_wasm_libs
  dir: datasketches
  entrypoint: bash
  args:
  - '-c'
  - |
    git clone https://github.com/apache/datasketches-cpp.git
    for dir in kll-sketch theta-sketch tuple-sketch; do
      cd $dir && make clean && make all
      cd ..
    done
  ###########################################################
  # Copy all libs to GCS bucket
  ###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: copy_js_to_gcs
  entrypoint: gcloud
  args:
    - storage
    - cp
    - js_builds/*
    - ${_JS_BUCKET}
  ###########################################################
  # Deploy UDFs and run unit tests
  ###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci:infrastructure-public-image-bqutil
  id: test_udfs
  dir: tests/dataform_testing_framework
  entrypoint: bash
  args:
    - deploy_and_run_tests.sh
  env:
    - PROJECT_ID=${PROJECT_ID}
    - BQ_LOCATION=${_BQ_LOCATION}
    - SHORT_SHA=${SHORT_SHA}
    - JS_BUCKET=${_JS_BUCKET}

timeout: 1800s # 30 minutes