This repository has been archived by the owner on May 2, 2024. It is now read-only.
Run CI on our patched branch #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - 'repro-*' | |
| env: | |
| # An envar that signals to tests we are executing in the CI environment | |
| CONTINUOUS_INTEGRATION: true | |
| MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| RETRY: .github/bin/retry | |
| jobs: | |
| maven-checks: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Free Disk Space | |
| run: | | |
| df -h | |
| sudo apt-get clean | |
| df -h | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven Checks | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install -B -V -T C1 -DskipTests -P ci -pl '!:presto-server-rpm' | |
| - name: Test Server RPM | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw verify -B -P ci -pl :presto-server-rpm | |
| - name: Clean Maven Output | |
| run: ./mvnw clean -pl '!:presto-server,!:presto-cli' | |
| web-ui-checks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Web UI Checks | |
| run: presto-main/bin/check_webui.sh | |
| product-tests-basic-environment: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-server-rpm,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing' | |
| - name: Run Product Tests Basic Environment | |
| run: presto-product-tests/bin/run_on_docker.sh multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cassandra,mysql_connector,postgresql_connector,mysql,kafka,avro | |
| product-tests-specific-environment1: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Mave install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-server-rpm,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing' | |
| - name: Product Tests Specific 1.1 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode -g hdfs_no_impersonation,avro | |
| - name: Product Tests Specific 1.2 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-no-impersonation -g hdfs_no_impersonation | |
| - name: Product Tests Specific 1.3 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation | |
| - name: Product Tests Specific 1.4 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header | |
| - name: Product Tests Specific 1.5 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-cross-realm -g storage_formats,cli,hdfs_impersonation | |
| - name: Product Tests Specific 1.6 | |
| run: presto-product-tests/bin/run_on_docker.sh multinode-tls-kerberos -g cli,group-by,join,tls | |
| product-tests-specific-environment2: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-server-rpm,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing' | |
| - name: Product Tests Specific 2.1 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc | |
| # SQL server image sporadically hangs during the startup | |
| # TODO: Uncomment it once issue is fixed | |
| # https://github.com/Microsoft/mssql-docker/issues/76 | |
| # - | | |
| # if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then | |
| # presto-product-tests/bin/run_on_docker.sh \ | |
| # singlenode-sqlserver -g sqlserver | |
| # fi | |
| - name: Product Tests Specific 2.2 | |
| run: presto-product-tests/bin/run_on_docker.sh multinode-tls -g smoke,cli,group-by,join,tls | |
| - name: Product Tests Specific 2.3 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-mysql -g mysql_connector,mysql | |
| - name: Product Tests Specific 2.4 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-postgresql -g postgresql_connector | |
| - name: Product Tests Specific 2.5 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-cassandra -g cassandra | |
| - name: Product Tests Specific 2.6 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-with-wire-encryption -g storage_formats,cli,hdfs_impersonation,authorization | |
| - name: Product Tests Specific 2.7 | |
| run: presto-product-tests/bin/run_on_docker.sh singlenode-kafka -g kafka | |
| hive-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Install Hive Module | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :presto-hive-hadoop2 | |
| - name: Run Hive Tests | |
| run: presto-hive-hadoop2/bin/run_hive_tests.sh | |
| - name: Run Hive S3 Tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.HIVE_AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.HIVE_AWS_SECRETKEY }} | |
| S3_BUCKET: "presto-ci-test" | |
| S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com" | |
| run: | | |
| if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then | |
| presto-hive-hadoop2/bin/run_hive_s3_tests.sh | |
| fi | |
| - name: Run Hive Glue Tests | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.HIVE_AWS_ACCESSKEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.HIVE_AWS_SECRETKEY }} | |
| run: | | |
| if [ "${HIVE_AWS_ACCESS_KEY_ID}" != "" ]; then | |
| ./mvnw test ${MAVEN_TEST} -pl :presto-hive -P test-hive-glue | |
| fi | |
| test-other-modules: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:presto-docs,!:presto-server,!:presto-server-rpm' | |
| - name: Maven Tests | |
| run: | | |
| ./mvnw test ${MAVEN_TEST} -pl ' | |
| !presto-tests, | |
| !presto-raptor, | |
| !presto-accumulo, | |
| !presto-cassandra, | |
| !presto-hive, | |
| !presto-kudu, | |
| !presto-docs, | |
| !presto-server, | |
| !presto-server-rpm, | |
| !presto-main, | |
| !presto-mongodb, | |
| !presto-spark-package, | |
| !presto-spark-launcher, | |
| !presto-spark-testing, | |
| !presto-spark-base, | |
| !presto-redis, | |
| !presto-elasticsearch, | |
| !presto-orc, | |
| !presto-thrift-connector' | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| modules: | |
| - ":presto-docs" | |
| - ":presto-tests -P presto-tests-execution-memory" | |
| - ":presto-tests -P presto-tests-general" | |
| - ":presto-tests -P ci-only-distributed-non-hash-gen" | |
| - ":presto-tests -P ci-only-tpch-distributed-queries" | |
| - ":presto-tests -P ci-only-local-queries" | |
| - ":presto-tests -P ci-only-distributed-queries" | |
| - ":presto-tests -P ci-only-aggregation-queries" | |
| - ":presto-tests -P ci-only-plan-determinism" | |
| - ":presto-tests -P ci-only-resource-manager" | |
| - ":presto-raptor" | |
| - ":presto-accumulo" | |
| - ":presto-cassandra -P test-cassandra-integration-smoke-test" | |
| - ":presto-hive" | |
| - ":presto-hive -P test-hive-materialized-queries" | |
| - ":presto-hive -P test-hive-materialized-aggregations" | |
| - ":presto-hive -P test-hive-recoverable-execution" | |
| - ":presto-hive -P test-hive-pushdown-filter-queries-basic" | |
| - ":presto-hive -P test-hive-pushdown-filter-queries-advanced" | |
| - ":presto-hive -P test-hive-repartitioning" | |
| - ":presto-hive -P test-hive-parquet" | |
| - ":presto-main" | |
| - ":presto-mongodb -P test-mongo-distributed-queries" | |
| - ":presto-redis -P test-redis-integration-smoke-test" | |
| - ":presto-elasticsearch" | |
| - ":presto-orc" | |
| - ":presto-thrift-connector" | |
| - ":presto-spark-base -P presto-spark-tests-smoke" | |
| - ":presto-spark-base -P presto-spark-tests-all-queries" | |
| - ":presto-spark-base -P presto-spark-tests-spill-queries" | |
| timeout-minutes: 80 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) | |
| - name: Maven Tests | |
| run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} | |
| kudu: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl presto-kudu | |
| - name: Kudu Tests | |
| run: | | |
| presto-kudu/bin/run_kudu_tests.sh 3 null | |
| presto-kudu/bin/run_kudu_tests.sh 1 "" | |
| presto-kudu/bin/run_kudu_tests.sh 1 presto:: | |
| spark-integration: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: 8 | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl presto-spark-launcher,presto-spark-package,presto-spark-testing | |
| - name: Maven Tests | |
| run: ./mvnw test ${MAVEN_TEST} -pl :presto-spark-testing -P test-presto-spark-integration-smoke-test |