diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 823a0e465bc3..ac1a2fa7aae7 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,21 +15,44 @@ jobs: - name: Check images exist run: ./tools/bin/check_images_exist.sh - - name: Cache java deps + # docker should use restore-keys since these are just used for layer caching, not gradle task caching + - name: Docker Caching uses: actions/cache@v2 with: - path: ~/.gradle - key: gradle-${{ hashFiles('**/*.gradle') }} + path: | + /tmp/docker-registry + key: docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-${{ github.sha }} restore-keys: | - gradle- + docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}- + docker-${{ runner.os }}- - - name: Cache node deps + - name: Pip Caching uses: actions/cache@v2 with: - path: ~/.npm - key: node-${{ hashFiles('**/package-lock.json') }} + path: | + ~/.cache/pip + key: pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} restore-keys: | - node- + pip-${{ runner.os }}- + + - name: Npm Caching + uses: actions/cache@v2 + with: + path: | + ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + npm-${{ runner.os }}- + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle and Python Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} - uses: actions/setup-java@v1 with: @@ -43,6 +66,20 @@ jobs: with: python-version: '3.7' + - name: Start local Docker registry + run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000 + + - name: Build + run: ./gradlew --no-daemon build --scan + + - name: Ensure no file change + run: git status --porcelain && test -z "$(git status --porcelain)" + + - name: Check documentation + if: success() && github.ref == 'refs/heads/master' + run: ./tools/site/link_checker.sh check_docs + + # this should happen as late as possible so we don't accidentally have tests in the wrong stage depending on creds - name: Write Integration Test Credentials run: ./tools/bin/ci_credentials.sh env: @@ -62,16 +99,6 @@ jobs: AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} - - name: Build - run: ./gradlew --no-daemon build --scan - - - name: Ensure no file change - run: git status --porcelain && test -z "$(git status --porcelain)" - - - name: Check documentation - if: success() && github.ref == 'refs/heads/master' - run: ./tools/site/link_checker.sh check_docs - - name: Run Integration Tests (PR) if: success() && github.ref != 'refs/heads/master' run: ./tools/bin/integration_test_pr.sh diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index fed0dd2ac0f5..7bcdc9d5eb3f 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -20,17 +20,15 @@ dependencies { // on rc version due to docker pull bug in current stable version. // issue: https://github.com/airbytehq/airbyte/issues/493 - testCompile "org.testcontainers:mssqlserver:1.15.0-rc2" + testImplementation "org.testcontainers:mssqlserver:1.15.0-rc2" testImplementation 'org.apache.commons:commons-text:1.9' testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' - testImplementation 'org.testcontainers:postgresql:1.15.0-rc2' testImplementation project(':airbyte-test-utils') integrationTestImplementation project(':airbyte-integrations:bases:standard-source-test') - testCompile "org.testcontainers:mssqlserver:1.15.0-rc2" implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 7a13b2980dbc..9c77df5f99ce 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -45,11 +45,17 @@ class AirbytePythonPlugin implements Plugin { project.task('installReqs', type: PythonTask) { module = "pip" command = "install -r requirements.txt" + inputs.file('requirements.txt') + outputs.file('build/installedreqs.txt') + outputs.cacheIf { true } } project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) { module = "pip" command = "install .[tests]" + inputs.file('setup.py') + outputs.file('build/installedtestreqs.txt') + outputs.cacheIf { true } } if(project.file('unit_tests').exists()) { diff --git a/tools/bin/build_image.sh b/tools/bin/build_image.sh index dccd508d03c0..712f6914b0a7 100755 --- a/tools/bin/build_image.sh +++ b/tools/bin/build_image.sh @@ -5,7 +5,7 @@ set -e ROOT_DIR="$1" PROJECT_DIR="$2" DOCKERFILE="$3" -TAG="$4" +TAGGED_IMAGE="$4" ID_FILE="$5" cd "$ROOT_DIR" @@ -14,4 +14,21 @@ assert_root cd "$PROJECT_DIR" -DOCKER_BUILDKIT=1 docker build -f "$DOCKERFILE" . -t "$TAG" --iidfile "$ID_FILE" +if [[ -z "$CI" ]]; then + # run standard build locally (not on CI) + DOCKER_BUILDKIT=1 docker build \ + -f "$DOCKERFILE" . \ + -t "$TAGGED_IMAGE" \ + --iidfile "$ID_FILE" +else + # run build with local docker registery for CI + docker pull localhost:5000/"$TAGGED_IMAGE" || true + DOCKER_BUILDKIT=1 docker build \ + -f "$DOCKERFILE" . \ + -t "$TAGGED_IMAGE" \ + --iidfile "$ID_FILE" \ + --cache-from localhost:5000/"$TAGGED_IMAGE" \ + --build-arg BUILDKIT_INLINE_CACHE=1 + docker tag "$TAGGED_IMAGE" localhost:5000/"$TAGGED_IMAGE" + docker push localhost:5000/"$TAGGED_IMAGE" +fi