From a0a92104fd0587721d3daf35061feaa37ad0258d Mon Sep 17 00:00:00 2001 From: filipe Date: Tue, 9 Feb 2021 15:46:27 -0300 Subject: [PATCH 01/17] add test and ci on github and gitlab --- .github/workflows/test_ci.yml | 48 ++++-- .gitlab-ci.yml | 146 ++++++++++++++++++ rony/base_files/ci/github/github_ci.yml | 4 + rony/base_files/ci/gitlab/.gitlab-ci.yml | 18 ++- rony/base_files/infrastructure/terragrunt.hcl | 28 ++-- rony/base_files/tests/__init__.py | 0 .../tests/test_build_lambda_package.py | 5 + rony/base_files/tests/test_dags.py | 9 ++ rony/base_files/tests/test_lambda.py | 3 - 9 files changed, 232 insertions(+), 29 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 rony/base_files/tests/__init__.py create mode 100644 rony/base_files/tests/test_build_lambda_package.py create mode 100644 rony/base_files/tests/test_dags.py diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 8edc835..caa6f06 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -31,20 +31,27 @@ jobs: BUCKET: "my-bucket" REGION: "my-region" run: | - cd rony/base_files - python -m pytest - + cd rony/base_files/tests + python -m pytest test_lambda.py - build_docker: - name: Build and test docker image + build_test_dags: + name: Build and test dags runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Build image + + - name: 'Validate DAGs' + uses: jayamanikharyono/airflow-dag-action@v0.1 + with: + requirementsFile: tests/requirements.txt + dagPaths: tests/dags + varFile: tests/var.json run: | - cd rony/base_files/etl - docker build -t "${PWD##*/}" . - docker run "${PWD##*/}" + pip install -U pip + pip install pytest + cd rony/base_files/tests + python -m pytest test_dags.py + terraform_validate: @@ -55,6 +62,8 @@ jobs: - name: HashiCorp - Setup Terraform uses: hashicorp/setup-terraform@v1.2.1 + with: + terraform_version: 0.14.6 - name: create zipfile for validation run: | @@ -66,6 +75,27 @@ jobs: terraform init terraform validate + + terragrunt_validate: + name: Terragrunt validation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: HashiCorp - Setup Terragrunt + uses: autero1/action-terragrunt@v1.0.0 + with: + terragrunt_version: 0.28.2 + + - name: create zipfile for validation + run: | + touch rony/base_files/infrastructure/lambda_function_payload.zip + + - name: Terragrunt validate + run: | + cd rony/base_files/infrastructure + terragrunt validate + package_install: name: Test package install runs-on: ubuntu-latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e297611 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,146 @@ +image: + name: ubuntu:latest + +stages: + - Test package + - Test structure + +.execute_pipeline: + only: + - branches + - merge_requests + +.package_test_py: + before_script: + - apt-get -qy update && apt-get -qy install python$PYTHON_VERSION python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - python -m pip install --upgrade pip + - pip install pytest + - pip install coverage + - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - pip install . + - cd rony/tests + - coverage run -m pytest && coverage report -m + extends: .execute_pipeline + +Test script package lambda: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip zip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install -U pip + - pip install pytest + - cd rony/base_files/tests + - python -m pytest test_build_lambda_package.py + +Build and test Lambda: + stage: Test package + variables: + BUCKET: "my-bucket" + REGION: "xx-xxxx-x" + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install -U pip + - pip install -r rony/base_files/etl/lambda_requirements.txt + - pip install pytest boto3 + - cd rony/base_files/tests + - python -m pytest test_lambda.py + extends: .execute_pipeline + +Build and test dags: + image: puckel/docker-airflow + stage: Test package + before_script: + - python -m pip install --upgrade pip + - pip install pytest + script: + - cd rony/base_files/tests + - python -m pytest test_dags.py + extends: .execute_pipeline + +Build and test docker image: + image: docker:latest + stage: Test package + services: + - docker:dind + script: + - cd rony/base_files/etl + - docker build -t "${PWD##*/}" . + - docker run "${PWD##*/}" + extends: .execute_pipeline + +Terraform validation: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install curl zip + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - terraform --version + script: + - touch rony/base_files/infrastructure/lambda_function_payload.zip + - cd rony/base_files/infrastructure + - terraform init + - terraform validate + extends: .execute_pipeline + +Terragrunt validation: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install curl zip + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt + - terragrunt --version + script: + - touch rony/base_files/infrastructure/lambda_function_payload.zip + - cd rony/base_files/infrastructure + - terragrunt validate + extends: .execute_pipeline + +Test package install (3.6): + stage: Test package + variables: + PYTHON_VERSION: "3.6" + extends: .package_test_py + +Test package install (3.7): + stage: Test package + variables: + PYTHON_VERSION: "3.7" + extends: .package_test_py + +Test package install (3.8): + stage: Test package + variables: + PYTHON_VERSION: "3.8" + extends: .package_test_py + + +Test project structure: + stage: Test structure + dependencies: + - Build and test Lambda + - Build and test dags + - Build and test docker image + - Terraform validation + - Terragrunt validation + - Test package install (3.6) + - Test package install (3.7) + - Test package install (3.8) + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install --upgrade pip + - pip install . + - rony info + - rony new projex + - cd projex + - cat etl/Dockerfile + - cat dags/titanic_example.py + - cat infrastructure/s3.tf + - cat scripts/build_lambda_package.sh + extends: .execute_pipeline + diff --git a/rony/base_files/ci/github/github_ci.yml b/rony/base_files/ci/github/github_ci.yml index cecf173..a4878eb 100644 --- a/rony/base_files/ci/github/github_ci.yml +++ b/rony/base_files/ci/github/github_ci.yml @@ -62,9 +62,13 @@ jobs: - name: HashiCorp - Setup Terraform uses: hashicorp/setup-terraform@v1.2.1 + with: + terraform_version: 0.14.6 - name: HashiCorp - Setup Terragrunt uses: autero1/action-terragrunt@v1.0.0 + with: + terragrunt_version: 0.28.2 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/rony/base_files/ci/gitlab/.gitlab-ci.yml b/rony/base_files/ci/gitlab/.gitlab-ci.yml index f67bc93..39356d6 100644 --- a/rony/base_files/ci/gitlab/.gitlab-ci.yml +++ b/rony/base_files/ci/gitlab/.gitlab-ci.yml @@ -17,7 +17,7 @@ cache: before_script: - apt-get -qy update && apt-get -qy install curl python3.8 python3-pip zip - - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.5/terraform_0.14.5_linux_amd64.zip && unzip terraform_0.14.5_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip @@ -26,7 +26,7 @@ stages: - plan - build -test-lambda: +build_lambda: stage: test script: - pip install -U pip @@ -36,7 +36,7 @@ test-lambda: only: - branches -test-iac: +build_dl: stage: test script: - sh scripts/build_lambda_package.sh @@ -46,6 +46,18 @@ test-iac: only: - branches +build_docker: + image: docker:latest + stage: test + services: + - docker:dind + script: + - cd etl + - docker build -t "${PWD##*/}" . + - docker run "${PWD##*/}" + except: + - master + plan: stage: plan script: diff --git a/rony/base_files/infrastructure/terragrunt.hcl b/rony/base_files/infrastructure/terragrunt.hcl index bc35613..fcea669 100644 --- a/rony/base_files/infrastructure/terragrunt.hcl +++ b/rony/base_files/infrastructure/terragrunt.hcl @@ -7,17 +7,17 @@ provider "aws" { } EOF } -remote_state { - backend = "s3" - generate = { - path = "backend.tf" - if_exists = "overwrite" - } - config = { - bucket = "my-bucket-state-backend" - - key = "${path_relative_to_include()}/terraform.tfstate" - region = "us-east-1" - encrypt = true - } -} +//remote_state { +// backend = "s3" +// generate = { +// path = "backend.tf" +// if_exists = "overwrite" +// } +// config = { +// bucket = "my-bucket-state-backend" +// +// key = "${path_relative_to_include()}/terraform.tfstate" +// region = "us-east-1" +// encrypt = true +// } +//} diff --git a/rony/base_files/tests/__init__.py b/rony/base_files/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rony/base_files/tests/test_build_lambda_package.py b/rony/base_files/tests/test_build_lambda_package.py new file mode 100644 index 0000000..d7b01de --- /dev/null +++ b/rony/base_files/tests/test_build_lambda_package.py @@ -0,0 +1,5 @@ +import os + +def test_zip_lambda(): + os.system('cd .. && sh scripts/build_lambda_package.sh') + assert os.path.isfile('../infrastructure/lambda_function_payload.zip') is True diff --git a/rony/base_files/tests/test_dags.py b/rony/base_files/tests/test_dags.py new file mode 100644 index 0000000..7af8074 --- /dev/null +++ b/rony/base_files/tests/test_dags.py @@ -0,0 +1,9 @@ +import os + +from airflow.models import DagBag + +def test_dags_load_with_no_errors(): + for dag_name in os.listdir('../dags'): + dag_bag = DagBag(include_examples=False) + dag_bag.process_file(dag_name) + assert len(dag_bag.import_errors) == 0 diff --git a/rony/base_files/tests/test_lambda.py b/rony/base_files/tests/test_lambda.py index 9c1b36e..dbc6c66 100644 --- a/rony/base_files/tests/test_lambda.py +++ b/rony/base_files/tests/test_lambda.py @@ -1,7 +1,4 @@ -import pytest import etl.lambda_function as lf -import os - def test_response(): res = lf.handler(None, None) From 4579339476531e662ebdf9886cffdf865ed074e6 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:02:50 -0300 Subject: [PATCH 02/17] rollback ci github --- .github/workflows/test_ci.yml | 62 ++++++++--------------------------- 1 file changed, 13 insertions(+), 49 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index caa6f06..03ccab1 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -25,35 +25,26 @@ jobs: pip install -U pip pip install -r rony/base_files/etl/lambda_requirements.txt pip install pytest boto3 - + - name: Run tests on lambda function env: BUCKET: "my-bucket" REGION: "my-region" run: | - cd rony/base_files/tests - python -m pytest test_lambda.py + cd rony/base_files + python -m pytest - build_test_dags: - name: Build and test dags + + build_docker: + name: Build and test docker image runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: 'Validate DAGs' - uses: jayamanikharyono/airflow-dag-action@v0.1 - with: - requirementsFile: tests/requirements.txt - dagPaths: tests/dags - varFile: tests/var.json + - name: Build image run: | - pip install -U pip - pip install pytest - cd rony/base_files/tests - python -m pytest test_dags.py - - - + cd rony/base_files/etl + docker build -t "${PWD##*/}" . + docker run "${PWD##*/}" terraform_validate: name: Terraform validation runs-on: ubuntu-latest @@ -62,40 +53,15 @@ jobs: - name: HashiCorp - Setup Terraform uses: hashicorp/setup-terraform@v1.2.1 - with: - terraform_version: 0.14.6 - name: create zipfile for validation run: | touch rony/base_files/infrastructure/lambda_function_payload.zip - - name: Terraform validate run: | cd rony/base_files/infrastructure terraform init terraform validate - - - terragrunt_validate: - name: Terragrunt validation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: HashiCorp - Setup Terragrunt - uses: autero1/action-terragrunt@v1.0.0 - with: - terragrunt_version: 0.28.2 - - - name: create zipfile for validation - run: | - touch rony/base_files/infrastructure/lambda_function_payload.zip - - - name: Terragrunt validate - run: | - cd rony/base_files/infrastructure - terragrunt validate - package_install: name: Test package install runs-on: ubuntu-latest @@ -120,14 +86,13 @@ jobs: cd rony/tests coverage run -m pytest && coverage report -m - test_project: name: Test project structure runs-on: ubuntu-latest needs: [package_install] steps: - uses: actions/checkout@v2 - + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -138,7 +103,6 @@ jobs: pip install --upgrade pip pip install . rony info - - name: Test Project structure run: | rony new projex @@ -146,4 +110,4 @@ jobs: cat etl/Dockerfile cat dags/titanic_example.py cat infrastructure/s3.tf - cat scripts/build_lambda_package.sh + cat scripts/build_lambda_package.sh \ No newline at end of file From 18727d3635c7d70b601ccd6abf552dc3e8b9124c Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:06:00 -0300 Subject: [PATCH 03/17] github ci Build and test Lambda --- .github/workflows/test_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 03ccab1..c2b4f00 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -31,8 +31,8 @@ jobs: BUCKET: "my-bucket" REGION: "my-region" run: | - cd rony/base_files - python -m pytest + cd rony/base_files/tests + python -m pytest test_lambda.py build_docker: From b8a493f40db0fd3aa48ca390e4c9c8de8ec28876 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:08:35 -0300 Subject: [PATCH 04/17] github ci Build and test dags --- .github/workflows/test_ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index c2b4f00..5788259 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -34,6 +34,19 @@ jobs: cd rony/base_files/tests python -m pytest test_lambda.py + build_test_dags: + name: Build and test dags + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: 'Validate DAGs' + uses: jayamanikharyono/airflow-dag-action@v0.1 + run: | + pip install -U pip + pip install pytest + cd rony/base_files/tests + python -m pytest test_dags.py build_docker: name: Build and test docker image From b2ecc63e383d00d60f957ce9f31ea0e4c3a7997f Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:11:40 -0300 Subject: [PATCH 05/17] github ci remove Build and test dags --- .github/workflows/test_ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 5788259..da63ec7 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -34,19 +34,19 @@ jobs: cd rony/base_files/tests python -m pytest test_lambda.py - build_test_dags: - name: Build and test dags - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: 'Validate DAGs' - uses: jayamanikharyono/airflow-dag-action@v0.1 - run: | - pip install -U pip - pip install pytest - cd rony/base_files/tests - python -m pytest test_dags.py +# build_test_dags: +# name: Build and test dags +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# +# - name: 'Validate DAGs' +# uses: jayamanikharyono/airflow-dag-action@v0.1 +# run: | +# pip install -U pip +# pip install pytest +# cd rony/base_files/tests +# python -m pytest test_dags.py build_docker: name: Build and test docker image From 66991822fead09817c6b123f305ff0e754adb941 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:25:48 -0300 Subject: [PATCH 06/17] github ci add Terragrunt validation --- .github/workflows/test_ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index da63ec7..51a4fb0 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -58,6 +58,7 @@ jobs: cd rony/base_files/etl docker build -t "${PWD##*/}" . docker run "${PWD##*/}" + terraform_validate: name: Terraform validation runs-on: ubuntu-latest @@ -75,6 +76,30 @@ jobs: cd rony/base_files/infrastructure terraform init terraform validate + + terragrunt_validate: + name: Terragrunt validation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: HashiCorp - Setup Terraform + uses: hashicorp/setup-terraform@v1.2.1 + + - name: HashiCorp - Setup Terragrunt + uses: autero1/action-terragrunt@v1.0.0 + with: + terragrunt_version: 0.28.2 + + - name: create zipfile for validation + run: | + touch rony/base_files/infrastructure/lambda_function_payload.zip + + - name: Terragrunt validate + run: | + cd rony/base_files/infrastructure + terragrunt validate + package_install: name: Test package install runs-on: ubuntu-latest From 4062dd37422b821635a2764262ea57cc801b3303 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:43:14 -0300 Subject: [PATCH 07/17] github ci add Test script lambda package and package install python3.9 --- .github/workflows/test_ci.yml | 12 +++++++++++- .gitlab-ci.yml | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 51a4fb0..7a363bd 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -9,6 +9,16 @@ on: branches: [ main ] jobs: + script_lambda_package: + name: Test script lambda package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Build Lambda package + run: | + sh rony/base_files/scripts/build_lambda_package.sh + build_test_lambda: name: Build and test Lambda runs-on: ubuntu-latest @@ -105,7 +115,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e297611..154a62b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,6 +117,12 @@ Test package install (3.8): PYTHON_VERSION: "3.8" extends: .package_test_py +Test package install (3.9): + stage: Test package + variables: + PYTHON_VERSION: "3.9" + extends: .package_test_py + Test project structure: stage: Test structure @@ -129,6 +135,7 @@ Test project structure: - Test package install (3.6) - Test package install (3.7) - Test package install (3.8) + - Test package install (3.9) before_script: - apt-get -qy update && apt-get -qy install python3.8 python3-pip - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip From 13f8e4f8541495b9fd3d5d1deadcd4b8ff1111be Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:48:27 -0300 Subject: [PATCH 08/17] github ci add smoke test das --- .github/workflows/test_ci.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 7a363bd..8bf26cf 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -44,19 +44,24 @@ jobs: cd rony/base_files/tests python -m pytest test_lambda.py -# build_test_dags: -# name: Build and test dags -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# -# - name: 'Validate DAGs' -# uses: jayamanikharyono/airflow-dag-action@v0.1 -# run: | -# pip install -U pip -# pip install pytest -# cd rony/base_files/tests -# python -m pytest test_dags.py + build_test_dags: + name: Build and test dags + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: 'Validate DAGs' + uses: jayamanikharyono/airflow-dag-action@v0.1 + run: | + pip install -U pip + pip install pytest + cd rony/base_files/tests + python -m pytest test_dags.py build_docker: name: Build and test docker image From 5aae288ea30d860c8129f85a2915a6cc28c8f61d Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:50:03 -0300 Subject: [PATCH 09/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 8bf26cf..0c0621f 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -55,8 +55,8 @@ jobs: with: python-version: 3.8 - - name: 'Validate DAGs' - uses: jayamanikharyono/airflow-dag-action@v0.1 +# - name: 'Validate DAGs' +# uses: jayamanikharyono/airflow-dag-action@v0.1 run: | pip install -U pip pip install pytest From b496487e6e036ba53684e52cdf55ff4d9275b450 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:52:27 -0300 Subject: [PATCH 10/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 0c0621f..c86c7d5 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -55,8 +55,10 @@ jobs: with: python-version: 3.8 -# - name: 'Validate DAGs' -# uses: jayamanikharyono/airflow-dag-action@v0.1 + - name: 'Airflow - Setup Airflow' + uses: jayamanikharyono/airflow-dag-action@v0.1 + + - name: 'Validate DAGs' run: | pip install -U pip pip install pytest From 09e4233b0974b945869ff119315a06a9785aac50 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:58:52 -0300 Subject: [PATCH 11/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index c86c7d5..2a33f98 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -55,10 +55,10 @@ jobs: with: python-version: 3.8 - - name: 'Airflow - Setup Airflow' + - name: 'Validate DAGs' uses: jayamanikharyono/airflow-dag-action@v0.1 - - name: 'Validate DAGs' + - name: 'Test DAGs' run: | pip install -U pip pip install pytest From b0746ac286c8f3dcd5cf2642345e793e5ac9087b Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 14:59:36 -0300 Subject: [PATCH 12/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 2a33f98..7099b68 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -56,7 +56,7 @@ jobs: python-version: 3.8 - name: 'Validate DAGs' - uses: jayamanikharyono/airflow-dag-action@v0.1 + uses: jayamanikharyono/airflow-dag-action - name: 'Test DAGs' run: | From 6bf0cad987cd22410cf1a2c39d0fcdfe41ba07bd Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 15:02:39 -0300 Subject: [PATCH 13/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 7099b68..9ef1f2f 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -55,10 +55,10 @@ jobs: with: python-version: 3.8 - - name: 'Validate DAGs' - uses: jayamanikharyono/airflow-dag-action + - name: 'Airflow - Setup Airflow' + uses: jayamanikharyono/airflow-dag-action@v0.11-alpha - - name: 'Test DAGs' + - name: 'Validate DAGs' run: | pip install -U pip pip install pytest From 33e9796b50b0f06a07e17c2ac7b15ac93279944f Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 15:09:30 -0300 Subject: [PATCH 14/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 9ef1f2f..a033859 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -50,20 +50,22 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 +# - name: Set up Python 3.8 +# uses: actions/setup-python@v2 +# with: +# python-version: 3.8 - name: 'Airflow - Setup Airflow' uses: jayamanikharyono/airflow-dag-action@v0.11-alpha - - - name: 'Validate DAGs' - run: | - pip install -U pip - pip install pytest - cd rony/base_files/tests - python -m pytest test_dags.py + with: + dagPaths: rony/base_files/dags + +# - name: 'Validate DAGs' +# run: | +# pip install -U pip +# pip install pytest +# cd rony/base_files/tests +# python -m pytest test_dags.py build_docker: name: Build and test docker image From f966f39a25467dc0a313e1c64ccd0495c25d5063 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 15:36:28 -0300 Subject: [PATCH 15/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index a033859..44e876c 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -56,9 +56,9 @@ jobs: # python-version: 3.8 - name: 'Airflow - Setup Airflow' - uses: jayamanikharyono/airflow-dag-action@v0.11-alpha - with: - dagPaths: rony/base_files/dags + uses: jayamanikharyono/airflow-dag-action@v0.1-alpha +# with: +# dagPaths: rony/base_files/dags # - name: 'Validate DAGs' # run: | From b79a6e03ac5c8d0f907ed300c4138d5bd94b3052 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 15:37:01 -0300 Subject: [PATCH 16/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 44e876c..5cc19c0 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -60,12 +60,12 @@ jobs: # with: # dagPaths: rony/base_files/dags -# - name: 'Validate DAGs' -# run: | -# pip install -U pip -# pip install pytest -# cd rony/base_files/tests -# python -m pytest test_dags.py + - name: 'Validate DAGs' + run: | + pip install -U pip + pip install pytest + cd rony/base_files/tests + python -m pytest test_dags.py build_docker: name: Build and test docker image From ddc8db4e75165b8d0096f4a0c8a370df81a1a4e3 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 10 Feb 2021 15:41:42 -0300 Subject: [PATCH 17/17] github ci add smoke test dags --- .github/workflows/test_ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 5cc19c0..fcd9562 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -57,15 +57,15 @@ jobs: - name: 'Airflow - Setup Airflow' uses: jayamanikharyono/airflow-dag-action@v0.1-alpha -# with: -# dagPaths: rony/base_files/dags - - - name: 'Validate DAGs' - run: | - pip install -U pip - pip install pytest - cd rony/base_files/tests - python -m pytest test_dags.py + with: + dagPaths: rony/base_files/dags + +# - name: 'Validate DAGs' +# run: | +# pip install -U pip +# pip install pytest +# cd rony/base_files/tests +# python -m pytest test_dags.py build_docker: name: Build and test docker image