diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 8edc835..fcd9562 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -9,12 +9,22 @@ on: branches: [ main ] jobs: + script_lambda_package: + name: Test script lambda package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Build Lambda package + run: | + sh rony/base_files/scripts/build_lambda_package.sh + build_test_lambda: name: Build and test Lambda runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -25,15 +35,37 @@ jobs: pip install -U pip pip install -r rony/base_files/etl/lambda_requirements.txt pip install pytest boto3 - + - name: Run tests on lambda function env: BUCKET: "my-bucket" REGION: "my-region" run: | - cd rony/base_files - python -m pytest - + cd rony/base_files/tests + python -m pytest test_lambda.py + + build_test_dags: + name: Build and test dags + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + +# - name: Set up Python 3.8 +# uses: actions/setup-python@v2 +# with: +# python-version: 3.8 + + - name: 'Airflow - Setup Airflow' + uses: jayamanikharyono/airflow-dag-action@v0.1-alpha + with: + dagPaths: rony/base_files/dags + +# - name: 'Validate DAGs' +# run: | +# pip install -U pip +# pip install pytest +# cd rony/base_files/tests +# python -m pytest test_dags.py build_docker: name: Build and test docker image @@ -46,7 +78,6 @@ jobs: docker build -t "${PWD##*/}" . docker run "${PWD##*/}" - terraform_validate: name: Terraform validation runs-on: ubuntu-latest @@ -59,19 +90,41 @@ jobs: - name: create zipfile for validation run: | touch rony/base_files/infrastructure/lambda_function_payload.zip - - name: Terraform validate run: | cd rony/base_files/infrastructure terraform init terraform validate + terragrunt_validate: + name: Terragrunt validation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: HashiCorp - Setup Terraform + uses: hashicorp/setup-terraform@v1.2.1 + + - name: HashiCorp - Setup Terragrunt + uses: autero1/action-terragrunt@v1.0.0 + with: + terragrunt_version: 0.28.2 + + - name: create zipfile for validation + run: | + touch rony/base_files/infrastructure/lambda_function_payload.zip + + - name: Terragrunt validate + run: | + cd rony/base_files/infrastructure + terragrunt validate + package_install: name: Test package install runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -90,14 +143,13 @@ jobs: cd rony/tests coverage run -m pytest && coverage report -m - test_project: name: Test project structure runs-on: ubuntu-latest needs: [package_install] steps: - uses: actions/checkout@v2 - + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -108,7 +160,6 @@ jobs: pip install --upgrade pip pip install . rony info - - name: Test Project structure run: | rony new projex @@ -116,4 +167,4 @@ jobs: cat etl/Dockerfile cat dags/titanic_example.py cat infrastructure/s3.tf - cat scripts/build_lambda_package.sh + cat scripts/build_lambda_package.sh \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..154a62b --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,153 @@ +image: + name: ubuntu:latest + +stages: + - Test package + - Test structure + +.execute_pipeline: + only: + - branches + - merge_requests + +.package_test_py: + before_script: + - apt-get -qy update && apt-get -qy install python$PYTHON_VERSION python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - python -m pip install --upgrade pip + - pip install pytest + - pip install coverage + - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - pip install . + - cd rony/tests + - coverage run -m pytest && coverage report -m + extends: .execute_pipeline + +Test script package lambda: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip zip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install -U pip + - pip install pytest + - cd rony/base_files/tests + - python -m pytest test_build_lambda_package.py + +Build and test Lambda: + stage: Test package + variables: + BUCKET: "my-bucket" + REGION: "xx-xxxx-x" + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install -U pip + - pip install -r rony/base_files/etl/lambda_requirements.txt + - pip install pytest boto3 + - cd rony/base_files/tests + - python -m pytest test_lambda.py + extends: .execute_pipeline + +Build and test dags: + image: puckel/docker-airflow + stage: Test package + before_script: + - python -m pip install --upgrade pip + - pip install pytest + script: + - cd rony/base_files/tests + - python -m pytest test_dags.py + extends: .execute_pipeline + +Build and test docker image: + image: docker:latest + stage: Test package + services: + - docker:dind + script: + - cd rony/base_files/etl + - docker build -t "${PWD##*/}" . + - docker run "${PWD##*/}" + extends: .execute_pipeline + +Terraform validation: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install curl zip + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - terraform --version + script: + - touch rony/base_files/infrastructure/lambda_function_payload.zip + - cd rony/base_files/infrastructure + - terraform init + - terraform validate + extends: .execute_pipeline + +Terragrunt validation: + stage: Test package + before_script: + - apt-get -qy update && apt-get -qy install curl zip + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt + - terragrunt --version + script: + - touch rony/base_files/infrastructure/lambda_function_payload.zip + - cd rony/base_files/infrastructure + - terragrunt validate + extends: .execute_pipeline + +Test package install (3.6): + stage: Test package + variables: + PYTHON_VERSION: "3.6" + extends: .package_test_py + +Test package install (3.7): + stage: Test package + variables: + PYTHON_VERSION: "3.7" + extends: .package_test_py + +Test package install (3.8): + stage: Test package + variables: + PYTHON_VERSION: "3.8" + extends: .package_test_py + +Test package install (3.9): + stage: Test package + variables: + PYTHON_VERSION: "3.9" + extends: .package_test_py + + +Test project structure: + stage: Test structure + dependencies: + - Build and test Lambda + - Build and test dags + - Build and test docker image + - Terraform validation + - Terragrunt validation + - Test package install (3.6) + - Test package install (3.7) + - Test package install (3.8) + - Test package install (3.9) + before_script: + - apt-get -qy update && apt-get -qy install python3.8 python3-pip + - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip + script: + - pip install --upgrade pip + - pip install . + - rony info + - rony new projex + - cd projex + - cat etl/Dockerfile + - cat dags/titanic_example.py + - cat infrastructure/s3.tf + - cat scripts/build_lambda_package.sh + extends: .execute_pipeline + diff --git a/rony/base_files/ci/github/github_ci.yml b/rony/base_files/ci/github/github_ci.yml index cecf173..a4878eb 100644 --- a/rony/base_files/ci/github/github_ci.yml +++ b/rony/base_files/ci/github/github_ci.yml @@ -62,9 +62,13 @@ jobs: - name: HashiCorp - Setup Terraform uses: hashicorp/setup-terraform@v1.2.1 + with: + terraform_version: 0.14.6 - name: HashiCorp - Setup Terragrunt uses: autero1/action-terragrunt@v1.0.0 + with: + terragrunt_version: 0.28.2 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/rony/base_files/ci/gitlab/.gitlab-ci.yml b/rony/base_files/ci/gitlab/.gitlab-ci.yml index f67bc93..39356d6 100644 --- a/rony/base_files/ci/gitlab/.gitlab-ci.yml +++ b/rony/base_files/ci/gitlab/.gitlab-ci.yml @@ -17,7 +17,7 @@ cache: before_script: - apt-get -qy update && apt-get -qy install curl python3.8 python3-pip zip - - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.5/terraform_0.14.5_linux_amd64.zip && unzip terraform_0.14.5_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform + - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip @@ -26,7 +26,7 @@ stages: - plan - build -test-lambda: +build_lambda: stage: test script: - pip install -U pip @@ -36,7 +36,7 @@ test-lambda: only: - branches -test-iac: +build_dl: stage: test script: - sh scripts/build_lambda_package.sh @@ -46,6 +46,18 @@ test-iac: only: - branches +build_docker: + image: docker:latest + stage: test + services: + - docker:dind + script: + - cd etl + - docker build -t "${PWD##*/}" . + - docker run "${PWD##*/}" + except: + - master + plan: stage: plan script: diff --git a/rony/base_files/infrastructure/terragrunt.hcl b/rony/base_files/infrastructure/terragrunt.hcl index bc35613..fcea669 100644 --- a/rony/base_files/infrastructure/terragrunt.hcl +++ b/rony/base_files/infrastructure/terragrunt.hcl @@ -7,17 +7,17 @@ provider "aws" { } EOF } -remote_state { - backend = "s3" - generate = { - path = "backend.tf" - if_exists = "overwrite" - } - config = { - bucket = "my-bucket-state-backend" - - key = "${path_relative_to_include()}/terraform.tfstate" - region = "us-east-1" - encrypt = true - } -} +//remote_state { +// backend = "s3" +// generate = { +// path = "backend.tf" +// if_exists = "overwrite" +// } +// config = { +// bucket = "my-bucket-state-backend" +// +// key = "${path_relative_to_include()}/terraform.tfstate" +// region = "us-east-1" +// encrypt = true +// } +//} diff --git a/rony/base_files/tests/__init__.py b/rony/base_files/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rony/base_files/tests/test_build_lambda_package.py b/rony/base_files/tests/test_build_lambda_package.py new file mode 100644 index 0000000..d7b01de --- /dev/null +++ b/rony/base_files/tests/test_build_lambda_package.py @@ -0,0 +1,5 @@ +import os + +def test_zip_lambda(): + os.system('cd .. && sh scripts/build_lambda_package.sh') + assert os.path.isfile('../infrastructure/lambda_function_payload.zip') is True diff --git a/rony/base_files/tests/test_dags.py b/rony/base_files/tests/test_dags.py new file mode 100644 index 0000000..7af8074 --- /dev/null +++ b/rony/base_files/tests/test_dags.py @@ -0,0 +1,9 @@ +import os + +from airflow.models import DagBag + +def test_dags_load_with_no_errors(): + for dag_name in os.listdir('../dags'): + dag_bag = DagBag(include_examples=False) + dag_bag.process_file(dag_name) + assert len(dag_bag.import_errors) == 0 diff --git a/rony/base_files/tests/test_lambda.py b/rony/base_files/tests/test_lambda.py index 9c1b36e..dbc6c66 100644 --- a/rony/base_files/tests/test_lambda.py +++ b/rony/base_files/tests/test_lambda.py @@ -1,7 +1,4 @@ -import pytest import etl.lambda_function as lf -import os - def test_response(): res = lf.handler(None, None)