diff --git a/.env.example b/.env.example index ab506609..355909ae 100644 --- a/.env.example +++ b/.env.example @@ -9,5 +9,10 @@ STATE_BUCKET_NAME= STATE_BUCKET_KEY= STATE_DYNAMO_TABLE= ASSUME_ROLE_ARNS='["", ""]' -COGNITO_APP_SECRET= -STAC_INGESTOR_API_URL= +VEDA_WORKFLOWS_CLIENT_SECRET_ID= +VEDA_PROGRAMMATIC_CLIENT_SECRET_ID= +VEDA_STAC_INGESTOR_API_URL= +VEDA_RASTER_URL= +VEDA_DATA_ACCESS_ROLE_ARN= +VEDA_STAC_URL= +WORKFLOW_ROOT_PATH= diff --git a/.flake8 b/.flake8 index c10cc066..8a618974 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] # taken from github actions ignore -ignore = E1, E2, E3, E5, W1, W2, W3, W5 \ No newline at end of file +ignore = E1, E2, E3, E5, W1, W2, W3, W5 diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml new file mode 100644 index 00000000..3a4e3716 --- /dev/null +++ b/.github/actions/terraform-deploy/action.yml @@ -0,0 +1,64 @@ +name: Deploy + +inputs: + env_aws_secret_name: + required: true + type: string + env-file: + type: string + default: ".env" + dir: + required: false + type: string + default: "." + script_path: + type: string + backend_stack_name: + type: string + auth_stack_name: + type: string + +runs: + using: "composite" + + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + + - name: Install python dependencies + shell: bash + working-directory: ${{ inputs.dir }} + run: pip install -r deploy_requirements.txt + + - name: Get relevant environment configuration from aws secrets + shell: bash + working-directory: ${{ inputs.dir }} + env: + SECRET_SSM_NAME: ${{ inputs.env_aws_secret_name }} + AWS_DEFAULT_REGION: us-west-2 + run: | + if [[ -z "${{ inputs.script_path }}" ]]; then + ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} + else + echo ${{ inputs.auth_stack_name}} + echo ${{ inputs.backend_stack_name}} + python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }} --stack-names ${{ inputs.auth_stack_name}},${{ inputs.backend_stack_name}} + source .env + echo "PREFIX=data-pipeline-$STAGE" >> ${{ inputs.env-file }} + cat .env + fi + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.3.3 + + - name: Deploy + shell: bash + working-directory: ${{ inputs.dir }} + run: | + ./scripts/deploy.sh ${{ inputs.env-file }} <<< init + ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 372c8a64..88696254 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,91 +1,77 @@ -name: CI/CD +name: CICD 🚀 + +permissions: + id-token: write + contents: read on: push: + branches: + - main + - dev + - production pull_request: + branches: + - main + - dev + - production types: [ opened, reopened, edited, synchronize ] jobs: gitflow-enforcer: - runs-on: ubuntu-latest - steps: - - name: Check branch - run: | - if [[ $GITHUB_BASE_REF == "main" ]]; then - if [[ $GITHUB_HEAD_REF != "dev" && $GITHUB_HEAD_REF != "revert-"* ]]; then - echo "ERROR: You can only merge to 'main' from 'dev' or a 'revert-*' branch" - exit 1 - fi - elif [[ $GITHUB_BASE_REF == "production" ]]; then - if [[ $GITHUB_HEAD_REF != "main" && $GITHUB_HEAD_REF != "revert-"* ]]; then - echo "ERROR: You can only merge to 'production' from 'main' or a 'revert-*' branch" - exit 1 - fi - fi - - - run-linters: - name: Run linters + name: GitFlow Enforcer 👮‍ runs-on: ubuntu-latest - needs: gitflow-enforcer + steps: + - name: Check branch + if: github.base_ref == 'main' && github.head_ref != 'dev' || github.base_ref == 'production' && github.head_ref != 'main' + run: | + echo "ERROR: You can only merge to main from dev and to production from main" + exit 1 + define-environment: + name: Set ✨ environment ✨ + needs: gitflow-enforcer + runs-on: ubuntu-latest steps: - - name: Check out Git repository - uses: actions/checkout@v2 + - name: Set the environment based on the branch + id: define_environment + run: | + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "env_name=staging" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/production" ]; then + echo "env_name=production" >> $GITHUB_OUTPUT + fi + - name: Print the environment + run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} - - name: Install Python dependencies - run: pip install black flake8 + deploy: + name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀 + runs-on: ubuntu-latest + if: ${{ needs.define-environment.outputs.env_name }} + needs: [gitflow-enforcer, define-environment] + environment: ${{ needs.define-environment.outputs.env_name }} + concurrency: ${{ needs.define-environment.outputs.env_name }} - - name: Run linters - uses: wearerequired/lint-action@v2 + steps: + - name: Checkout + uses: actions/checkout@v3 with: - continue_on_error: false - black: true - flake8: true - flake8_args: "--ignore E1,E2,E3,E5,W1,W2,W3,W5" # black already handles formatting, this prevents conflicts - - deploy-to-dev: - needs: run-linters - if: github.ref_name == 'dev' - concurrency: development - uses: "./.github/workflows/deploy.yml" - with: - environment: development - env-file: ".env_dev" - stage: "dev" - role-session-name: "veda-data-airflow-github-development-deployment" - aws-region: "us-west-2" - - secrets: inherit - - deploy-to-staging: - needs: run-linters - if: github.ref_name == 'main' - concurrency: staging - uses: "./.github/workflows/deploy.yml" - with: - environment: staging - env-file: ".env_staging" - stage: "staging" - role-session-name: "veda-data-airflow-github-staging-deployment" - aws-region: "us-west-2" - secrets: inherit - - deploy-to-production: - needs: run-linters - if: github.ref_name == 'production' - concurrency: production - uses: "./.github/workflows/deploy.yml" - with: - environment: production - env-file: ".env_prod" - stage: "production" - role-session-name: "veda-data-airflow-github-production-deployment" - aws-region: "us-west-2" + lfs: "true" + submodules: "recursive" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "veda-airflow-github-${{ needs.define-environment.outputs.env_name }}-deployment" + aws-region: "us-west-2" - secrets: inherit + - name: Run deployment + uses: "./.github/actions/terraform-deploy" + with: + env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 582ebc08..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Deploy - -permissions: - id-token: write - contents: read - -on: - workflow_call: - inputs: - stage: - type: string - required: true - env-file: - type: string - required: true - environment: - type: string - required: true - aws-region: - type: string - required: true - role-session-name: - required: false - type: string - default: github-actions-deployment - secrets: - DEPLOYMENT_ROLE_ARN: - required: true - AWS_ACCOUNT_ID: - required: true - VPC_ID: - required: true - ASSUME_ROLE_ARNS: - required: true - VECTOR_VPC: - required: true - VECTOR_SECURITY_GROUP: - required: true - VECTOR_SECRET_NAME: - required: true - - - -jobs: - deploy: - runs-on: ubuntu-latest - environment: ${{ inputs.environment }} - env: - STAGE: ${{ inputs.stage }} - AWS_REGION: ${{ inputs.aws-region }} - VPC_ID: ${{ secrets.VPC_ID }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - ASSUME_ROLE_ARNS: ${{ secrets.ASSUME_ROLE_ARNS }} - COGNITO_APP_SECRET: ${{ secrets.COGNITO_APP_SECRET }} - VECTOR_VPC: ${{ secrets.VECTOR_VPC }} - VECTOR_SECURITY_GROUP: ${{ secrets.VECTOR_SECURITY_GROUP }} - VECTOR_SECRET_NAME: ${{ secrets.VECTOR_SECRET_NAME }} - STAC_INGESTOR_API_URL: ${{ vars.STAC_INGESTOR_API_URL }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - lfs: "true" - submodules: "recursive" - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - run: pip install -r deploy_requirements.txt - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: ${{ inputs.role-session-name }} - aws-region: ${{ inputs.aws-region }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: 1.3.3 - - - name: Deploy - run: | - ./scripts/deploy.sh ${{ inputs.env-file }} <<< init - ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy diff --git a/dags/requirements-constraints.txt b/dags/requirements-constraints.txt index 8ecc3eca..27a3ef26 100644 --- a/dags/requirements-constraints.txt +++ b/dags/requirements-constraints.txt @@ -631,4 +631,3 @@ zipp==3.10.0 zope.event==4.5.0 zope.interface==5.5.1 zstandard==0.19.0 - diff --git a/dags/veda_data_pipeline/groups/discover_group.py b/dags/veda_data_pipeline/groups/discover_group.py index 63176509..c6d5dcd1 100644 --- a/dags/veda_data_pipeline/groups/discover_group.py +++ b/dags/veda_data_pipeline/groups/discover_group.py @@ -20,7 +20,7 @@ def discover_from_s3_task(ti): config = ti.dag_run.conf # (event, chunk_size=2800, role_arn=None, bucket_output=None): MWAA_STAC_CONF = Variable.get("MWAA_STACK_CONF", deserialize_json=True) - read_assume_arn = Variable.get("ASSUME_ROLE_READ_ARN") + read_assume_arn = Variable.get("ASSUME_ROLE_READ_ARN", default_var=None) return s3_discovery_handler( event=config, role_arn=read_assume_arn, diff --git a/dags/veda_data_pipeline/groups/processing_group.py b/dags/veda_data_pipeline/groups/processing_group.py index 5670a3af..e1f75766 100644 --- a/dags/veda_data_pipeline/groups/processing_group.py +++ b/dags/veda_data_pipeline/groups/processing_group.py @@ -49,7 +49,7 @@ def subdag_process(): overrides={ "containerOverrides": [ { - "name": f"{mwaa_stack_conf.get('PREFIX')}-veda-build_stac", + "name": f"{mwaa_stack_conf.get('PREFIX')}-veda-stac-build", "command": [ "/usr/local/bin/python", "handler.py", @@ -59,7 +59,7 @@ def subdag_process(): "environment": [ { "name": "EXTERNAL_ROLE_ARN", - "value": Variable.get("ASSUME_ROLE_READ_ARN"), + "value": Variable.get("ASSUME_ROLE_READ_ARN", default_var=''), }, { "name": "BUCKET", @@ -82,7 +82,7 @@ def subdag_process(): }, }, awslogs_group=mwaa_stack_conf.get("LOG_GROUP_NAME"), - awslogs_stream_prefix=f"ecs/{mwaa_stack_conf.get('PREFIX')}-veda-build_stac", # prefix with container name + awslogs_stream_prefix=f"ecs/{mwaa_stack_conf.get('PREFIX')}-veda-stac-build", # prefix with container name ) submit_to_stac_ingestor = PythonOperator( task_id="submit_to_stac_ingestor", diff --git a/dags/veda_data_pipeline/groups/transfer_group.py b/dags/veda_data_pipeline/groups/transfer_group.py index 62065d7e..585a3aa3 100644 --- a/dags/veda_data_pipeline/groups/transfer_group.py +++ b/dags/veda_data_pipeline/groups/transfer_group.py @@ -25,7 +25,7 @@ def cogify_choice(ti): def transfer_data(ti): """Transfer data from one S3 bucket to another; s3 copy, no need for docker""" config = ti.dag_run.conf - role_arn = Variable.get("ASSUME_ROLE_READ_ARN") + role_arn = Variable.get("ASSUME_ROLE_READ_ARN", default_var="") # (event, chunk_size=2800, role_arn=None, bucket_output=None): return data_transfer_handler(event=config, role_arn=role_arn) @@ -66,7 +66,9 @@ def subdag_transfer(): "environment": [ { "name": "EXTERNAL_ROLE_ARN", - "value": Variable.get("ASSUME_ROLE_READ_ARN"), + "value": Variable.get( + "ASSUME_ROLE_READ_ARN", default_var="" + ), }, ], "memory": 2048, diff --git a/dags/veda_data_pipeline/veda_process_raster_pipeline.py b/dags/veda_data_pipeline/veda_process_raster_pipeline.py index 16bdd83a..2555c6a9 100644 --- a/dags/veda_data_pipeline/veda_process_raster_pipeline.py +++ b/dags/veda_data_pipeline/veda_process_raster_pipeline.py @@ -20,7 +20,7 @@ "discovery": "s3", "datetime_range": "month", "discovered": 33, - "payload": "s3://veda-uah-sit-mwaa-853558080719/events/geoglam/s3_discover_output_6c46b57a-7474-41fe-977a-19d164531cdc.json" + "payload": "s3://veda-uah-sit-mwaa-853558080719/events/geoglam/s3_discover_output_6c46b57a-7474-41fe-977a-.json" } ``` - [Supports linking to external content](https://github.com/NASA-IMPACT/veda-data-pipelines) @@ -36,6 +36,7 @@ "payload": " pystac.Item: @@ -89,12 +101,23 @@ def generate_stac(event: events.RegexEvent) -> pystac.Item: rasterio_kwargs = {} rasterio_kwargs["session"] = get_sts_session() with rasterio.Env( - session=rasterio_kwargs["session"], + session=rasterio_kwargs.get("session"), options={**rasterio_kwargs}, ): + bboxes = [] for asset_name, asset_definition in event.assets.items(): with rasterio.open(asset_definition["href"]) as src: + # Get BBOX and Footprint + dataset_geom = stac.get_dataset_geom(src, densify_pts=0, precision=-1) + bboxes.append(dataset_geom["bbox"]) + media_type = stac.get_media_type(src) + proj_info = { + f"proj:{name}": value + for name, value in stac.get_projection_info(src).items() + } + raster_info = {"raster:bands": stac.get_raster_info(src, max_size=1024)} + # The default asset name for cogs is "cog_default", so we need to intercept 'default' if asset_name == "default": asset_name = "cog_default" @@ -103,10 +126,16 @@ def generate_stac(event: events.RegexEvent) -> pystac.Item: description=asset_definition["description"], href=asset_definition["href"], media_type=media_type, - roles=[], + roles=["data", "layer"], + extra_fields={**proj_info, **raster_info}, ) + + minx, miny, maxx, maxy = zip(*bboxes) + bbox = [min(minx), min(miny), max(maxx), max(maxy)] + create_item_response = create_item( item_id=event.item_id, + bbox=bbox, properties=properties, datetime=single_datetime, collection=event.collection, diff --git a/docker_tasks/cogify_transfer/handler.py b/docker_tasks/cogify_transfer/handler.py index 1f233658..e0c8036f 100644 --- a/docker_tasks/cogify_transfer/handler.py +++ b/docker_tasks/cogify_transfer/handler.py @@ -49,13 +49,14 @@ def transfer_file(s3_client, file_key, local_file_path, destination_bucket, coll def cogify_transfer_handler(event, context): - external_role_arn = os.environ["EXTERNAL_ROLE_ARN"] - creds = assume_role(external_role_arn, "veda-data-pipelines_data-transfer") - kwargs = { - "aws_access_key_id": creds["AccessKeyId"], - "aws_secret_access_key": creds["SecretAccessKey"], - "aws_session_token": creds["SessionToken"], - } + kwargs = {} + if external_role_arn := os.environ["EXTERNAL_ROLE_ARN"]: + creds = assume_role(external_role_arn, "veda-data-pipelines_data-transfer") + kwargs = { + "aws_access_key_id": creds["AccessKeyId"], + "aws_secret_access_key": creds["SecretAccessKey"], + "aws_session_token": creds["SessionToken"], + } source_s3 = boto3.client("s3") target_s3 = boto3.client("s3", **kwargs) diff --git a/docker_tasks/cogify_transfer/requirements.txt b/docker_tasks/cogify_transfer/requirements.txt index 1a9fc916..56e091b1 100644 --- a/docker_tasks/cogify_transfer/requirements.txt +++ b/docker_tasks/cogify_transfer/requirements.txt @@ -3,7 +3,7 @@ awslambdaric boto3 pystac==1.4.0 python-cmr -rasterio==1.3.0 +rasterio==1.3.3 rio-cogeo==4.0.0 shapely smart-open==6.3.0 diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl index 4bb1efc0..61c6933c 100644 --- a/infrastructure/.terraform.lock.hcl +++ b/infrastructure/.terraform.lock.hcl @@ -2,48 +2,44 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/archive" { - version = "2.3.0" + version = "2.4.0" hashes = [ - "h1:NaDbOqAcA9d8DiAS5/6+5smXwN3/+twJGb3QRiz6pNw=", - "h1:OmE1tPjiST8iQp6fC0N3Xzur+q2RvgvD7Lz0TpKSRBw=", - "h1:pTPG9Kf1Qg2aPsZLXDa6OvLqsEXaMrKnp0Z4Q/TIBPA=", - "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", - "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", + "h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=", + "zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594", + "zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b", + "zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff", + "zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", - "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", - "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", - "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", - "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", - "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", - "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", - "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", - "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", + "zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c", + "zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44", + "zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410", + "zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18", + "zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc", + "zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df", + "zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f", ] } provider "registry.terraform.io/hashicorp/aws" { - version = "4.58.0" + version = "4.67.0" constraints = "~> 4.0, >= 4.54.0" hashes = [ - "h1:YIRXIr1ji0HLWLU0ae+UbUNOHc9MJaLrMHxH3LIQ/Vk=", - "h1:xXjZy36R+YOFyLjuF+rgi0NDLwnkFwrJ2t9NfsjRM/E=", - "h1:znLROwEAINbYzAG5X7Ep04whM7KxkQGrvhFdhSvNKEk=", - "zh:14b2b2dfbc7ee705c412d762b1485ee08958c816a64ac74f5769e946e4a1d265", - "zh:17a37e6825e2023b18987d31c0cbb9336654ea146b68e6c90710ea4636af71ae", - "zh:273127c69fb244577e5c136c46164d34f77b0c956c18d27f63d1072dd558f924", - "zh:4b2b6416d34fb3e1051c99d2a84045b136976140e34381d5fbf90e32db15272e", - "zh:7e6a8571ff15d51f892776265642ee01004b8553fd4f6f2014b6f3f2834670c7", - "zh:847c76ab2381b66666d0f79cf1ac697b5bfd0d9c3009fd11bc6ad6545d1eb427", - "zh:9a52cae08ba8d27d0639a8d2b8c61591027883058bf0cc5a639cffe1e299f019", + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9df647e8322d6f94f1843366ba39d21c4b36c8e7dcdc03711d52e27f73b0e974", - "zh:9e52037e68409802ff913b166c30e3f2035af03865cbef0c1b03762bce853941", - "zh:a30288e7c3c904d6998d1709835d7c5800a739f8608f0837f960286a2b8b6e59", - "zh:a7f24e3bda3be566468e4ad62cef1016f68c6f5a94d2e3e979485bc05626281b", - "zh:ba326ba80f5e39829b67a6d1ce54ba52b171e5e13a0a91ef5f9170a9b0cc9ce4", - "zh:c4e3fe9f2be6e244a3dfce599f4b0be9e8fffaece64cbc65f3195f825f65489b", - "zh:f20a251af37039bb2c7612dbd2c5df3a25886b4cc78f902385a2850ea6e30d08", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", ] } @@ -51,8 +47,6 @@ provider "registry.terraform.io/hashicorp/external" { version = "2.3.1" constraints = ">= 1.0.0" hashes = [ - "h1:9rJggijNdRdFk//ViQPGZdK0xu9XU/9qBDijNsZJMg0=", - "h1:bROCw6g5D/3fFnWeJ01L4IrdnJl1ILU8DGDgXCtYzaY=", "h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=", "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", @@ -73,8 +67,6 @@ provider "registry.terraform.io/hashicorp/local" { version = "2.4.0" constraints = ">= 1.0.0" hashes = [ - "h1:Bs7LAkV/iQTLv72j+cTMrvx2U3KyXrcVHaGbdns1NcE=", - "h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=", "h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=", "zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9", "zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf", @@ -95,8 +87,6 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.1" constraints = ">= 2.0.0" hashes = [ - "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", - "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", diff --git a/infrastructure/custom_policies/main.tf b/infrastructure/custom_policies/main.tf index 80930972..1f2ebfb9 100644 --- a/infrastructure/custom_policies/main.tf +++ b/infrastructure/custom_policies/main.tf @@ -64,36 +64,24 @@ data "aws_iam_policy_document" "mwaa_executor_policies" { } statement { - effect = "Allow" + effect = length(var.assume_role_arns) > 0 ? "Allow" : "Deny" actions = [ "sts:AssumeRole" ] - resources = var.assume_role_arns + resources = length(var.assume_role_arns) > 0 ? var.assume_role_arns: ["*"] } + statement { effect = "Allow" actions = [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*" + "s3:Copy*", + "s3:Put*", ] resources = [ - "arn:aws:s3:::veda-data-pipelines-staging-lambda-ndjson-bucket", - "arn:aws:s3:::veda-data-pipelines-staging-lambda-ndjson-bucket/*", - "arn:aws:s3:::veda-data-read-staging", - "arn:aws:s3:::veda-data-read-staging/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::nex-gddp-cmip6-cog", - "arn:aws:s3:::nex-gddp-cmip6-cog/*", - + "arn:aws:s3:::*", ] } @@ -105,14 +93,7 @@ data "aws_iam_policy_document" "mwaa_executor_policies" { "s3:List*" ] resources = [ - "arn:aws:s3:::climatedashboard-data", - "arn:aws:s3:::climatedashboard-data/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::nasa-maap-data-store", - "arn:aws:s3:::nasa-maap-data-store/*", - "arn:aws:s3:::covid-eo-blackmarble", - "arn:aws:s3:::covid-eo-blackmarble/*" + "*", ] } @@ -120,9 +101,7 @@ data "aws_iam_policy_document" "mwaa_executor_policies" { effect = "Allow" actions = ["airflow:CreateCliToken"] resources = [var.mwaa_arn] - } - } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 10d2d09c..5c1a6ab9 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -1,9 +1,9 @@ module "mwaa" { - source = "https://github.com/NASA-IMPACT/mwaa_tf_module/releases/download/v1.1.5/mwaa_tf_module.zip" + source = "https://github.com/NASA-IMPACT/mwaa_tf_module/releases/download/v1.1.7.0/mwaa_tf_module.zip" prefix = var.prefix vpc_id = var.vpc_id iam_role_additional_arn_policies = merge(module.custom_policy.custom_policy_arns_map) - permissions_boundary_arn = var.iam_role_permissions_boundary + permissions_boundary_arn = var.iam_policy_permissions_boundary_name == "null" ? null : "arn:aws:iam::${local.account_id}:policy/${var.iam_policy_permissions_boundary_name}" subnet_tagname = var.subnet_tagname local_requirement_file_path = "${path.module}/../dags/requirements.txt" local_dag_folder = "${path.module}/../dags/" @@ -45,11 +45,10 @@ module "custom_policy" { vector_secret_name = var.vector_secret_name } - data "aws_subnets" "private" { filter { name = "vpc-id" - values = [var.vector_vpc] + values = [var.vector_vpc == null ? "" : var.vector_vpc] } tags = { @@ -58,6 +57,7 @@ data "aws_subnets" "private" { } resource "aws_security_group" "vector_sg" { + count = var.vector_vpc == "null" ? 0 : 1 name = "${var.prefix}_veda_vector_sg" vpc_id = var.vector_vpc @@ -71,12 +71,13 @@ resource "aws_security_group" "vector_sg" { } resource "aws_vpc_security_group_ingress_rule" "vector_rds_ingress" { + count = var.vector_vpc == "null" ? 0 : 1 security_group_id = var.vector_security_group from_port = 5432 to_port = 5432 ip_protocol = "tcp" - referenced_security_group_id = aws_security_group.vector_sg.id + referenced_security_group_id = aws_security_group.vector_sg[count.index].id } resource "local_file" "mwaa_variables" { @@ -95,13 +96,7 @@ resource "local_file" "mwaa_variables" { aws_region = local.aws_region cognito_app_secret = var.cognito_app_secret stac_ingestor_api_url = var.stac_ingestor_api_url - assume_role_read_arn = var.assume_role_arns[0] - assume_role_write_arn = var.assume_role_arns[1] vector_secret_name = var.vector_secret_name - vector_subnet_1 = data.aws_subnets.private.ids[0] - vector_subnet_2 = data.aws_subnets.private.ids[1] - vector_security_group = aws_security_group.vector_sg.id - vector_vpc = var.vector_vpc }) filename = "/tmp/mwaa_vars.json" } @@ -113,12 +108,12 @@ resource "local_file" "mwaa_variables" { # ECR repository to host workflows API image resource "aws_ecr_repository" "workflows_api_lambda_repository" { name = "veda_${var.stage}_workflows-api-lambda-repository" -} +} resource "null_resource" "if_change_run_provisioner" { triggers = { - always_run = "${timestamp()}" - } + always_run = "${timestamp()}" + } provisioner "local-exec" { command = <<-EOT set -e @@ -131,7 +126,8 @@ resource "null_resource" "if_change_run_provisioner" { # IAM Role for Lambda Execution resource "aws_iam_role" "lambda_execution_role" { - name = "veda_${var.stage}_lambda_execution_role" + name = "veda_${var.stage}_lambda_execution_role" + permissions_boundary = var.iam_policy_permissions_boundary_name == "null" ? null : "arn:aws:iam::${local.account_id}:policy/${var.iam_policy_permissions_boundary_name}" assume_role_policy = jsonencode({ Version = "2012-10-17", @@ -147,28 +143,51 @@ resource "aws_iam_role" "lambda_execution_role" { }) } -resource "aws_iam_policy" "ecr_access" { - name = "veda_${var.stage}_ECR_Access_For_Lambda" +resource "aws_iam_policy" "lambda_access" { + name = "veda_${var.stage}_Access_For_Lambda" path = "/" - description = "ECR access policy for Lambda function" - policy = jsonencode({ + description = "Access policy for Lambda function" + policy = jsonencode({ Version = "2012-10-17", Statement = [ { - Effect = "Allow", - Action = [ + Effect = "Allow", + Action = [ "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", + "ecr:BatchGetImage" + ], + Resource = [ + "${aws_ecr_repository.workflows_api_lambda_repository.arn}", + ], + }, + { + Effect = "Allow", + Action = [ + "secretsmanager:GetSecretValue" + ], + Resource = [ + "arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:${var.workflows_client_secret}*" ], - Resource = "${aws_ecr_repository.workflows_api_lambda_repository.arn}", }, + { + Effect: "Allow", + Action: "sts:AssumeRole", + Resource: var.data_access_role_arn + }, + { + Action: "airflow:CreateCliToken", + Resource: [ + "arn:aws:airflow:${var.aws_region}:${local.account_id}:environment/veda-pipeline-${var.stage}-mwaa" + ], + Effect: "Allow" + } ], }) } -resource "aws_iam_role_policy_attachment" "ecr_access_attach" { +resource "aws_iam_role_policy_attachment" "lambda_access_attach" { role = aws_iam_role.lambda_execution_role.name - policy_arn = aws_iam_policy.ecr_access.arn + policy_arn = aws_iam_policy.lambda_access.arn } resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { @@ -180,23 +199,19 @@ resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { resource "aws_lambda_function" "workflows_api_handler" { function_name = "veda_${var.stage}_workflows_api_handler" role = aws_iam_role.lambda_execution_role.arn - package_type = "Image" - timeout = 30 - + package_type = "Image" + timeout = 30 image_uri = "${aws_ecr_repository.workflows_api_lambda_repository.repository_url}:latest" environment { variables = { - JWKS_URL = var.jwks_url - USERPOOL_ID = var.cognito_userpool_id - CLIENT_ID = var.cognito_client_id - CLIENT_SECRET = var.cognito_client_secret - STAGE = var.stage - DATA_ACCESS_ROLE_ARN = var.data_access_role_arn - WORKFLOW_ROOT_PATH = var.workflow_root_path - INGEST_URL = var.ingest_url - RASTER_URL = var.raster_url - STAC_URL = var.stac_url - MWAA_ENV = module.mwaa.airflow_url + WORKFLOWS_CLIENT_SECRET_ID = var.workflows_client_secret + STAGE = var.stage + DATA_ACCESS_ROLE_ARN = var.data_access_role_arn + WORKFLOW_ROOT_PATH = var.workflow_root_path + INGEST_URL = var.stac_ingestor_api_url + RASTER_URL = var.raster_url + STAC_URL = var.stac_url + MWAA_ENV = "veda-pipeline-${var.stage}-mwaa" } } } @@ -210,9 +225,9 @@ resource "aws_apigatewayv2_api" "workflows_http_api" { # Lambda Integration for API Gateway resource "aws_apigatewayv2_integration" "workflows_lambda_integration" { - api_id = aws_apigatewayv2_api.workflows_http_api.id - integration_type = "AWS_PROXY" - integration_uri = aws_lambda_function.workflows_api_handler.invoke_arn + api_id = aws_apigatewayv2_api.workflows_http_api.id + integration_type = "AWS_PROXY" + integration_uri = aws_lambda_function.workflows_api_handler.invoke_arn payload_format_version = "2.0" } diff --git a/infrastructure/mwaa_environment_variables.tpl b/infrastructure/mwaa_environment_variables.tpl index 8f8a2810..2c587871 100644 --- a/infrastructure/mwaa_environment_variables.tpl +++ b/infrastructure/mwaa_environment_variables.tpl @@ -11,13 +11,7 @@ "ACCOUNT_ID": "${account_id}", "AWS_REGION": "${aws_region}" }, - "ASSUME_ROLE_READ_ARN": "${assume_role_read_arn}", - "ASSUME_ROLE_WRITE_ARN": "${assume_role_write_arn}", "COGNITO_APP_SECRET": "${cognito_app_secret}", "STAC_INGESTOR_API_URL": "${stac_ingestor_api_url}", - "VECTOR_SECRET_NAME": "${vector_secret_name}", - "VECTOR_ECS_CONF":{ - "VECTOR_SECURITY_GROUP": ["${vector_security_group}"], - "VECTOR_SUBNETS": ["${vector_subnet_1}", "${vector_subnet_2}"] - } -} \ No newline at end of file + "VECTOR_SECRET_NAME": "${vector_secret_name}" +} diff --git a/infrastructure/outputs.tf b/infrastructure/outputs.tf index 272ecc69..e33739d7 100644 --- a/infrastructure/outputs.tf +++ b/infrastructure/outputs.tf @@ -7,3 +7,9 @@ output "mwaa_s3_name" { output "mwaa_subnets" { value = module.mwaa.subnets } +output "airflow_env" { + value = module.mwaa.mwaa_environment_name +} +output "workflows_api" { + value = aws_apigatewayv2_api.workflows_http_api.api_endpoint +} \ No newline at end of file diff --git a/infrastructure/task_definition.tf b/infrastructure/task_definition.tf index f11f8abe..e63e679f 100644 --- a/infrastructure/task_definition.tf +++ b/infrastructure/task_definition.tf @@ -4,7 +4,7 @@ resource "aws_ecs_task_definition" "veda_task_definition" { container_definitions = jsonencode([ { - name = "${var.prefix}-veda-build_stac" + name = "${var.prefix}-veda-stac-build" image = "${local.account_id}.dkr.ecr.${local.aws_region}.amazonaws.com/${var.prefix}-veda-build_stac" essential = true, logConfiguration = { diff --git a/infrastructure/terraform.tfvars.tmpl b/infrastructure/terraform.tfvars.tmpl index d18c2548..4d12ddbd 100644 --- a/infrastructure/terraform.tfvars.tmpl +++ b/infrastructure/terraform.tfvars.tmpl @@ -1,22 +1,18 @@ -aws_profile="${AWS_PROFILE}" aws_region="${AWS_REGION}" prefix="${PREFIX}" vpc_id="${VPC_ID}" subnet_tagname="${SUBNET_TAGNAME}" -iam_role_permissions_boundary="${IAM_ROLE_PERMISSIONS_BOUNDARY}" +iam_policy_permissions_boundary_name="${PERMISSIONS_BOUNDARY_POLICY_NAME:-null}" stage="${STAGE:-dev}" assume_role_arns=$ASSUME_ROLE_ARNS -cognito_app_secret="${COGNITO_APP_SECRET}" -stac_ingestor_api_url="${STAC_INGESTOR_API_URL}" +cognito_app_secret="${VEDA_PROGRAMMATIC_CLIENT_SECRET_ID}" +workflows_client_secret="${VEDA_WORKFLOWS_CLIENT_SECRET_ID}" +stac_ingestor_api_url="${VEDA_STAC_INGESTOR_API_URL}" +data_access_role_arn="${VEDA_DATA_ACCESS_ROLE_ARN}" +raster_url="${VEDA_RASTER_URL}" +stac_url="${VEDA_STAC_URL}" vector_secret_name="${VECTOR_SECRET_NAME}" vector_security_group="${VECTOR_SECURITY_GROUP}" -vector_vpc="${VECTOR_VPC}" -data_access_role_arn="${DATA_ACCESS_ROLE_ARN}" +vector_vpc="${VECTOR_VPC:-null}" workflow_root_path="${WORKFLOW_ROOT_PATH}" -ingest_url="${INGEST_URL}" -raster_url="${RASTER_URL}" -stac_url="${STAC_URL}" cloudfront_id="${CLOUDFRONT_ID}" -jwks_url="${JWKS_URL}" -cognito_userpool_id="${COGNITO_USERPOOL_ID}" -cognito_client_id="${COGNITO_CLIENT_ID}" diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index c3ab0085..c4101409 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -11,8 +11,9 @@ variable "prefix" { description = "Deployment prefix" } -variable "iam_role_permissions_boundary" { +variable "iam_policy_permissions_boundary_name" { description = "Permission boundaries" + default = null } variable "assume_role_arns" { @@ -37,21 +38,7 @@ variable "cognito_app_secret" { type = string } -variable "jwks_url" { - type = string -} - -variable "cognito_userpool_id" { - type = string -} - -variable "cognito_client_id" { - type = string -} - -variable "cognito_client_secret" { - description = "Optional secret, if required by cognito user pool" - default = "" +variable "workflows_client_secret" { type = string } @@ -73,22 +60,14 @@ variable "vector_security_group" { type = string } variable "vector_vpc" { - type = string + type = string + default = "null" } variable "data_access_role_arn" { type = string } -variable "workflow_root_path" { - type = string - default = "/api/workflows" -} - -variable "ingest_url" { - type = string -} - variable "raster_url" { type = string } @@ -97,6 +76,11 @@ variable "stac_url" { type = string } +variable "workflow_root_path" { + type = string + default = "/api/workflows" +} + variable "cloudfront_id" { type = string } diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh new file mode 100755 index 00000000..3723b75c --- /dev/null +++ b/scripts/sync-env.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Use this script to load environment variables for a deployment from AWS Secrets + +for s in $(aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do + echo "$s" >> $GITHUB_ENV + echo "$s" >> .env +done +source .env +export PREFIX=veda-pipeline-${STAGE} + +cat << EXPORT_ENVS >> .env +PREFIX=$PREFIX +AWS_REGION=us-west-2 +STATE_BUCKET_NAME=${PREFIX}-tf-state-shared +STATE_BUCKET_KEY=veda-mwaa/${PREFIX}-mwaa/terraform.tfstate +STATE_DYNAMO_TABLE=${PREFIX}-shared-state-mwaa-lock-state +EXPORT_ENVS diff --git a/workflows_api/runtime/src/auth.py b/workflows_api/runtime/src/auth.py index 5975cfa7..f4bafd3c 100644 --- a/workflows_api/runtime/src/auth.py +++ b/workflows_api/runtime/src/auth.py @@ -1,10 +1,5 @@ -import base64 -import hashlib -import hmac import logging -from typing import Dict -import boto3 import requests import src.config as config from authlib.jose import JsonWebKey, JsonWebToken, JWTClaims, KeySet, errors @@ -24,7 +19,13 @@ def get_settings() -> config.Settings: def get_jwks_url(settings: config.Settings = Depends(get_settings)) -> str: - return settings.jwks_url + import boto3 + import json + client = boto3.client("secretsmanager") + response = client.get_secret_value(SecretId=settings.workflows_client_secret_id) + secrets = json.loads(response["SecretString"]) + + return f"https://cognito-idp.{secrets['aws_region']}.amazonaws.com/{secrets['userpool_id']}/.well-known/jwks.json" @cached(TTLCache(maxsize=1, ttl=3600)) @@ -56,7 +57,7 @@ def decode_token( claims.setdefault("aud", claims["client_id"]) claims.validate() - return claims, token + return claims except errors.JoseError: # logger.exception("Unable to decode token") raise HTTPException(status_code=403, detail="Bad auth token") @@ -66,56 +67,7 @@ def get_username(claims: security.HTTPBasicCredentials = Depends(decode_token)): return claims["sub"] -def get_and_validate_token( +def get_token( token: security.HTTPAuthorizationCredentials = Depends(token_scheme), ): - decode_token(token) return token - - -def _get_secret_hash(username: str, client_id: str, client_secret: str) -> str: - # A keyed-hash message authentication code (HMAC) calculated using - # the secret key of a user pool client and username plus the client - # ID in the message. - message = username + client_id - dig = hmac.new( - bytearray(client_secret, "utf-8"), - msg=message.encode("UTF-8"), - digestmod=hashlib.sha256, - ).digest() - return base64.b64encode(dig).decode() - - -def authenticate_and_get_token( - username: str, - password: str, - user_pool_id: str, - app_client_id: str, - app_client_secret: str, -) -> Dict: - client = boto3.client("cognito-idp") - if app_client_secret: - auth_params = { - "USERNAME": username, - "PASSWORD": password, - "SECRET_HASH": _get_secret_hash(username, app_client_id, app_client_secret), - } - else: - auth_params = { - "USERNAME": username, - "PASSWORD": password, - } - try: - resp = client.admin_initiate_auth( - UserPoolId=user_pool_id, - ClientId=app_client_id, - AuthFlow="ADMIN_USER_PASSWORD_AUTH", - AuthParameters=auth_params, - ) - except client.exceptions.NotAuthorizedException: - return { - "message": "Login failed, please make sure the credentials are correct." - } - except Exception as e: - return {"message": f"Login failed with exception {e}"} - return resp["AuthenticationResult"] diff --git a/workflows_api/runtime/src/collection_publisher.py b/workflows_api/runtime/src/collection_publisher.py index 1b059a81..fb6dbd60 100644 --- a/workflows_api/runtime/src/collection_publisher.py +++ b/workflows_api/runtime/src/collection_publisher.py @@ -21,7 +21,7 @@ def ingest(self, collection: DashboardCollection, token: str, ingest_api: str): does necessary preprocessing, and loads into the PgSTAC collection table """ - collection = collection.model_dump(by_alias=True) + collection = collection.json(by_alias=True) url = f"{ingest_api}/collections" headers = { @@ -161,7 +161,7 @@ def ingest(self, collection: DashboardCollection, token: str, ingest_api: str): does necessary preprocessing, and loads into the PgSTAC collection table """ - collection = collection.model_dump(by_alias=True) + collection = collection.json(by_alias=True) url = f"{ingest_api}/collections" headers = { diff --git a/workflows_api/runtime/src/config.py b/workflows_api/runtime/src/config.py index 7e05179a..05611da0 100644 --- a/workflows_api/runtime/src/config.py +++ b/workflows_api/runtime/src/config.py @@ -1,19 +1,15 @@ -from pydantic import AnyHttpUrl, BaseSettings, Field, constr +from pydantic import BaseSettings, Field, constr AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+") AwsStepArn = constr(regex=r"^arn:aws:states:.+:\d{12}:stateMachine:.+") class Settings(BaseSettings): - jwks_url: AnyHttpUrl = Field( - description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json" # noqa - ) data_access_role_arn: AwsArn = Field( # type: ignore description="ARN of AWS Role used to validate access to S3 data" ) - userpool_id: str = Field(description="The Cognito Userpool used for authentication") - client_id: str = Field(description="The Cognito APP client ID") - client_secret: str = Field("", description="The Cognito APP client secret") + + workflows_client_secret_id: str = Field(description="The Cognito APP Secret that contains cognito creds") stage: str = Field(description="API stage") workflow_root_path: str = Field(description="Root path of API") ingest_url: str = Field(description="URL of ingest API") diff --git a/workflows_api/runtime/src/main.py b/workflows_api/runtime/src/main.py index 0c76d1d5..1b7ef991 100644 --- a/workflows_api/runtime/src/main.py +++ b/workflows_api/runtime/src/main.py @@ -67,7 +67,7 @@ def validate_dataset(dataset: schemas.COGDataset): "/dataset/publish", tags=["Dataset"], dependencies=[Depends(auth.get_username)] ) async def publish_dataset( - token=Depends(auth.get_and_validate_token), + token=Depends(auth.get_token), dataset: Union[schemas.ZarrDataset, schemas.COGDataset] = Body( ..., discriminator="data_type" ), diff --git a/workflows_api/runtime/src/validators.py b/workflows_api/runtime/src/validators.py index 0abee3ba..1541a327 100644 --- a/workflows_api/runtime/src/validators.py +++ b/workflows_api/runtime/src/validators.py @@ -91,3 +91,14 @@ def collection_exists(collection_id: str) -> bool: f"Invalid collection '{collection_id}', received " f"{response.status_code} response code from STAC API" ) + +def time_density_is_valid(is_periodic: bool, time_density: Union[str, None]): + """ + Ensures that the time_density is valid based on the value of is_periodic + """ + if is_periodic and not time_density: + raise ValueError("If is_periodic is true, time_density must be set.") + + # Literal[str, None] doesn't quite work for null field inputs from a dict() + if time_density and time_density not in ["day", "month", "year"]: + raise ValueError("If set, time_density must be one of 'day, 'month' or 'year'") \ No newline at end of file