Merge pull request #259 from NASA-IMPACT/mcp

Remove buckets on destroy
NASA-IMPACT · Oct 20, 2023 · 6bd9923 · 6bd9923
2 parents fa1fd41 + 4e67b57
commit 6bd9923
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 76 deletions.
diff --git a/README.md b/README.md
@@ -1,78 +1,106 @@
-# hls-orchestration
+# HLS Orchestration
 
 AWS Stack for processing HLS data.
 
 ![Alt text](/docs/HLS_architecture.png)
 
-For more detailed data flow diagrams and architecture see [architecture](/docs/architecture.md).
+For more detailed data flow diagrams and architecture see
+[architecture](/docs/architecture.md).
 
-For more details about all of the HLS project's components see [hls-project](https://github.com/nasa-impact/hls-project).
+For more details about all of the HLS project's components see
+[hls-project](https://github.com/nasa-impact/hls-project).
 
+## Requirements
 
-### Requirements
-Python>=3.7 \
-tox \
-aws-cli \
-jq \
-An IAM role with sufficient permissions for creating, destroying and modifying the relevant stack resources.
+- Python>=3.7
+- tox
+- aws-cli
+- jq
+- An IAM role with sufficient permissions for creating, destroying, and
+  modifying the relevant stack resources.
 
-### Environment Settings
-Environment variables are set in `environment.sh`. Copy `environment.sh.sample` to `environment.sh` and update the settings prior to running any commands.  The following variables can be overridden from the calling shell's environment
-```
-$ export HLS_STACKNAME=<Name of your stack>
-$ export HLS_LAADS_TOKEN=<Token used for accessing the Laads Data>
-$ export HLS_SENTINEL_OUTPUT_BUCKET_ROLE_ARN=<GCC Role for accessing output bucket>
+## Environment Settings
+
+Environment variables are set in `environment.sh`.  Copy `environment.sh.sample`
+to `environment.sh` and update the settings prior to running any commands.  The
+following variables can be overridden from the calling shell's environment:
+
+```plain
+export HLS_STACKNAME=<Name of your stack>
+export HLS_LAADS_TOKEN=<Token used for accessing the Laads Data>
+export HLS_SENTINEL_OUTPUT_BUCKET_ROLE_ARN=<GCC Role for accessing output bucket>
 ```
 
-### Synth
+## Synth
+
 Display generated cloud formation template that will be used to deploy.
+
+```plain
+source environment.sh && tox -e dev -r -- synth
 ```
-$ source ./environment.sh && tox -e dev -r -- synth
-```
 
-### Diff
+## Diff
+
 Display a diff of the current deployment and any changes created.
-```
-$ source ./environment.sh && tox -e dev -r -- diff
-```
 
-### Deploy
-Deploy current version of stack.
-```
-$ source ./environment.sh && tox -e dev -r -- deploy
+```plain
+source environment.sh && tox -e dev -r -- diff
 ```
 
-The repository is configured to create automatic deployments to the `hls-development` stack when PRs are merged into the `dev` branch.  This deployment uses [Github Actions Environments](https://docs.github.com/en/actions/reference/environments) to manage the environment configuration rather than the `environment.sh`.
+## Deploy
 
-Deployments to GCC have restrictions over creating VPCs and the types of AMIs which can be utilized. To deploy to GCC your shell will require the following environment settings.
+Deploy current version of stack:
+
+```plain
+source environment.sh && tox -e dev -r -- deploy
 ```
+
+The repository is configured to create automatic deployments to the
+`hls-development` stack when PRs are merged into the `dev` branch.  This
+deployment uses
+[Github Actions Environments](https://docs.github.com/en/actions/reference/environments)
+to manage the environment configuration rather than the `environment.sh`.
+
+Deployments to GCC have restrictions over creating VPCs and the types of AMIs
+which can be utilized.  To deploy to GCC your shell will require the following
+environment settings:
+
+```plain
 export GCC=true
 export AWS_DEFAULT_REGION=us-west-2
 export HLS_GCC_ACCOUNT=<The GCC account id>
 export HLS_GCC_VPCID=<The vpc id provided by GCC administrators>
 export HLS_GCC_BOUNDARY_ARN=<The boundary policy arn>
 ```
 
+## Setup Logging Database
 
-### Setup Logging Database
-After `deploy` is run and the stack is created run
-```
-$ source ./environment.sh && ./scripts/setupdb.sh
+After `deploy` is run and the stack is created run:
+
+```plain
+source environment.sh && scripts/setupdb.sh
 ```
+
 To bootstrap the logging database.
 
-### Development
-For active stack development run
-```
-$ source ./environment.sh && tox -e dev -r -- version
-```
-This creates a local virtualenv in the directory `devenv`.  To use it for development
+## Development
+
+For active stack development run:
+
+```plain
+source environment.sh && tox -e dev -r -- version
 ```
-$ source devenv/bin/activate
+
+This creates a local virtualenv in the directory `devenv`.  To use it for development:
+
+```plain
+source devenv/bin/activate
 ```
 
-### Tests
+## Tests
+
 To run unit test for all included Lambda functions
-```
+
+```plain
 tox -r
 ```
diff --git a/cdk.context.json b/cdk.context.json
@@ -0,0 +1,5 @@
+{
+  "acknowledged-issue-numbers": [
+    19836
+  ]
+}
diff --git a/setup.py b/setup.py
@@ -46,6 +46,7 @@
         "flake8",
         "nodeenv",
         "isort",
+        "mypy",
         "pre-commit",
         "pre-commit-hooks",
     ],

diff --git a/stack/stack.py b/stack/stack.py
@@ -105,32 +105,23 @@ def getenv(key, default):
 except ValueError:
     MAXV_CPUS = 1200
 
-if getenv("HLS_REPLACE_EXISTING", "true") == "true":
-    REPLACE_EXISTING = True
-else:
-    REPLACE_EXISTING = False
-
-if getenv("HLS_USE_CLOUD_WATCH", "false") == "true":
-    USE_CLOUD_WATCH = True
-else:
-    USE_CLOUD_WATCH = False
-
-if getenv("GCC", None) == "true":
-    GCC = True
-else:
-    GCC = False
+REPLACE_EXISTING = getenv("HLS_REPLACE_EXISTING", "true") == "true"
+USE_CLOUD_WATCH = getenv("HLS_USE_CLOUD_WATCH", "false") == "true"
+GCC = getenv("GCC", None) == "true"
 
 
 class HlsStack(core.Stack):
     def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
         super().__init__(scope, id, **kwargs)
+
         if GCC:
+            from permission_boundary import PermissionBoundaryAspect
+
             vpcid = os.environ["HLS_GCC_VPCID"]
             boundary_arn = os.environ["HLS_GCC_BOUNDARY_ARN"]
             image_id = aws_ssm.StringParameter.from_string_parameter_attributes(
                 self, "gcc_ami", parameter_name="/mcp/amis/aml2-ecs"
             ).string_value
-            from permission_boundary import PermissionBoundaryAspect
 
             core.Aspects.of(self).add(PermissionBoundaryAspect(boundary_arn))
         else:
@@ -151,31 +142,38 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
 
         # Must be created as part of the stack due to trigger requirements
         self.sentinel_input_bucket = aws_s3.Bucket(
-            self, "SentinelInputBucket", bucket_name=SENTINEL_INPUT_BUCKET
+            self,
+            "SentinelInputBucket",
+            bucket_name=SENTINEL_INPUT_BUCKET,
+            removal_policy=core.RemovalPolicy.DESTROY,
         )
 
         self.sentinel_input_bucket_historic = aws_s3.Bucket(
             self,
             "SentinelInputBucketHistoric",
             bucket_name=SENTINEL_INPUT_BUCKET_HISTORIC,
+            removal_policy=core.RemovalPolicy.DESTROY,
         )
 
         self.landsat_input_bucket_historic = aws_s3.Bucket(
             self,
             "LandsatInputBucketHistoric",
             bucket_name=LANDSAT_INPUT_BUCKET_HISTORIC,
+            removal_policy=core.RemovalPolicy.DESTROY,
         )
 
         self.landsat_intermediate_output_bucket = aws_s3.Bucket(
             self,
             "LandsatIntermediateBucket",
             bucket_name=LANDSAT_INTERMEDIATE_OUTPUT_BUCKET,
+            removal_policy=core.RemovalPolicy.DESTROY,
         )
 
         self.gibs_intermediate_output_bucket = aws_s3.Bucket(
             self,
             "GibsIntermediateBucket",
             bucket_name=GIBS_INTERMEDIATE_OUTPUT_BUCKET,
+            removal_policy=core.RemovalPolicy.DESTROY,
         )
 
         self.efs = Efs(self, "Efs", network=self.network)

diff --git a/tox.ini b/tox.ini
@@ -6,9 +6,9 @@ extras = test
 envdir = toxenv
 passenv = AWS_DEFAULT_REGION
 commands =
-      pip install -e ./layers/hls_lambda_layer/python
-      python -m pytest --cov=lambda_functions --ignore=node_modules --ignore=cdk.out
-      flake8
+  pip install -e ./layers/hls_lambda_layer/python
+  python -m pytest --cov=lambda_functions --ignore=node_modules --ignore=cdk.out
+  flake8
 
 [cdk]
 extras = dev
@@ -19,22 +19,6 @@ passenv =
 commands =
   nodeenv --node=18.17.1 -p
   npm install -g [email protected]
-  # NOTICES
-  #
-  # 19836	AWS CDK v1 End-of-Support June 1, 2023
-  #
-  #       Overview: AWS CDK v1 is currently in maintenance mode. Support for AWS
-  #                 CDK v1 will end entirely on June 1, 2023. Migrate to AWS CDK
-  #                 v2 to continue to get the latest features and fixes!
-  #
-  #       Affected versions: framework: 1.*, cli: 1.*
-  #
-  #       More information at: https://github.com/aws/aws-cdk/issues/19836
-  #
-  #
-  # If you don’t want to see a notice anymore, use "cdk acknowledge <id>".
-  # For example, "cdk acknowledge 19836".
-  cdk acknowledge 19836
   cdk --version
 
 [testenv:dev]