Merge branch 'main' of github.com:hackforla/incubator into vrms-clien…

…t-terraform-migration

.github/ISSUE_TEMPLATE/blank-issue.md

@@ -2,7 +2,7 @@
 name: Blank Issue
 about: Consistent formatting make Issues concise and easy to navigate
 title: ''
-labels: ''
+labels: 'complexity: missing, feature: missing, role: missing, size: missing'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/custom.md

@@ -6,3 +6,5 @@ labels: ''
 assignees: ''
 
 ---
+
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: 'Feature Suggestion: '
-labels: documentation, product
+labels: documentation
 assignees: ''
 
 ---

.github/workflows/terraform-plan.yml

@@ -0,0 +1,89 @@
+name: PR Terraform Plan
+
+on: [pull_request]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  changed-files:
+    name: Get changed terraform directories
+    runs-on: ubuntu-latest
+    outputs:
+      module-change: ${{ steps.changed-files.outputs.module-change }}
+      project-change: ${{ steps.changed-files.outputs.project-change }}
+      environment-change: ${{ steps.changed-files.outputs.environment-change }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get changed files
+        id: changed-files
+        uses: dorny/paths-filter@v2
+        with:
+          filters: |
+            module-change:
+              - 'terraform-modules/**'
+            project-change:
+              - 'terraform-incubator/*/project/*.tf'
+            environment-change:
+              - 'terraform-incubator/*/!(project)/*.tf'
+          list-files: json
+      - name: List all changed files
+        run: echo '${{ steps.changed-files.outputs.module-change_files }}'; echo '${{ steps.changed-files.outputs.project-change_files }}'; echo '${{ steps.changed-files.outputs.environment-change_files }}'
+  plan-all:
+    runs-on: ubuntu-latest
+    name: Terraform plan - all directories
+    needs: [changed-files]
+    if: ${{ needs.changed-files.outputs.module-change == 'true' }}
+    strategy:
+      matrix:
+        directory: ${{ needs.changed-files.outputs.environment-change }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{secrets.INCUBATOR_AWS_ACCESS_KEY_ID}}
+      AWS_SECRET_ACCESS_KEY: ${{secrets.INCUBATOR_AWS_SECRET_ACCESS_KEY}}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: terraform plan
+        uses: dflook/terraform-plan@v1
+        with:
+          path: ${{ matrix.directory }}
+  plan-project:
+    runs-on: ubuntu-latest
+    name: Terraform plan - Project changes
+    needs: [changed-files]
+    if: ${{ needs.changed-files.outputs.project-change == 'true' && needs.changed-files.outputs.module-change == 'false'}}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{secrets.INCUBATOR_AWS_ACCESS_KEY_ID}}
+      AWS_SECRET_ACCESS_KEY: ${{secrets.INCUBATOR_AWS_SECRET_ACCESS_KEY}}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: terraform plan
+        uses: dflook/terraform-plan@v1
+        with:
+          path: terraform-incubator/people-depot/dev
+  plan-environment:
+    runs-on: ubuntu-latest
+    name: Terraform plan - Env changes
+    needs: [changed-files]
+    if: ${{ needs.changed-files.outputs.environment-change == 'true' && needs.changed-files.outputs.module-change == 'false' && needs.changed-files.outputs.project-change == 'false' }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{secrets.INCUBATOR_AWS_ACCESS_KEY_ID}}
+      AWS_SECRET_ACCESS_KEY: ${{secrets.INCUBATOR_AWS_SECRET_ACCESS_KEY}}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: terraform plan
+        uses: dflook/terraform-plan@v1
+        with:
+          path: terraform-incubator/people-depot/dev

documentation/terraform-add-new-project.md

@@ -0,0 +1,22 @@
+# Adding a new Project to Terraform
+
+* Fork Incubator (if you haven't)
+* Pull main branch
+* Create feature branch
+
+```shell
+> mkdir -p terraform-incubator/{projectname}/project terraform-incubator/{projectname}/dev
+```
+
+...
+
+* Commit
+* Push
+* Create PR to Incubator
+
+```
+aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 035866691871.dkr.ecr.us-west-2.amazonaws.com
+```
+
+
+## ACM?

documentation/terraform-migrate-project.md

@@ -84,7 +84,9 @@ incubator
 This is the listing from a workstation set up to deploy `people-depot-backend`
 with Terragrunt,
 which is why we see a `project.hcl` here.
-To start this process, you'll need the `project.hcl` for the project you're trying to migrate.
+To start this process, you'll need the `project.hcl` for the project you're trying to migrate. 
+
+DISCLAIMER: For some projects that file is hard (impossible?) to locate. You can actually still do the migration without it by reading the Terraform state directly which we'll show later in the documentation but it's **much** easier to do with the `project.hcl` file so reach out to the relevant slack channels to find it.
 
 Inside that file
 (`incubator/projects-dev/people-depot-backend/project.hcl`)
@@ -309,6 +311,23 @@ what value was assigned for a particular module,
 that's the process to the hunt it down in Terragrunt.
 This is one of the reasons we're moving away from it.
 
+If your project does not have a `project.hcl` file or you haven't been able to locate it, you can still get these values by looking at the Terraform state for those resources and pulling them directly.
+Run `terraform show > state.txt` from the directory that you've created for the project and environment within terraform-incubator (i.e. terraform-incubator/people-depot/dev). In order for that command to actually pull the state down, you will need to have configured the provider and backend blocks within the `main.tf` file as described [here](#per-environment-configs):
+```terraform
+terraform {
+  backend "s3" {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/projects-{ENV}/{PROJECT}/terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+provider "aws" {
+  region = "us-west-2"
+}
+```
+You will need to locate the state file and fill in the correct `key`. Usually it's straightforward and you can just fill in the `ENV` and `PROJECT` but you can also check the location in S3 yourself in this bucket - s3://hlfa-incubator-terragrunt/terragrunt-states/incubator/. Once you get this confiugred properly and pointing at the right state, run `terraform init` and then `terraform show > state.txt` which should populate the text file with the resources that are currently stored in that state. From there, you can back into the values you need from the `project.hcl` by looking at the resource configuration in the state.
 
 getting back to our `project/main.tf` -
 we'll skip a bunch of values we can collect this way,

flake.nix

@@ -17,6 +17,7 @@
         terraform
         terragrunt
         tfautomv
+        ssm-session-manager-plugin
       ];
       GIT_TEMPLATE_DIR="";
     };

terraform-incubator/access-the-data/main.tf

@@ -0,0 +1,148 @@
+locals {
+  // we use tf to create the zone, but other projects might
+  // have an existing zone and get it with a data block
+  zone_id = module.zone.zone_id
+
+  envs = {
+    dev = {
+      environment = "dev"
+      host_names  = ["dev"]
+      container_env = {
+        CKAN_SITE_URL = "https://dev.accessthedata.org"
+      }
+    }
+  }
+}
+
+module "zone" {
+  source = "../../terraform-modules/project-zone"
+
+  zone_name            = "accessthedata.org"
+  github_at_apex       = true
+  shared_configuration = local.shared_configuration
+}
+
+module "database" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/database"
+
+  shared_configuration = local.shared_configuration
+  environment          = each.value.environment
+  db_name              = "accessthedata"
+  owner_name           = "ckan"
+}
+
+module "datastore_database" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/database"
+
+  shared_configuration = local.shared_configuration
+  environment          = each.value.environment
+  db_name              = "accessthedata_datastore"
+  owner_name           = "ckands"
+  viewer_name          = "ckands_ro"
+}
+
+module "secrets" {
+  for_each     = local.envs
+  source       = "../../terraform-modules/cheap-secrets"
+  scope-name   = "ckan-${each.key}"
+  secret-names = ["csrf", "admin-password"]
+}
+
+module "access-the-data" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/multi-container-service"
+
+  shared_configuration = local.shared_configuration
+
+  region           = "us-west-2"
+  project_name     = "access-the-data"
+  application_type = "fullstack"
+  environment      = each.value.environment
+  zone_id          = local.zone_id
+
+  vpc_cidr = "10.10.0.0/16"
+
+  containers = {
+    ckan = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+      port   = 80
+
+      subdomains    = each.value.host_names
+      path_patterns = ["/*"]
+      env_vars = merge({
+        DATABASE      = "postgres"
+        POSTGRES_HOST = module.database[each.key].host
+        POSTGRES_PORT = module.database[each.key].port
+
+        // SQLALCHEMY has been set up in the container =
+        // we don't know the PG password, so we can't build the URLs
+
+        # Taken verbatim from .env
+        CKAN_DB                  = module.database[each.key].database
+        CKAN_DB_USER             = module.database[each.key].owner
+        CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
+        CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
+        CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
+        CKAN_VERSION             = "2.10.0"
+        CKAN_SITE_ID             = "default"
+
+        CKAN_PORT      = "5000"
+        CKAN_PORT_HOST = "5000"
+
+        CKAN_SYSADMIN_NAME  = "ckan_admin"
+        CKAN_SYSADMIN_EMAIL = "[email protected]"
+        CKAN_STORAGE_PATH   = "/var/lib/ckan"
+
+        CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
+        CKAN_SMTP_STARTTLS  = "True"
+        CKAN_SMTP_USER      = "user"
+        CKAN_SMTP_PASSWORD  = "pass"
+        CKAN_SMTP_MAIL_FROM = "ckan@localhost"
+
+        CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
+        CKAN_REDIS_URL                      = "redis://redis:6379/1"
+        CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
+        CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
+        CKAN__HARVEST__MQ__HOSTNAME         = "redis"
+
+        CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
+        CKAN__HARVEST__MQ__TYPE     = "redis"
+        CKAN__HARVEST__MQ__PORT     = "6379"
+        CKAN__HARVEST__MQ__REDIS_DB = "1"
+        CKAN__FAVICON               = "favicon.png"
+      }, lookup(each.value.container_env, "ckan", {}))
+      secrets = {
+        CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
+        CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
+        CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
+        CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
+        CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
+      }
+    }
+
+    datapusher = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+    }
+
+    solr = {
+      tag    = "latest"
+      cpu    = 512
+      memory = 4096
+    }
+
+    redis = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+    }
+  }
+}

terraform-incubator/access-the-data/moves.tf

@@ -0,0 +1,8 @@
+moved {
+  from = aws_route53_record.apex
+  to   = module.zone.aws_route53_record.apex
+}
+moved {
+  from = aws_route53_zone.this
+  to   = module.zone.aws_route53_zone.this
+}

terraform-incubator/access-the-data/versions.tf

@@ -0,0 +1,42 @@
+// Get configuration from the shared infrastructure
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+
+  config = {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/./terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+
+locals {
+  shared_configuration = data.terraform_remote_state.shared.outputs.configuration
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+// Set up Postgres provider to create the database
+terraform {
+  required_providers {
+    postgresql = {
+      source  = "cyrilgdn/postgresql"
+      version = "~> 1.21.0"
+    }
+  }
+}
+data "aws_ssm_parameter" "rds_credentials" {
+  name = "rds_credentials"
+}
+data "aws_db_instance" "shared" {
+  db_instance_identifier = local.shared_configuration.db_identifier
+}
+provider "postgresql" {
+  host      = data.aws_db_instance.shared.address
+  password  = data.aws_ssm_parameter.rds_credentials.value
+  username  = "postgres"
+  superuser = false
+}

terraform-incubator/people-depot/project/main.tf

@@ -16,7 +16,7 @@ module "people_depot" {
   container_cpu   = 256
   aws_managed_dns = false
   container_env_vars = {
-    SQL_HOST          = "incubator-prod-database.cewewwrvdqjn.us-west-2.rds.amazonaws.com"
+    SQL_HOST          = data.terraform_remote_state.shared.outputs.db_instance_endpoint
     COGNITO_USER_POOL = "us-west-2_Fn4rkZpuB"
 
     COGNITO_AWS_REGION   = "us-west-2"