diff --git a/data/README.md b/data/README.md index cd364516f..27384c7e4 100644 --- a/data/README.md +++ b/data/README.md @@ -8,6 +8,10 @@ Seed data for the API can be loaded using the included Makefile. You can execute - To run the seed data importer natively, run `make seed-data` - To run the seed data importer in a docker container, use the convenient `./data.sh seed-data` script. -When natively executing the import, you need to manually set up dependencies, like `python` packages (see `requirements.txt`) as well as other system dependencies (see `Dockerfile`). You also need to manually set several env vars to allow the application to connect to the Postgres database that will host the data. +When natively executing the import, you need to manually set up dependencies, like `python` packages ( +see `requirements.txt`) as well as other system dependencies (see `Dockerfile`). You also need to manually set several +env vars to allow the application to connect to the Postgres database that will host the data. -If running using a docker container, application dependencies will be automatically installed for you (you do need to have docker and docker-compose properly set up) and env vars will be loaded from the `.env` file at the root of the project - see the `env.default` file for an example/skeleton of the `.env` file you'll need to have. +If running using a docker container, application dependencies will be automatically installed for you (you do need to +have docker and docker-compose properly set up) and env vars will be loaded from the `.env` file at the root of the +project - see the `env.default` file for an example/skeleton of the `.env` file you'll need to have. diff --git a/infrastructure/kubernetes/.terraform.lock.hcl b/infrastructure/kubernetes/.terraform.lock.hcl index b6e6f9af4..4abd15797 100644 --- a/infrastructure/kubernetes/.terraform.lock.hcl +++ b/infrastructure/kubernetes/.terraform.lock.hcl @@ -5,8 +5,7 @@ provider "registry.terraform.io/gavinbunney/kubectl" { version = "1.14.0" constraints = "~> 1.14.0" hashes = [ - "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", - "h1:mX2AOFIMIxJmW5kM8DT51gloIOKCr9iT6W8yodnUyfs=", + "h1:ItrWfCZMzM2JmvDncihBMalNLutsAk7kyyxVRaipftY=", "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", @@ -23,8 +22,7 @@ provider "registry.terraform.io/hashicorp/aws" { version = "4.34.0" constraints = "~> 4.34.0" hashes = [ - "h1:JRqeU/5qR61U+z86mC68C5hp0XHZXxmRK9dupTIAhGg=", - "h1:TMVXbfjowAI4MjMDCU7AJwCUzfufoSC/v6/v85sAOlg=", + "h1:SDqaa/BVMQMzQ1bWQfrcsC4jfaywFeUq03jsojDNnyY=", "zh:2bdc9b908008c1e874d8ba7e2cfabd856cafb63c52fef51a1fdeef2f5584bffd", "zh:43c5364e3161be3856e56494cbb8b21d513fc05875f1b40e66e583602154dd0a", "zh:44e763adae92489f223f65866c1f8b5342e7e85b95daa8d1f483a2afb47f7db3", @@ -41,23 +39,24 @@ provider "registry.terraform.io/hashicorp/aws" { } provider "registry.terraform.io/hashicorp/github" { - version = "5.18.3" + version = "6.4.0" hashes = [ - "h1:Z/0vjFX80YzM3Oeq0mBbn4XYwb1POggjsu3RVQcbjNc=", - "zh:050b37d96628cb7451137755929ca8d21ea546bc46d11a715652584070e83ff2", - "zh:053051061f1b7f7673b0ceffac1f239ba28b0e5b375999206fd39976e85d9f2b", - "zh:0c300a977ca66d0347ed62bb116fd8fc9abb376a554d4c192d14f3ea71c83500", - "zh:1d5a1a5243eba78819d2f92ff2d504ebf9a9008a6670fb5f5660f44eb6a156d8", - "zh:a13ac15d251ebf4e7dc40acb0e40df066f443f4c7799186a29e2e44addc7d8e7", - "zh:a316d94b885953c036ebc9fba64a23da93974746bc3ac9d207462a6f02d44540", - "zh:a658a00373bff5979cc227052c693cbde8ca4c8f9fef1bc8094a3516f2e2a96d", - "zh:a7bfc6ad8465d5dc11b6f19d6805364de87fffe27622bb4f37da2319bb1c4956", - "zh:d7379a76861f1a6bfc36eca7a20f1f477711247563b105744d69d7bd1f365fad", - "zh:de1cd959fd4821248e8d21570601193408648474e74f49597f1d0c43185a4ab7", - "zh:e0b281240dd6f2aa405b2d6fe329bc15ab877161affe163fb150d1efca2fccdb", - "zh:e372c171358757a983d7aa878abfd05a84484fb4d22167e45c9c1267e78ed060", - "zh:f6d3116526030b3f6905f530cd6c04b23d42890d973fa2abe10ce9c89cb1db80", - "zh:f99eec731e03cc6a28996c875bd435887cd7ea75ec07cc77b9e768bb12da2227", + "h1:YiGCvjr7R77HGTzw81legWicEHApVTli8O+ooDpLexE=", + "zh:00f431c2a2510efcb1115442dda5e90815bcb16e1a3301679ade0139fa963d3b", + "zh:12a862f4317b3cb65682c1b687650cd91eeee99e63774bdcfa8bcfc64bad097b", + "zh:226d5e09ff27f94cb9336089181d26f85cb30219b863a579597f2e107f37de49", + "zh:402ecaa5add568a52ee01d816810f3b90f693be35c680fcdc9b6284bf55326f1", + "zh:60e3bdd9fbefb3c1d790bc08889c1dc0e83636b82284faaa709411aa4f96bb9f", + "zh:625099eeff2f8aaecd22a24a451b326828435c8f9de86f2e5e99872e7b467fa7", + "zh:79e8b665421009df2260f50e10da1f7a7863b557ece96e2b07dfd2fad1e86fcd", + "zh:98e471fefc93dcfedeec750c694110db7d3331dc3a256191d30b9d2f70d12157", + "zh:a17702765e1fa92d1c288ddfd97075819ad61b344b341be7e09c554c841a6d9e", + "zh:ca72ccf40624ae26bf4660d8dd84a51638f0a1e78d5f19fdfaafaef97f838af6", + "zh:d009ab5527d45c44c424d26cd2eb51a5a6a6448f3fb1023b675789588cc08d64", + "zh:e5811be1e942a75b14dfcd3e03523d8df60cfbde0d7e24d75e78480a02a58949", + "zh:e6008ad28225ad6996b06bcd7f3070863329df406a56754e7fb9c31d6301ace4", + "zh:f1d93f56ea4f87183a5de4780704907605851d95a2d285a9ec755bf784c5569c", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", ] } @@ -65,7 +64,7 @@ provider "registry.terraform.io/hashicorp/google" { version = "4.51.0" constraints = "4.51.0" hashes = [ - "h1:7JFdiV9bvV6R+AeWzvNbVeoega481sJY3PqtIbrwTsM=", + "h1:8lpgWoonXz+Y2kM4h/UZEe6W/WZwaga6bhfwmb11grA=", "zh:001bf7478e495d497ffd4054453c97ab4dd3e6a24d46496d51d4c8094e95b2b1", "zh:19db72113552dd295854a99840e85678d421312708e8329a35787fff1baeed8b", "zh:42c3e629ace225a2cb6cf87b8fabeaf1c56ac8eca6a77b9e3fc489f3cc0a9db5", @@ -85,7 +84,7 @@ provider "registry.terraform.io/hashicorp/helm" { version = "2.7.1" constraints = "~> 2.7.0" hashes = [ - "h1:11oWNeohjD8Fy9S7WQSKY3GmDZi7gVdMRp8/Wqxn410=", + "h1:L5qLTfZH7PnZt9+YnS7iYmPBEDQOpEjZiF0v50BRNi8=", "zh:13e2467092deeff01c4cfa2b54ba4510aa7a9b06c58f22c4215b0f4333858364", "zh:4549843db4fdf5d8150e8c0734e67b54b5c3bcfc914e3221e6952f428fb984d2", "zh:55b5f83ed52f93dd00a73c33c948326052efd700350c19e63bb1679b12bfcda6", @@ -105,8 +104,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.14.0" constraints = "~> 2.14.0" hashes = [ - "h1:4zSUEWLVFn2Sji7mWT64XQGWwBQVDqTGXGfW4ZBB16U=", - "h1:FFeFf2j2ipbMlrbhmIv8M7bzX3Zq8SQHeFkkQGALh1k=", + "h1:7fQ/FCIZjnff7EzkyQ6Jh+wBAIZFwHoi5qOxQ1/OD6I=", "zh:1363fcd6eb3c63113eaa6947a4e7a9f78a6974ea344e89b662d97a78e2ccb70c", "zh:166352455666b7d584705ceeb00f24fb9b884ab84e3a1a6019dc45d6539c9174", "zh:4615249ce5311f6fbea9738b25b6e6159e7dcf4693b0a24bc6a5720d1bfd38d0", @@ -123,21 +121,21 @@ provider "registry.terraform.io/hashicorp/kubernetes" { } provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" + version = "3.2.3" hashes = [ - "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", ] } @@ -145,8 +143,7 @@ provider "registry.terraform.io/hashicorp/random" { version = "3.4.3" constraints = "~> 3.4.3" hashes = [ - "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", - "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", @@ -165,18 +162,7 @@ provider "registry.terraform.io/hashicorp/random" { provider "registry.terraform.io/hashicorp/template" { version = "2.2.0" hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + "h1:JJXe0PVSWezQwo0MJXgUxP+8xI9k+undKpdxoyKM8y8=", ] } @@ -184,7 +170,7 @@ provider "registry.terraform.io/integrations/github" { version = "5.17.0" constraints = "5.17.0" hashes = [ - "h1:CWw2DL8qmBp/LkqZAC3HiNFskw4bPyZYXgVgwUK7Lew=", + "h1:IoFnaH4zc8LpMs3cWSJGwEsyHXaLP8vzF1PNLwblMiM=", "zh:0caa38dab96d68621a1ae7087ca3b86f42aa0e6fc250f906299f1a34c9dd1e54", "zh:1119f8dacb2da0de0735e9ae586702e5f9758b963e548b5fa09a9f216d00bbc4", "zh:16bed2a93216aa573d1b2ff7cd371c9df3d454284204a4695d5b30f7325f49b3", diff --git a/infrastructure/kubernetes/main.tf b/infrastructure/kubernetes/main.tf index 85de0e034..81529d18f 100644 --- a/infrastructure/kubernetes/main.tf +++ b/infrastructure/kubernetes/main.tf @@ -9,7 +9,7 @@ terraform { data "terraform_remote_state" "core" { backend = "s3" - config = { + config = { bucket = var.tf_state_bucket region = var.aws_region key = "core.tfstate" @@ -56,14 +56,14 @@ resource "github_actions_secret" "mapbox_api_token_secret" { module "aws_environment" { for_each = merge(var.aws_environments, { staging = merge({ - load_fresh_data = false + load_fresh_data = false data_import_arguments = ["seed-data"] - image_tag = "staging" + image_tag = "staging" }, lookup(var.aws_environments, "staging", {})), production = merge({ - load_fresh_data = false + load_fresh_data = false data_import_arguments = ["seed-data"] - image_tag = "main" + image_tag = "main" }, lookup(var.aws_environments, "production", {})), }) source = "./modules/aws/env" @@ -76,11 +76,11 @@ module "aws_environment" { allowed_account_id = var.allowed_account_id gmaps_api_key = var.gmaps_api_key sendgrid_api_key = var.sendgrid_api_key - eudr_credentials = jsonencode(var.eudr_credentials) - load_fresh_data = lookup(each.value, "load_fresh_data", false) - data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"]) - image_tag = lookup(each.value, "image_tag", each.key) - repo_branch = lookup(each.value, "image_tag", each.key) + eudr_credentials = jsonencode(var.eudr_credentials) + load_fresh_data = lookup(each.value, "load_fresh_data", false) + data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"]) + image_tag = lookup(each.value, "image_tag", each.key) + repo_branch = lookup(each.value, "image_tag", each.key) private_subnet_ids = data.terraform_remote_state.core.outputs.private_subnet_ids repo_name = var.repo_name domain = var.domain @@ -88,8 +88,9 @@ module "aws_environment" { client_container_registry_url = data.terraform_remote_state.core.outputs.aws_client_container_registry_url tiler_container_registry_url = data.terraform_remote_state.core.outputs.aws_tiler_container_registry_url data_import_container_registry_url = data.terraform_remote_state.core.outputs.aws_data_import_container_registry_url - api_env_vars = lookup(each.value, "api_env_vars", []) - api_secrets = lookup(each.value, "api_secrets", []) + data_import_env_vars = lookup(each.value, "data_import_env_vars", []) + api_env_vars = lookup(each.value, "api_env_vars", []) + api_secrets = lookup(each.value, "api_secrets", []) science_bucket_name = data.terraform_remote_state.core.outputs.science_bucket_name providers = { @@ -108,10 +109,10 @@ module "gcp_environment" { tf_state_bucket = var.tf_state_bucket allowed_account_id = var.allowed_account_id gmaps_api_key = var.gmaps_api_key - load_fresh_data = lookup(each.value, "load_fresh_data", false) - data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"]) - image_tag = lookup(each.value, "image_tag", each.key) - repo_branch = lookup(each.value, "repo_branch", each.key) + load_fresh_data = lookup(each.value, "load_fresh_data", false) + data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"]) + image_tag = lookup(each.value, "image_tag", each.key) + repo_branch = lookup(each.value, "repo_branch", each.key) private_subnet_ids = data.terraform_remote_state.core.outputs.private_subnet_ids repo_name = var.repo_name domain = var.domain @@ -119,8 +120,8 @@ module "gcp_environment" { client_container_registry_url = data.terraform_remote_state.core.outputs.gcp_client_container_registry_url tiler_container_registry_url = data.terraform_remote_state.core.outputs.gcp_tiler_container_registry_url data_import_container_registry_url = data.terraform_remote_state.core.outputs.gcp_data_import_container_registry_url - api_env_vars = lookup(each.value, "api_env_vars", []) - api_secrets = lookup(each.value, "api_secrets", []) + api_env_vars = lookup(each.value, "api_env_vars", []) + api_secrets = lookup(each.value, "api_secrets", []) science_bucket_name = data.terraform_remote_state.core.outputs.science_bucket_name gcp_project = var.gcp_project_id gcp_region = var.gcp_region diff --git a/infrastructure/kubernetes/modules/aws/env/main.tf b/infrastructure/kubernetes/modules/aws/env/main.tf index 408903efd..6cc521195 100644 --- a/infrastructure/kubernetes/modules/aws/env/main.tf +++ b/infrastructure/kubernetes/modules/aws/env/main.tf @@ -43,7 +43,7 @@ locals { for env in var.api_env_vars : env.name => env.value } api_env_vars_map = merge(local.default_api_env_vars_map, local.overlapping_api_env_vars_map) - api_env_vars = [ + api_env_vars = [ for name, value in local.api_env_vars_map : { name = name value = value @@ -168,7 +168,7 @@ module "k8s_tiler" { value = "${module.k8s_api.api_service_name}.${var.environment}.svc.cluster.local" }, { - name = "API_PORT" + name = "API_PORT" // TODO: get port from api k8s service value = 3000 }, @@ -224,7 +224,7 @@ module "k8s_data_import" { load_data = var.load_fresh_data arguments = var.data_import_arguments - env_vars = [ + env_vars = concat(var.data_import_env_vars, [ { name = "API_POSTGRES_PORT" value = "5432" @@ -237,7 +237,7 @@ module "k8s_data_import" { name = "S3_COG_PATH" value = "processed/cogs" }, - ] + ]) secrets = [ { @@ -305,8 +305,8 @@ module "data-import-group" { max_size = 2 desired_size = 1 namespace = var.environment - subnet_ids = [var.private_subnet_ids[0]] - labels = { + subnet_ids = [var.private_subnet_ids[0]] + labels = { type : "data-import-${var.environment}" } } diff --git a/infrastructure/kubernetes/modules/aws/env/variables.tf b/infrastructure/kubernetes/modules/aws/env/variables.tf index 2b5f486cc..7651c9ec6 100644 --- a/infrastructure/kubernetes/modules/aws/env/variables.tf +++ b/infrastructure/kubernetes/modules/aws/env/variables.tf @@ -36,7 +36,7 @@ variable "domain" { } variable "private_subnet_ids" { - type = list(string) + type = list(string) description = "IDs of the subnets used in the EKS cluster" } @@ -80,8 +80,8 @@ variable "load_fresh_data" { } variable "data_import_arguments" { - type = list(string) - default = ["seed-data"] + type = list(string) + default = ["seed-data"] description = "Arguments to pass to the initial data import process" } @@ -116,7 +116,7 @@ variable "api_env_vars" { value = string })) description = "Key-value pairs of env vars to make available to the api container" - default = [] + default = [] } @@ -127,7 +127,7 @@ variable "api_secrets" { secret_key = string })) description = "List of secrets to make available to the api container" - default = [] + default = [] } @@ -137,5 +137,14 @@ variable "tiler_env_vars" { value = string })) description = "Key-value pairs of env vars to make available to the tiler container" - default = [] + default = [] +} + +variable data_import_env_vars { + type = list(object({ + name = string + value = any + })) + description = "Key-value pairs of env vars to make available to the data import container" + default = [] }