From 4dacaff88ca90c37f0397aaecc60a74357a1dc23 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 29 Jul 2024 11:51:48 -0400 Subject: [PATCH 1/9] Make pypgstac an ARG in dockerfile --- ingest_api/runtime/Dockerfile | 5 ++++- ingest_api/runtime/requirements.txt | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ingest_api/runtime/Dockerfile b/ingest_api/runtime/Dockerfile index c2955171..0bf6040e 100644 --- a/ingest_api/runtime/Dockerfile +++ b/ingest_api/runtime/Dockerfile @@ -1,9 +1,12 @@ FROM public.ecr.aws/sam/build-python3.9:latest +ARG PGSTAC_VERSION=0.7.4 +RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}" + WORKDIR /tmp COPY ingest_api/runtime/requirements.txt /tmp/ingestor/requirements.txt -RUN pip install -r /tmp/ingestor/requirements.txt -t /asset --no-binary pydantic uvicorn +RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn RUN rm -rf /tmp/ingestor # TODO this is temporary until we use a real packaging system like setup.py or poetry COPY ingest_api/runtime/src /asset/src diff --git a/ingest_api/runtime/requirements.txt b/ingest_api/runtime/requirements.txt index de984f61..ff839e66 100644 --- a/ingest_api/runtime/requirements.txt +++ b/ingest_api/runtime/requirements.txt @@ -9,7 +9,6 @@ psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.10.12 pyjwt>=2.8.0 -pypgstac==0.7.4 python-multipart==0.0.7 requests>=2.27.1 s3fs==2023.3.0 From 0f93644c5e1432027a2357e542b6e82acb361c6c Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 29 Jul 2024 13:16:42 -0400 Subject: [PATCH 2/9] change ARG to ENV --- ingest_api/runtime/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ingest_api/runtime/Dockerfile b/ingest_api/runtime/Dockerfile index 0bf6040e..827f7fc2 100644 --- a/ingest_api/runtime/Dockerfile +++ b/ingest_api/runtime/Dockerfile @@ -1,12 +1,12 @@ FROM public.ecr.aws/sam/build-python3.9:latest -ARG PGSTAC_VERSION=0.7.4 -RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}" +ENV VEDA_DB_PGSTAC_VERSION=0.7.4 +RUN echo "Using PGSTAC Version ${VEDA_DB_PGSTAC_VERSION}" WORKDIR /tmp COPY ingest_api/runtime/requirements.txt /tmp/ingestor/requirements.txt -RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn +RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${VEDA_DB_PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn RUN rm -rf /tmp/ingestor # TODO this is temporary until we use a real packaging system like setup.py or poetry COPY ingest_api/runtime/src /asset/src From 358e8b76a160cd8fd4bb2e2d533c204236383173 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 29 Jul 2024 14:12:52 -0400 Subject: [PATCH 3/9] Add env, env version may change though --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index a126d76c..40099e6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,6 +128,7 @@ services: - PGPASSWORD=password - PGDATABASE=postgis - DYNAMODB_ENDPOINT=http://localhost:8085 + - VEDA_DB_PGSTAC_VERSION=0.6.6 ports: - "8083:8083" command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py" From ec3e160715a58f8e8a8780e3894fe526ad9043e6 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 5 Aug 2024 11:18:51 -0400 Subject: [PATCH 4/9] Change ENV to ARG and add logic for build arg --- docker-compose.yml | 2 +- ingest_api/infrastructure/config.py | 4 ++++ ingest_api/infrastructure/construct.py | 4 +++- ingest_api/runtime/Dockerfile | 6 +++--- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 40099e6e..5c513a03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,7 @@ services: - PGPASSWORD=password - PGDATABASE=postgis - DYNAMODB_ENDPOINT=http://localhost:8085 - - VEDA_DB_PGSTAC_VERSION=0.6.6 + - VEDA_DB_PGSTAC_VERSION=0.7.10 ports: - "8083:8083" command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py" diff --git a/ingest_api/infrastructure/config.py b/ingest_api/infrastructure/config.py index 13651b6b..771d346d 100644 --- a/ingest_api/infrastructure/config.py +++ b/ingest_api/infrastructure/config.py @@ -67,6 +67,10 @@ class IngestorConfig(BaseSettings): ingest_root_path: str = Field("", description="Root path for ingest API") custom_host: Optional[str] = Field(description="Custom host name") + db_pgstac_version: str = Field( + ..., + description="Version of PgStac database, i.e. 0.5", + ) class Config: case_sensitive = False diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index 1e763d09..f0cd2dcb 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -148,7 +148,8 @@ def build_api_lambda( ) ], ) - + pgstac_version = IngestorConfig.db_pgstac_version + handler = aws_lambda.Function( self, "api-handler", @@ -156,6 +157,7 @@ def build_api_lambda( path=os.path.abspath(code_dir), file="ingest_api/runtime/Dockerfile", platform="linux/amd64", + build_args={"PGSTAC_VERSION": pgstac_version}, ), runtime=aws_lambda.Runtime.PYTHON_3_9, timeout=Duration.seconds(30), diff --git a/ingest_api/runtime/Dockerfile b/ingest_api/runtime/Dockerfile index 827f7fc2..ee5453ab 100644 --- a/ingest_api/runtime/Dockerfile +++ b/ingest_api/runtime/Dockerfile @@ -1,12 +1,12 @@ FROM public.ecr.aws/sam/build-python3.9:latest -ENV VEDA_DB_PGSTAC_VERSION=0.7.4 -RUN echo "Using PGSTAC Version ${VEDA_DB_PGSTAC_VERSION}" +ARG PGSTAC_VERSION +RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}" WORKDIR /tmp COPY ingest_api/runtime/requirements.txt /tmp/ingestor/requirements.txt -RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${VEDA_DB_PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn +RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn RUN rm -rf /tmp/ingestor # TODO this is temporary until we use a real packaging system like setup.py or poetry COPY ingest_api/runtime/src /asset/src From 485dbc9aea99eb6dce5a050c8acde4c592231592 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 6 Aug 2024 11:54:57 -0600 Subject: [PATCH 5/9] lint --- ingest_api/infrastructure/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index f0cd2dcb..3e24025f 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -149,7 +149,7 @@ def build_api_lambda( ], ) pgstac_version = IngestorConfig.db_pgstac_version - + handler = aws_lambda.Function( self, "api-handler", From ea88a2c082c20de38a893d22c23ff2e5f8bd2f2a Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 8 Aug 2024 15:01:05 -0600 Subject: [PATCH 6/9] add pgstac version to ingest build api lambda params --- ingest_api/infrastructure/construct.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index 5c308966..7d653996 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -66,6 +66,7 @@ def __init__( "db_secret": db_secret, "db_vpc": db_vpc, "db_security_group": db_security_group, + "pgstac_version": config.db_pgstac_version } if config.raster_data_access_role_arn: @@ -124,6 +125,7 @@ def build_api_lambda( db_vpc: ec2.IVpc, db_security_group: ec2.ISecurityGroup, data_access_role: Union[iam.IRole, None] = None, + pgstac_version: str, code_dir: str = "./", ) -> apigateway.LambdaRestApi: stack_name = Stack.of(self).stack_name @@ -142,7 +144,6 @@ def build_api_lambda( ) ], ) - pgstac_version = IngestorConfig.db_pgstac_version handler = aws_lambda.Function( self, From 826f46a327b1ad3287cae42e8d30a9f11975f830 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 8 Aug 2024 15:02:34 -0600 Subject: [PATCH 7/9] lint --- ingest_api/infrastructure/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index 7d653996..5270fd8c 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -66,7 +66,7 @@ def __init__( "db_secret": db_secret, "db_vpc": db_vpc, "db_security_group": db_security_group, - "pgstac_version": config.db_pgstac_version + "pgstac_version": config.db_pgstac_version, } if config.raster_data_access_role_arn: From ca008d4a87e8c2b6085656171af3e7244820fbb3 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 12 Aug 2024 09:52:22 -0400 Subject: [PATCH 8/9] Update conflict mistake and add to build_ingest --- ingest_api/infrastructure/construct.py | 3 +++ ingest_api/runtime/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index 5270fd8c..f6574056 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -294,6 +294,7 @@ def __init__( db_vpc=db_vpc, db_security_group=db_security_group, db_vpc_subnets=db_vpc_subnets, + pgstac_version=config.db_pgstac_version, ) def build_ingestor( @@ -305,6 +306,7 @@ def build_ingestor( db_vpc: ec2.IVpc, db_security_group: ec2.ISecurityGroup, db_vpc_subnets: ec2.SubnetSelection, + pgstac_version: str, code_dir: str = "./", ) -> aws_lambda.Function: handler = aws_lambda.Function( @@ -314,6 +316,7 @@ def build_ingestor( path=os.path.abspath(code_dir), file="ingest_api/runtime/Dockerfile", platform="linux/amd64", + build_args={"PGSTAC_VERSION": pgstac_version}, ), handler="ingestor.handler", runtime=aws_lambda.Runtime.PYTHON_3_9, diff --git a/ingest_api/runtime/requirements.txt b/ingest_api/runtime/requirements.txt index ff839e66..66537327 100644 --- a/ingest_api/runtime/requirements.txt +++ b/ingest_api/runtime/requirements.txt @@ -8,7 +8,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.10.12 -pyjwt>=2.8.0 +pypgstac==0.7.4 python-multipart==0.0.7 requests>=2.27.1 s3fs==2023.3.0 From e6f10d6cc8edb190884d08fa5313c6bdeefbeed4 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 12 Aug 2024 14:57:21 -0400 Subject: [PATCH 9/9] Remove pypgstac from requirements file --- ingest_api/runtime/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/ingest_api/runtime/requirements.txt b/ingest_api/runtime/requirements.txt index 66537327..f6e3f3d4 100644 --- a/ingest_api/runtime/requirements.txt +++ b/ingest_api/runtime/requirements.txt @@ -8,7 +8,6 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.10.12 -pypgstac==0.7.4 python-multipart==0.0.7 requests>=2.27.1 s3fs==2023.3.0