diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7840cb8..f7f5ce2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10'] steps: - uses: actions/checkout@v3 @@ -56,7 +56,7 @@ jobs: needs: [tests] runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v') - + defaults: run: working-directory: infrastructure/aws @@ -98,7 +98,7 @@ jobs: TITILER_XARRAY_DEBUG: True STACK_ALARM_EMAIL: ${{ secrets.ALARM_EMAIL }} STACK_STAGE: development - + # Build and deploy to production deployment whenever there a new tag is pushed - name: Build & Deploy Production if: startsWith(github.ref, 'refs/tags/v') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d84490..e6ef50a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,3 +30,4 @@ repos: exclude: tests/.* additional_dependencies: - types-attrs + - types-redis diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index 0a62609..0400810 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -67,46 +67,72 @@ def __init__( ], ) - security_group = ec2.SecurityGroup( - self, - "ElastiCacheSecurityGroup", - vpc=vpc, - description="Allow local access to ElastiCache redis", - allow_all_outbound=True, - ) - security_group.add_ingress_rule( - ec2.Peer.ipv4(vpc.vpc_cidr_block), ec2.Port.tcp(6379) - ) + if settings.enable_cache: + security_group = ec2.SecurityGroup( + self, + "ElastiCacheSecurityGroup", + vpc=vpc, + description="Allow local access to ElastiCache redis", + allow_all_outbound=True, + ) + security_group.add_ingress_rule( + ec2.Peer.ipv4(vpc.vpc_cidr_block), ec2.Port.tcp(6379) + ) - # Create the redis cluster - redis_cluster = elasticache.CfnCacheCluster( - self, - f"{id}-redis-cluster", - engine="redis", - cache_node_type="cache.t3.small", - num_cache_nodes=1, - vpc_security_group_ids=[security_group.security_group_id], - cache_subnet_group_name=f"{id}-cache-subnet-group", - cluster_name=f"{id}-redis-cluster", - ) + # Create the redis cluster + redis_cluster = elasticache.CfnCacheCluster( + self, + f"{id}-redis-cluster", + engine="redis", + cache_node_type="cache.t3.small", + num_cache_nodes=1, + vpc_security_group_ids=[security_group.security_group_id], + cache_subnet_group_name=f"{id}-cache-subnet-group", + cluster_name=f"{id}-redis-cluster", + ) - # Define the subnet group for the ElastiCache cluster - subnet_group = elasticache.CfnSubnetGroup( - self, - f"{id}-cache-subnet-group", - description="Subnet group for ElastiCache", - subnet_ids=vpc.select_subnets(subnet_type=ec2.SubnetType.PUBLIC).subnet_ids, - cache_subnet_group_name=f"{id}-cache-subnet-group", - ) + # Define the subnet group for the ElastiCache cluster + subnet_group = elasticache.CfnSubnetGroup( + self, + f"{id}-cache-subnet-group", + description="Subnet group for ElastiCache", + subnet_ids=vpc.select_subnets( + subnet_type=ec2.SubnetType.PUBLIC + ).subnet_ids, + cache_subnet_group_name=f"{id}-cache-subnet-group", + ) - # Add dependency - ensure subnet group is created before the cache cluster - redis_cluster.add_depends_on(subnet_group) + # Add dependency - ensure subnet group is created before the cache cluster + redis_cluster.add_depends_on(subnet_group) - veda_reader_role = iam.Role.from_role_arn( - self, - "veda-reader-dev-role", - role_arn=f"arn:aws:iam::{self.account}:role/veda-data-reader-dev", - ) + if settings.data_access_role_name is not None: + data_access_role = iam.Role.from_role_arn( + self, + "data-access-role", + role_arn=f"arn:aws:iam::{self.account}:role/{settings.data_access_role_name}", + ) + else: + data_access_role = iam.Role( + self, + "data-access-role", + assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), + ) + + data_access_role.add_managed_policy( + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3ReadOnlyAccess") + ) + + titiler_env = { + **DEFAULT_ENV, + **environment, + } + + if settings.enable_cache: + titiler_env.update( + {"TITILER_XARRAY_CACHE_HOST": redis_cluster.attr_redis_endpoint_address} + ) + + dockerfile_name = "Dockerfile.redis" if settings.enable_cache else "Dockerfile" lambda_function = aws_lambda.Function( self, @@ -114,23 +140,19 @@ def __init__( runtime=runtime, code=aws_lambda.Code.from_docker_build( path=os.path.abspath(context_dir), - file="infrastructure/aws/lambda/Dockerfile", + file=f"infrastructure/aws/lambda/{dockerfile_name}", platform="linux/amd64", ), handler="handler.handler", memory_size=memory, reserved_concurrent_executions=concurrent, timeout=Duration.seconds(timeout), - environment={ - **DEFAULT_ENV, - **environment, - "TITILER_XARRAY_CACHE_HOST": redis_cluster.attr_redis_endpoint_address, - }, + environment=titiler_env, log_retention=logs.RetentionDays.ONE_WEEK, vpc=vpc, vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), allow_public_subnet=True, - role=veda_reader_role, + role=data_access_role, ) # Create an S3 VPC Endpoint diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 4a48fc8..072f4a6 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -29,6 +29,10 @@ class StackSettings(pydantic.BaseSettings): timeout: int = 30 memory: int = 3009 + enable_cache: bool = True + + data_access_role_name: Optional[str] = "veda-data-reader-dev" + # The maximum of concurrent executions you want to reserve for the function. # Default: - No specific limit - account limit. max_concurrent: Optional[int] diff --git a/infrastructure/aws/lambda/Dockerfile b/infrastructure/aws/lambda/Dockerfile index a3723e4..1506519 100644 --- a/infrastructure/aws/lambda/Dockerfile +++ b/infrastructure/aws/lambda/Dockerfile @@ -26,6 +26,11 @@ RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc RUN rm -rdf /asset/boto3* RUN rm -rdf /asset/botocore* +RUN find /asset -type f -path '*LICENSE*' -delete +RUN find /asset -type f -path '*README*' -delete +RUN find /asset -type f -path '*AUTHORS*' -delete +RUN find /asset -type f -path '*pyproject*' -delete +RUN find /asset -type f -path '*setupcf*' -delete COPY infrastructure/aws/lambda/handler.py /asset/handler.py diff --git a/infrastructure/aws/lambda/Dockerfile.redis b/infrastructure/aws/lambda/Dockerfile.redis new file mode 100644 index 0000000..93f104f --- /dev/null +++ b/infrastructure/aws/lambda/Dockerfile.redis @@ -0,0 +1,38 @@ +ARG PYTHON_VERSION=3.10 + +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} + +WORKDIR /tmp + +COPY pyproject.toml pyproject.toml +COPY LICENSE LICENSE +COPY README.md README.md +COPY titiler/ titiler/ + +# Install dependencies +# HACK: aiobotocore has a tight botocore dependency +# https://github.com/aio-libs/aiobotocore/issues/862 +# and becaise we NEED to remove both boto3 and botocore to save space for the package +# we have to force using old package version that seems `almost` compatible with Lambda env botocore +# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html +RUN pip install --upgrade pip + +RUN pip install ".[cache]" "mangum>=0.10.0" "botocore==1.29.76" "aiobotocore==2.5.0" -t /asset --no-binary pydantic; + +# Reduce package size and remove useless files +RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; +RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f +RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf +RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc +RUN rm -rdf /asset/boto3* +RUN rm -rdf /asset/botocore* +RUN find /asset -type f -path '*LICENSE*' -delete +RUN find /asset -type f -path '*README*' -delete +RUN find /asset -type f -path '*AUTHORS*' -delete +RUN find /asset -type f -path '*pyproject*' -delete +RUN find /asset -type f -path '*setupcf*' -delete + +COPY infrastructure/aws/lambda/handler.py /asset/handler.py + +CMD ["echo", "hello world"] diff --git a/infrastructure/aws/package-lock.json b/infrastructure/aws/package-lock.json index ad52264..679f8ba 100644 --- a/infrastructure/aws/package-lock.json +++ b/infrastructure/aws/package-lock.json @@ -9,13 +9,13 @@ "version": "0.1.0", "license": "MIT", "dependencies": { - "cdk": "2.76.0-alpha.0" + "cdk": "2.138.0" } }, "node_modules/aws-cdk": { - "version": "2.76.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.76.0.tgz", - "integrity": "sha512-y6VHtqUpYenn6mGIBFbcGGXIoXfKA3o0eGL/eeD/gUJ9TcPrgMLQM1NxSMb5JVsOk5BPPXzGmvB0gBu40utGqg==", + "version": "2.138.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.138.0.tgz", + "integrity": "sha512-48xvfEaiM07RB+p05RHqRAVtKcxkGzXnwdTo765Ba0rUFK2ZQ9leykVeBDvdCj9u0eMv2fCRspP/wQxPOU2H/g==", "bin": { "cdk": "bin/cdk" }, @@ -27,17 +27,17 @@ } }, "node_modules/cdk": { - "version": "2.76.0-alpha.0", - "resolved": "https://registry.npmjs.org/cdk/-/cdk-2.76.0-alpha.0.tgz", - "integrity": "sha512-HNfX5c7MU18LxthZRcapqEhG0IFgQeNOhtsTR1QiL/7dhy2TjvK26dYcJ67KIHfzMfm5EUjvOXdP1SPdW+eOOA==", + "version": "2.138.0", + "resolved": "https://registry.npmjs.org/cdk/-/cdk-2.138.0.tgz", + "integrity": "sha512-BqOvVgvi+UEXGzuhFotHJLcUExMpZFe9jAr2TttY5j9d1HOaTijbf/YkN1hVGkNjeNIU2m61DHKdRU//kT4JVQ==", "dependencies": { - "aws-cdk": "2.76.0" + "aws-cdk": "2.138.0" }, "bin": { "cdk": "bin/cdk" }, "engines": { - "node": ">= 8.10.0" + "node": ">= 14.15.0" } }, "node_modules/fsevents": { @@ -56,19 +56,19 @@ }, "dependencies": { "aws-cdk": { - "version": "2.76.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.76.0.tgz", - "integrity": "sha512-y6VHtqUpYenn6mGIBFbcGGXIoXfKA3o0eGL/eeD/gUJ9TcPrgMLQM1NxSMb5JVsOk5BPPXzGmvB0gBu40utGqg==", + "version": "2.138.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.138.0.tgz", + "integrity": "sha512-48xvfEaiM07RB+p05RHqRAVtKcxkGzXnwdTo765Ba0rUFK2ZQ9leykVeBDvdCj9u0eMv2fCRspP/wQxPOU2H/g==", "requires": { "fsevents": "2.3.2" } }, "cdk": { - "version": "2.76.0-alpha.0", - "resolved": "https://registry.npmjs.org/cdk/-/cdk-2.76.0-alpha.0.tgz", - "integrity": "sha512-HNfX5c7MU18LxthZRcapqEhG0IFgQeNOhtsTR1QiL/7dhy2TjvK26dYcJ67KIHfzMfm5EUjvOXdP1SPdW+eOOA==", + "version": "2.138.0", + "resolved": "https://registry.npmjs.org/cdk/-/cdk-2.138.0.tgz", + "integrity": "sha512-BqOvVgvi+UEXGzuhFotHJLcUExMpZFe9jAr2TttY5j9d1HOaTijbf/YkN1hVGkNjeNIU2m61DHKdRU//kT4JVQ==", "requires": { - "aws-cdk": "2.76.0" + "aws-cdk": "2.138.0" } }, "fsevents": { diff --git a/infrastructure/aws/package.json b/infrastructure/aws/package.json index 040bfa6..490d9fe 100644 --- a/infrastructure/aws/package.json +++ b/infrastructure/aws/package.json @@ -5,7 +5,7 @@ "license": "MIT", "private": true, "dependencies": { - "cdk": "2.76.0-alpha.0" + "cdk": "2.138.0" }, "scripts": { "cdk": "cdk" diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index 68ad0c6..2c1b97c 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,7 +1,7 @@ -aws-cdk-lib==2.76.0 -aws_cdk-aws_apigatewayv2_alpha==2.76.0a0 -aws_cdk-aws_apigatewayv2_integrations_alpha==2.76.0a0 +aws-cdk-lib==2.138.0 +aws_cdk-aws_apigatewayv2_alpha==2.114.1a0 +aws_cdk-aws_apigatewayv2_integrations_alpha==2.114.1a0 constructs>=10.0.0 pydantic~=1.0 -python-dotenv +python-dotenv \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 08abb4a..3fe1c84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "titiler.xarray" description = "TiTiler extension for xarray." readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" authors = [ {name = "Vincent Sarago", email = "vincent@developmentseed.com"}, ] @@ -17,31 +17,31 @@ classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: GIS", ] dynamic = ["version"] dependencies = [ - "cftime", - "h5netcdf", - "xarray", - "rioxarray", - "zarr", - "fakeredis", + "cftime==1.6.3", + "h5netcdf==1.3.0", + "xarray==2024.3.0", + "rioxarray==0.15.0", + "zarr==2.17.2", + "h5py==3.10.0", + "rasterio==1.3.9", "fsspec", "s3fs", "aiohttp", - "requests", + "requests==2.31.0", "pydantic==2.0.2", - "titiler.core>=0.14.1,<0.15", - "pydantic-settings~=2.0", + "titiler.core==0.14.1", + "pydantic-settings==2.0.3", "pandas==1.5.3", - "redis", - "fastapi>=0.100.0,<0.107.0", - "starlette<0.28", + "fastapi==0.106.0", + "starlette==0.27.0", ] [project.optional-dependencies] @@ -51,9 +51,11 @@ test = [ "pytest-asyncio", "httpx", "yappi", + "fakeredis" ] dev = [ - "pre-commit" + "pre-commit", + "fakeredis" ] debug = [ "yappi" @@ -61,7 +63,9 @@ debug = [ server = [ "uvicorn" ] - +cache = [ + "redis==5.0.3" +] [project.urls] Homepage = "https://github.com/developmentseed/titiler-xarray" Issues = "https://github.com/developmentseed/titiler-xarray/issues" diff --git a/titiler/xarray/main.py b/titiler/xarray/main.py index 65d0e2f..8065fc3 100644 --- a/titiler/xarray/main.py +++ b/titiler/xarray/main.py @@ -19,7 +19,6 @@ from titiler.xarray import __version__ as titiler_version from titiler.xarray.factory import ZarrTilerFactory from titiler.xarray.middleware import ServerTimingMiddleware -from titiler.xarray.redis_pool import get_redis from titiler.xarray.settings import ApiSettings logging.getLogger("botocore.credentials").disabled = True @@ -97,8 +96,12 @@ def ping(): return {"ping": "pong!"} -@app.get("/clear_cache") -def clear_cache(cache_client=Depends(get_redis)): - """Clear the cache.""" - cache_client.flushall() - return {"status": "cache cleared!"} +if api_settings.enable_cache: + + from titiler.xarray.redis_pool import get_redis + + @app.get("/clear_cache") + def clear_cache(cache_client=Depends(get_redis)): + """Clear the cache.""" + cache_client.flushall() + return {"status": "cache cleared!"} diff --git a/titiler/xarray/reader.py b/titiler/xarray/reader.py index 187af17..dcdd286 100644 --- a/titiler/xarray/reader.py +++ b/titiler/xarray/reader.py @@ -20,7 +20,6 @@ from titiler.xarray.settings import ApiSettings api_settings = ApiSettings() -cache_client = get_redis() def parse_protocol(src_path: str, reference: Optional[bool] = False): @@ -90,6 +89,7 @@ def xarray_open_dataset( """Open dataset.""" # Generate cache key and attempt to fetch the dataset from cache if api_settings.enable_cache: + cache_client = get_redis() cache_key = f"{src_path}_{group}" if group is not None else src_path data_bytes = cache_client.get(cache_key) if data_bytes: diff --git a/titiler/xarray/redis_pool.py b/titiler/xarray/redis_pool.py index 2c26d0c..11ac563 100644 --- a/titiler/xarray/redis_pool.py +++ b/titiler/xarray/redis_pool.py @@ -1,13 +1,20 @@ """ Redis singleton class. """ import os -import fakeredis -import redis # type: ignore - from titiler.xarray.settings import ApiSettings api_settings = ApiSettings() +try: + import redis +except ImportError: + redis = None # type: ignore + +try: + import fakeredis +except ImportError: + fakeredis = None + class RedisCache: """Redis connection pool singleton class.""" @@ -17,6 +24,11 @@ class RedisCache: @classmethod def get_instance(cls): """Get the redis connection pool.""" + + assert ( + redis + ), "`redis` must be installed to enable caching. Please install titiler-xarray with the `cache` optional dependencies that include `redis`." + if cls._instance is None: cls._instance = redis.ConnectionPool( host=api_settings.cache_host, port=6379, db=0 @@ -27,6 +39,11 @@ def get_instance(cls): def get_redis(): """Get a redis connection.""" if os.getenv("TEST_ENVIRONMENT"): + + assert ( + fakeredis + ), "`fakeredis` must be installed to enable caching in test environment. Please install titiler-xarray with the `dev` optional dependencies that include `fakeredis`." + server = fakeredis.FakeServer() # Use fakeredis in a test environment return fakeredis.FakeRedis(server=server)