Merge branch 'morosi-cookiecutter' into 'master'

Use the cookiecutter template for e3-aws

See merge request it/e3-aws!7

.gitlab-ci.yml

@@ -1,39 +1,88 @@
 variables:
-  GITLAB_REMOTE: "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/"
+  GITLAB_REMOTE:
+    description: "The remote gitlab URL used."
+    value: "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/"
+  LATEST_PYTHON:
+    description: "The latest python version used to test this project."
+    options:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    value: "3.12"
 
 stages:
-  - check
-  - test
+  - checks
+  - tests linux
+  - tests windows
 
 default:
+  services:
+    - run_as_root:false
   interruptible: true
+
+# Common
+
+.tox-common:
+  before_script:
+    - python -m pip install --force tox
+  script:
+    # Should be quoted using \' to deal with ':' in the command
+    - 'echo "Tox run environment: ${CI_TOX_ENV:=py${PYTHON_VERSION:0:1}${PYTHON_VERSION:2:2}-cov-xdist}"'
+    - python -m tox --colored yes -e ${CI_TOX_ENV}
+
+### Linux jobs ###
+
+.linux-image:
   services:
     - image:all-pythons
   before_script:
-    - git config --global --add
-        url."${GITLAB_REMOTE}/it/black.git".insteadOf
-        https://github.com/ambv/black
-    - git config --global --add
-        url."${GITLAB_REMOTE}/it/flake8.git".insteadOf
-        https://github.com/pycqa/flake8
-    - source /it/activate-${PYTHON_VERSION}
-    - python -m pip install --force tox
+    - source /it/activate-py${PYTHON_VERSION:0:1}${PYTHON_VERSION:2:2}
+    - python -m pip install -U pip
+
+.linux-common:
+  extends:
+    - .linux-image
+    - .tox-common
+  before_script:
+    - !reference [.linux-image, before_script]
+    - !reference [.tox-common, before_script]
+
+# Stage: Checks
 
 check:
-  stage: check
-  script:
+  stage: checks
+  extends: .linux-common
+  needs: []
+  before_script:
+    - !reference [.linux-common, before_script]
+    - git config --global --add
+      url."${GITLAB_REMOTE}/it/black.git".insteadOf
+      https://github.com/ambv/black
+    - git config --global --add
+      url."${GITLAB_REMOTE}/it/flake8.git".insteadOf
+      https://github.com/pycqa/flake8
     - python -m pip install pre-commit
     - pre-commit install
+  script:
     - pre-commit run -a --show-diff-on-failure
-    - python -m tox --colored yes -e check
+    - !reference [.linux-common, script]
   variables:
-    PYTHON_VERSION: py311
+    PYTHON_VERSION: ${LATEST_PYTHON}
+    CI_TOX_ENV: check
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 
-.test-py-common:
-  stage: test
-  script:
-    - python -m tox --colored yes -e ${PYTHON_VERSION}-cov
+.test-linux:
+  stage: tests linux
+  extends: .linux-common
+  services:
+    - !reference [.linux-common, services]
+    - cpu:4
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
   artifacts:
     when: always
     paths:
@@ -44,23 +93,102 @@ check:
         path: coverage.xml
       junit: pytest-report.xml
 
-test-py39:
-  extends: .test-py-common
-  variables:
-    PYTHON_VERSION: py39
+linux python:
+  extends: .test-linux
+  needs: ["check"]
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 
-test-py310:
-  extends: .test-py-common
-  variables:
-    PYTHON_VERSION: py310
+# A job triggered by 'run linux tests'. This jobs will run without waiting any others
+# jobs.
+linux python (always):
+  extends: .test-linux
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_MERGE_REQUEST_ID && $CI_PROJECT_NAME == "e3-core" && $ALWAYS_LINUX_TESTS == "y"
 
-test-py311:
-  extends: .test-py-common
+# A manual job to run Linux tests even if "check" job has failed
+run linux tests:
+  stage: tests linux
+  needs: []
+  trigger:
+    include: .gitlab-ci.yml
+    strategy: depend
   variables:
-    PYTHON_VERSION: py311
+    ALWAYS_LINUX_TESTS: "y"
+    ALWAYS_WINDOWS_TESTS: "n"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: true
+  # Contrary to what the documentation might suggest, manual_confirmation
+  # is not currently usable with our gitlab.
+  # However, when it is, adding a manual confirmation to warn the user that
+  # this job should only be used when previous steps have failed seems
+  # useful. Something like:
+  #
+  # manual_confirmation: |-
+  #     Are you sure you want to run Linux tests?
+  #
+  #     This is only useful if the previous stages have failed and you still want to run the tests.
 
-test-py312:
-  extends: .test-py-common
-  variables:
-    PYTHON_VERSION: py312
+### Windows jobs ###
 
+.windows-image:
+  services:
+    - image:e3-windows-core-2022
+    - platform:x86_64-windows-2022
+    - cpu:2
+    - mem:4
+  before_script:
+    - source /it/activate-python ${PYTHON_VERSION}
+    - mkdir -p "C:/tmp/Trash"
+    - python -m pip install -U pip
+
+.windows-common:
+  extends:
+    - .windows-image
+    - .tox-common
+  before_script:
+    - !reference [.windows-image, before_script]
+    - !reference [.tox-common, before_script]
+
+.test-windows:
+  stage: tests windows
+  extends: .windows-common
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
+
+# A job tiggered by 'Run Windows tests'. This jobs will run without waiting any others
+# jobs.
+windows python (always):
+  extends: .test-windows
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_MERGE_REQUEST_ID && $CI_PROJECT_NAME == "e3-aws" && $ALWAYS_WINDOWS_TESTS == "y"
+
+# A manual job to run Windows tests even if previous jobs have failed
+run windows tests:
+  stage: tests windows
+  needs: []
+  trigger:
+    include: .gitlab-ci.yml
+    strategy: depend
+  variables:
+    ALWAYS_LINUX_TESTS: "n"
+    ALWAYS_WINDOWS_TESTS: "y"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      allow_failure: true
+  # Contrary to what the documentation might suggest, manual_confirmation
+  # is not currently usable with our gitlab.
+  # However, when it is, adding a manual confirmation to warn the user that
+  # this job should only be used when previous steps have failed seems
+  # useful. Something like:
+  #
+  # manual_confirmation: |-
+  #     Are you sure you want to run Windows tests?
+  #
+  #     This is only useful if the previous stages have failed and you still want to run the tests.

pyproject.toml

@@ -29,6 +29,7 @@ s3-boto3 = "e3.aws.handler.s3:S3Handler"
 test = [
   "awscli",
   "pytest",
+  "pytest-html",
   "mock",
   "requests_mock",
   "httpretty",
@@ -38,6 +39,9 @@ test = [
 
 check = [
   "mypy==1.8.0",
+  "pytest",
+  "flask",
+  "moto[sts, dynamodb]",
   "bandit",
   "pip-audit",
   "types-colorama",

src/e3/aws/__init__.py

@@ -27,6 +27,7 @@
     from typing import Any, TypedDict, Callable
     import botocore.client
     import botocore.stub
+    from datetime import datetime
 
     class AWSCredentials(TypedDict, total=False):
         """Annotate a dict containing AWS credentials.
@@ -44,7 +45,7 @@ class AWSCredentials(TypedDict, total=False):
         AccessKeyId: str
         SecretAccessKey: str
         SessionToken: str
-        Expiration: str
+        Expiration: datetime
 
 
 class AWSSessionRunError(E3Error):
@@ -372,7 +373,7 @@ def wrapper(*args, **kwargs):
     return decorator
 
 
-def assume_profile_main():
+def assume_profile_main() -> None:
     """Generate shell commands to set credentials for a profile."""
     argument_parser = argparse.ArgumentParser()
     argument_parser.add_argument(
@@ -416,7 +417,7 @@ def assume_profile_main():
             print(f"export {k}={v}")
 
 
-def assume_role_main():
+def assume_role_main() -> None:
     """Generate shell commands to set credentials for a role."""
     argument_parser = argparse.ArgumentParser()
     argument_parser.add_argument(
@@ -449,14 +450,18 @@ def assume_role_main():
     credentials = s.assume_role_get_credentials(
         args.role_arn, role_session_name, session_duration=session_duration
     )
-    credentials["Expiration"] = credentials["Expiration"].timestamp()
+    credentials_float = credentials | {
+        "Expiration": credentials["Expiration"].timestamp()
+    }
     if args.json:
-        print(json.dumps(credentials))
+        print(json.dumps(credentials_float))
     else:
-        credentials = {
-            key_to_envvar[k]: v for k, v in credentials.items() if k in key_to_envvar
+        credentials_float = {
+            key_to_envvar[k]: v
+            for k, v in credentials_float.items()
+            if k in key_to_envvar
         }
-        for k, v in credentials.items():
+        for k, v in credentials_float.items():
             print(f"export {k}={v}")
 
 

src/e3/aws/pricing/__init__.py

@@ -5,10 +5,10 @@
 from e3.aws.util import get_region_name
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Union
     import botocore
 
-    _CacheKey = tuple[str | None, str | None, str | None]
+    _CacheKey = tuple[Union[str, None], Union[str, None], Union[str, None]]
 
     # This is only to avoid repeating the type everywhere
     PriceInformation = dict[str, Any]

tests/tests_e3_aws/assume_profile_main_test.py

@@ -45,7 +45,11 @@ def get_frozen_credentials(self) -> ReadOnlyCredentials:
     "json,expected_output",
     [(False, EXPECTED_DEFAULT_OUTPUT), (True, EXPECTED_JSON_OUTPUT)],
 )
-def test_assume_profile_main_json(json: bool, expected_output: str, capfd):
+def test_assume_profile_main_json(
+    json: bool,
+    expected_output: str,
+    capfd: pytest.CaptureFixture[str],
+) -> None:
     """Test the credentials returned by assume_profile_main."""
     with (
         mock.patch(

tests/tests_e3_aws/dynamodb/main_test.py

@@ -122,7 +122,7 @@ def test_update_item(client: DynamoDB) -> None:
     client.update_item(
         item=customers[0],
         table_name=TABLE_NAME,
-        keys=PRIMARY_KEYS,
+        keys=("name", "S"),
         data={"age": 33},
     )
 
@@ -138,7 +138,7 @@ def test_update_item_condition(client: DynamoDB) -> None:
     client.update_item(
         item=customers[0],
         table_name=TABLE_NAME,
-        keys=PRIMARY_KEYS,
+        keys=("name", "S"),
         data={"age": 33},
         condition_expression="attribute_exists(#n) AND #a = :a",
         expression_attribute_names={"#n": "name", "#a": "age"},

tests/tests_e3_aws/pricing/main_test.py

@@ -66,8 +66,8 @@
         {"Field": "capacitystatus", "Type": "TERM_MATCH", "Value": "Used"},
         {"Field": "preInstalledSw", "Type": "TERM_MATCH", "Value": "NA"},
         {"Field": "tenancy", "Type": "TERM_MATCH", "Value": "shared"},
-    ]
-    + GET_PRODUCTS_PARAMS["Filters"],
+        *GET_PRODUCTS_PARAMS["Filters"],
+    ],
 }
 
 

tests/tests_e3_aws/troposphere/apigateway/apigateway_test.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from typing import Any, cast
 import json
 import os
 import pytest
@@ -283,7 +284,9 @@
     },
     "TestapiIntegration": {
         "Properties": {
-            **EXPECTED_TEMPLATE["TestapiIntegration"]["Properties"],
+            **cast(
+                dict[str, Any], EXPECTED_TEMPLATE["TestapiIntegration"]["Properties"]
+            ),
             "IntegrationUri": "arn:aws:lambda:eu-west-1:123456789012:function:"
             "mypylambda:${stageVariables.lambdaAlias}",
         },