diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8fa74448..72e26579 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,4 +14,4 @@ on: jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.13.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 129ff5f1..7646baa5 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.13.2 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 4b923e29..c549b2cb 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -13,4 +13,4 @@ on: jobs: call-labeled-pr-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.13.2 diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml index 9b8cb038..174418bc 100644 --- a/.github/workflows/release-checklist-comment.yml +++ b/.github/workflows/release-checklist-comment.yml @@ -10,7 +10,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.13.2 permissions: pull-requests: write secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f189f068..4ad84e3e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.13.2 with: release_prefix: GRFN Ingest release_branch: prod diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 2260fc12..37bef528 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -4,20 +4,8 @@ on: push jobs: - flake8: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - - run: | - python -m pip install --upgrade pip - python -m pip install flake8 flake8-import-order flake8-builtins # FIXME add flake8-blind-except - - run: flake8 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names metadata_construction verify + call-ruff-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.13.2 cfn-lint: runs-on: ubuntu-latest diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index b8adefe6..6afa9a86 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -8,7 +8,7 @@ on: jobs: call-bump-version-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.13.2 with: user: tools-bot email: UAF-asf-apd@alaska.edu diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4d0edd1..4978a735 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,9 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: mamba-org/provision-with-micromamba@v16 + - uses: mamba-org/setup-micromamba@v2 + with: + environment-file: environment.yml - shell: bash -l {0} run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f05ed9..3460dccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.2] +### Changed +- Replaced `flake8` with `ruff`. + ## [2.0.1] ### Changed - Upgraded all Lambda functions to Python 3.12 diff --git a/cmr-token/src/cmr_token.py b/cmr-token/src/cmr_token.py index 4bde5d36..e4b15ac3 100644 --- a/cmr-token/src/cmr_token.py +++ b/cmr-token/src/cmr_token.py @@ -6,6 +6,7 @@ import boto3 import requests_pkcs12 + log = getLogger() log.setLevel('INFO') s3 = boto3.client('s3') diff --git a/ingest/src/ingest.py b/ingest/src/ingest.py index 4d45d03e..f0aa17ab 100755 --- a/ingest/src/ingest.py +++ b/ingest/src/ingest.py @@ -51,7 +51,7 @@ def lambda_handler(event, context): 'Browse': { 'Bucket': config['browse_bucket'], 'Key': browse_output_key, - } + }, } log.info('Done processing %s', event['ProductName']) return output diff --git a/invoke/src/invoke.py b/invoke/src/invoke.py index abec94e2..37f9fb20 100644 --- a/invoke/src/invoke.py +++ b/invoke/src/invoke.py @@ -39,8 +39,9 @@ def invoke_ingest(config): log.warning('Processed %s of %s messages. Exiting.', messages_processed, config['max_messages_to_process']) break - messages = queue.receive_messages(MaxNumberOfMessages=config['max_messages_per_receive'], - WaitTimeSeconds=config['wait_time_in_seconds']) + messages = queue.receive_messages( + MaxNumberOfMessages=config['max_messages_per_receive'], WaitTimeSeconds=config['wait_time_in_seconds'] + ) if not messages: log.info('No messages found. Exiting.') break diff --git a/metadata-construction/src/metadata_construction.py b/metadata-construction/src/metadata_construction.py index 1137e11c..a8ec0e5c 100644 --- a/metadata-construction/src/metadata_construction.py +++ b/metadata-construction/src/metadata_construction.py @@ -5,6 +5,7 @@ import boto3 + log = getLogger() log.setLevel('INFO') CONFIG = json.loads(os.getenv('CONFIG')) @@ -51,7 +52,7 @@ def get_sds_metadata(obj): def format_polygon(polygon): coordinates = [] for long, lat in reversed(polygon): - coordinates.append({"Latitude": lat, "Longitude": long}) + coordinates.append({'Latitude': lat, 'Longitude': long}) return coordinates @@ -111,38 +112,32 @@ def render_granule_metadata(sds_metadata, config, product, browse) -> dict: 'Type': 'Update', }, ], - "DataGranule": { - "ArchiveAndDistributionInformation": [ - { - "Name": os.path.basename(product['Key']), - "SizeInBytes": get_s3_file_size(product) - } + 'DataGranule': { + 'ArchiveAndDistributionInformation': [ + {'Name': os.path.basename(product['Key']), 'SizeInBytes': get_s3_file_size(product)} ], - "DayNightFlag": "Unspecified", - "ProductionDateTime": sds_metadata['creation_timestamp'] + 'DayNightFlag': 'Unspecified', + 'ProductionDateTime': sds_metadata['creation_timestamp'], }, - "Platforms": [ - {"ShortName": platform} for platform in sorted(set(sds_metadata['metadata']['platform'])) - ], - "OrbitCalculatedSpatialDomains": [ - {"OrbitNumber": orbit} for orbit in sds_metadata['metadata']['orbit_number'] + 'Platforms': [{'ShortName': platform} for platform in sorted(set(sds_metadata['metadata']['platform']))], + 'OrbitCalculatedSpatialDomains': [{'OrbitNumber': orbit} for orbit in sds_metadata['metadata']['orbit_number']], + 'InputGranules': sds_metadata['metadata']['reference_scenes'] + sds_metadata['metadata']['secondary_scenes'], + 'AdditionalAttributes': [ + {'Name': 'ASCENDING_DESCENDING', 'Values': [sds_metadata['metadata']['orbit_direction']]}, + {'Name': 'BEAM_MODE', 'Values': [sds_metadata['metadata']['beam_mode']]}, + {'Name': 'POLARIZATION', 'Values': [sds_metadata['metadata']['polarization']]}, + {'Name': 'PERPENDICULAR_BASELINE', 'Values': [str(sds_metadata['metadata']['perpendicular_baseline'])]}, + {'Name': 'VERSION', 'Values': [sds_metadata['metadata']['version']]}, + {'Name': 'FRAME_NUMBER', 'Values': [str(sds_metadata['metadata']['frame_number'])]}, + {'Name': 'PATH_NUMBER', 'Values': [str(sds_metadata['metadata']['track_number'])]}, + {'Name': 'TEMPORAL_BASELINE_DAYS', 'Values': [str(sds_metadata['metadata']['temporal_baseline_days'])]}, ], - "InputGranules": sds_metadata['metadata']['reference_scenes'] + sds_metadata['metadata']['secondary_scenes'], - "AdditionalAttributes": [ - {"Name": "ASCENDING_DESCENDING", "Values": [sds_metadata['metadata']['orbit_direction']]}, - {"Name": "BEAM_MODE", "Values": [sds_metadata['metadata']['beam_mode']]}, - {"Name": "POLARIZATION", "Values": [sds_metadata['metadata']['polarization']]}, - {"Name": "PERPENDICULAR_BASELINE", "Values": [str(sds_metadata['metadata']['perpendicular_baseline'])]}, - {"Name": "VERSION", "Values": [sds_metadata['metadata']['version']]}, - {"Name": "FRAME_NUMBER", "Values": [str(sds_metadata['metadata']['frame_number'])]}, - {"Name": "PATH_NUMBER", "Values": [str(sds_metadata['metadata']['track_number'])]}, - {"Name": "TEMPORAL_BASELINE_DAYS", "Values": [str(sds_metadata['metadata']['temporal_baseline_days'])]} - ] } if 'weather_model' in sds_metadata['metadata']: - umm['AdditionalAttributes'].append({"Name": "WEATHER_MODEL", - "Values": sds_metadata['metadata']['weather_model']}) + umm['AdditionalAttributes'].append( + {'Name': 'WEATHER_MODEL', 'Values': sds_metadata['metadata']['weather_model']} + ) return umm diff --git a/metadata-to-cmr/src/cmr.py b/metadata-to-cmr/src/cmr.py index 92005879..88e9d5b4 100644 --- a/metadata-to-cmr/src/cmr.py +++ b/metadata-to-cmr/src/cmr.py @@ -9,6 +9,7 @@ import boto3 import requests + log = getLogger() diff --git a/metadata-to-cmr/src/daemon.py b/metadata-to-cmr/src/daemon.py index df498846..c335cdaf 100644 --- a/metadata-to-cmr/src/daemon.py +++ b/metadata-to-cmr/src/daemon.py @@ -5,6 +5,7 @@ import boto3 from botocore.client import Config from botocore.exceptions import ClientError + from cmr import get_session, process_task @@ -44,8 +45,11 @@ def daemon_loop(config, get_remaining_time_in_millis_fcn): sfn_client = get_sfn_client(config['sfn_connect_timeout']) while True: if get_remaining_time_in_millis_fcn() < config['max_task_time_in_millis']: - log.info('Remaining time %s less than max task time %s. Exiting.', get_remaining_time_in_millis_fcn(), - config['max_task_time_in_millis']) + log.info( + 'Remaining time %s less than max task time %s. Exiting.', + get_remaining_time_in_millis_fcn(), + config['max_task_time_in_millis'], + ) break task = get_task(sfn_client, config['activity']) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9717bc2e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[project] +requires-python = "==3.12" + +[tool.ruff] +line-length = 120 +# The directories to consider when resolving first- vs. third-party imports. +# See: https://docs.astral.sh/ruff/settings/#src +src = ["**/src", "tests"] + +[tool.ruff.format] +indent-style = "space" +quote-style = "single" + +[tool.ruff.lint] +extend-select = [ + "I", # isort: https://docs.astral.sh/ruff/rules/#isort-i + "UP", # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up + + # TODO: uncomment the following extensions and address their warnings: + #"D", # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d + #"ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + #"PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +case-sensitive = true +lines-after-imports = 2 diff --git a/requirements-cmr-token.txt b/requirements-cmr-token.txt index 7f88e884..ba30a67a 100644 --- a/requirements-cmr-token.txt +++ b/requirements-cmr-token.txt @@ -1,2 +1,2 @@ -boto3==1.34.131 +boto3==1.35.90 requests-pkcs12==1.25 diff --git a/requirements-ingest.txt b/requirements-ingest.txt index b0b6f1c7..0e8753fd 100644 --- a/requirements-ingest.txt +++ b/requirements-ingest.txt @@ -1 +1 @@ -boto3==1.34.131 +boto3==1.35.90 diff --git a/requirements-invoke.txt b/requirements-invoke.txt index b0b6f1c7..0e8753fd 100644 --- a/requirements-invoke.txt +++ b/requirements-invoke.txt @@ -1 +1 @@ -boto3==1.34.131 +boto3==1.35.90 diff --git a/requirements-metadata-construction.txt b/requirements-metadata-construction.txt index b0b6f1c7..0e8753fd 100644 --- a/requirements-metadata-construction.txt +++ b/requirements-metadata-construction.txt @@ -1 +1 @@ -boto3==1.34.131 +boto3==1.35.90 diff --git a/requirements-metadata-to-cmr.txt b/requirements-metadata-to-cmr.txt index 92db5fe9..a1d00c6e 100644 --- a/requirements-metadata-to-cmr.txt +++ b/requirements-metadata-to-cmr.txt @@ -1,2 +1,2 @@ -boto3==1.34.131 +boto3==1.35.90 requests==2.32.3 diff --git a/requirements-notify.txt b/requirements-notify.txt index b0b6f1c7..0e8753fd 100644 --- a/requirements-notify.txt +++ b/requirements-notify.txt @@ -1 +1 @@ -boto3==1.34.131 +boto3==1.35.90 diff --git a/requirements-verify.txt b/requirements-verify.txt index 5fe66a32..4c628235 100644 --- a/requirements-verify.txt +++ b/requirements-verify.txt @@ -1,2 +1,2 @@ -boto3==1.34.131 -jsonschema==4.22.0 +boto3==1.35.90 +jsonschema==4.23.0 diff --git a/tests/test_metadata_construction.py b/tests/test_metadata_construction.py index cbcecea0..487b4ffb 100644 --- a/tests/test_metadata_construction.py +++ b/tests/test_metadata_construction.py @@ -19,7 +19,7 @@ def test_get_file_content_from_s3(s3_stubber): s3_stubber.add_response( method='get_object', expected_params={'Bucket': 'myBucket', 'Key': 'myKey'}, - service_response={'Body': io.StringIO('myContent')} + service_response={'Body': io.StringIO('myContent')}, ) assert metadata_construction.get_file_content_from_s3('myBucket', 'myKey') == 'myContent' @@ -32,10 +32,7 @@ def test_write_to_file(tmp_path): def test_get_s3_file_size(s3_stubber): - obj = { - 'Bucket': 'myBucket', - 'Key': 'myKey' - } + obj = {'Bucket': 'myBucket', 'Key': 'myKey'} s3_stubber.add_response(method='head_object', expected_params=obj, service_response={'ContentLength': 123}) assert metadata_construction.get_s3_file_size(obj) == 123 @@ -43,7 +40,7 @@ def test_get_s3_file_size(s3_stubber): def test_get_sds_metadata(test_data_dir, s3_stubber): obj = { 'Bucket': 'ingest-test-aux', - 'Key': 'S1-GUNW-D-R-123-tops-20240212_20240107-032647-00038E_00036N-PP-2e78-v3_0_0' + 'Key': 'S1-GUNW-D-R-123-tops-20240212_20240107-032647-00038E_00036N-PP-2e78-v3_0_0', } sds_metadata_file = test_data_dir / 'granule1' / 'sds_metadata.json' @@ -55,7 +52,7 @@ def test_get_sds_metadata(test_data_dir, s3_stubber): def test_create_granule_metadata_in_s3_g1(test_data_dir, mocker): - sds_metadata =json.loads((test_data_dir / 'granule1'/ 'sds_metadata.json').read_text()) + sds_metadata = json.loads((test_data_dir / 'granule1' / 'sds_metadata.json').read_text()) inputs = json.loads((test_data_dir / 'granule1' / 'inputs.json').read_text()) config = json.loads((test_data_dir / 'granule1' / 'config.json').read_text()) @@ -80,7 +77,7 @@ def test_create_granule_metadata_in_s3_g1(test_data_dir, mocker): def test_create_granule_metadata_in_s3_g2(test_data_dir, mocker): - sds_metadata =json.loads((test_data_dir / 'granule2'/ 'sds_metadata.json').read_text()) + sds_metadata = json.loads((test_data_dir / 'granule2' / 'sds_metadata.json').read_text()) inputs = json.loads((test_data_dir / 'granule2' / 'inputs.json').read_text()) config = json.loads((test_data_dir / 'granule2' / 'config.json').read_text()) diff --git a/tests/test_verify.py b/tests/test_verify.py index 6d449a2c..76fe875d 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -17,7 +17,7 @@ def test_get_file_content_from_s3(s3_stubber): s3_stubber.add_response( method='get_object', expected_params={'Bucket': 'myBucket', 'Key': 'myKey'}, - service_response={'Body': io.StringIO('myContent')} + service_response={'Body': io.StringIO('myContent')}, ) assert verify.get_file_content_from_s3('myBucket', 'myKey') == 'myContent' diff --git a/verify/src/verify.py b/verify/src/verify.py index d5c0852a..b9a2c558 100644 --- a/verify/src/verify.py +++ b/verify/src/verify.py @@ -35,7 +35,7 @@ def get_file_content_from_s3(bucket, key): def get_json_from_file(filename): - with open(filename, 'r') as f: + with open(filename) as f: content = f.read() return json.loads(content)