Merge pull request #124 from asfadmin/add-ruff

replace flake8 with ruff
asfadmin · Jan 2, 2025 · dee27ae · dee27ae
2 parents 71bade0 + 735306b
commit dee27ae
Show file tree

Hide file tree

Showing 6 changed files with 101 additions and 63 deletions.
diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml
@@ -4,20 +4,8 @@ on: push
 
 jobs:
 
-  flake8:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-
-      - run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 flake8-import-order flake8-builtins  # FIXME add flake8-blind-except
-      - run: flake8 --max-line-length=120 --import-order-style=pycharm --statistics
+  call-ruff-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
 
   cfn-lint:
     runs-on: ubuntu-latest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.1]
+
+### Changed
+- Replaced `flake8` with `ruff`.
+
 ## [0.2.0]
 
 ### Changed

diff --git a/ems-report/src/ems_report.py b/ems-report/src/ems_report.py
@@ -21,17 +21,21 @@ def setup():
 
 
 def get_elasticsearch_connection(host):
-    auth = AWSRequestsAuth(aws_access_key=getenv('AWS_ACCESS_KEY_ID'),
-                           aws_secret_access_key=getenv('AWS_SECRET_ACCESS_KEY'),
-                           aws_token=getenv('AWS_SESSION_TOKEN'),
-                           aws_host=host,
-                           aws_region=getenv('AWS_REGION'),
-                           aws_service='es')
-    es = Elasticsearch(hosts=[{'host': host, 'port': 443}],
-                       use_ssl=True,
-                       verify_certs=True,
-                       http_auth=auth,
-                       connection_class=RequestsHttpConnection)
+    auth = AWSRequestsAuth(
+        aws_access_key=getenv('AWS_ACCESS_KEY_ID'),
+        aws_secret_access_key=getenv('AWS_SECRET_ACCESS_KEY'),
+        aws_token=getenv('AWS_SESSION_TOKEN'),
+        aws_host=host,
+        aws_region=getenv('AWS_REGION'),
+        aws_service='es',
+    )
+    es = Elasticsearch(
+        hosts=[{'host': host, 'port': 443}],
+        use_ssl=True,
+        verify_certs=True,
+        http_auth=auth,
+        connection_class=RequestsHttpConnection,
+    )
     return es
 
 
@@ -71,14 +75,16 @@ def upload_report(records, report_date, config):
         for r in records:
             r['category'] = get_category(r['file_name'])
             r['parsed_time'] = datetime.strptime(r['request_time'], '%Y-%m-%dT%H:%M:%S+00:00')
-            f.write('[{parsed_time:%d/%b/%Y:%H:%M:%S}]|&|{category}|&|{ip_address}|&|{user_id}|&|{bytes_sent}|&|'
-                    '{http_status}\n'.format(**r))
+            f.write(
+                '[{parsed_time:%d/%b/%Y:%H:%M:%S}]|&|{category}|&|{ip_address}|&|{user_id}|&|{bytes_sent}|&|'
+                '{http_status}\n'.format(**r)
+            )
         f.flush()
         s3.upload_file(f.name, config['bucket'], report_name)
 
 
 def generate_ems_report(report_date, config):
-    log.info('Generating GRFN EMS report for {:%Y-%m-%d}'.format(report_date))
+    log.info(f'Generating GRFN EMS report for {report_date:%Y-%m-%d}')
     records = get_records(report_date, config['elasticsearch'])
     upload_report(records, report_date, config['output'])
 

diff --git a/log-parse/src/log_parse.py b/log-parse/src/log_parse.py
@@ -21,11 +21,11 @@
     'mappings': {
         'properties': {
             'request_time': {'type': 'date'},
-            'file_name':    {'type': 'keyword'},
-            'user_id':      {'type': 'keyword'},
-            'ip_address':   {'type': 'ip'},
-            'http_status':  {'type': 'long'},
-            'bytes_sent':   {'type': 'long'},
+            'file_name': {'type': 'keyword'},
+            'user_id': {'type': 'keyword'},
+            'ip_address': {'type': 'ip'},
+            'http_status': {'type': 'long'},
+            'bytes_sent': {'type': 'long'},
         },
     },
     'settings': {
@@ -42,17 +42,21 @@ def setup():
 
 
 def get_elasticsearch_connection(host):
-    auth = AWSRequestsAuth(aws_access_key=getenv('AWS_ACCESS_KEY_ID'),
-                           aws_secret_access_key=getenv('AWS_SECRET_ACCESS_KEY'),
-                           aws_token=getenv('AWS_SESSION_TOKEN'),
-                           aws_host=host,
-                           aws_region=getenv('AWS_REGION'),
-                           aws_service='es')
-    es = Elasticsearch(hosts=[{'host': host, 'port': 443}],
-                       use_ssl=True,
-                       verify_certs=True,
-                       http_auth=auth,
-                       connection_class=RequestsHttpConnection)
+    auth = AWSRequestsAuth(
+        aws_access_key=getenv('AWS_ACCESS_KEY_ID'),
+        aws_secret_access_key=getenv('AWS_SECRET_ACCESS_KEY'),
+        aws_token=getenv('AWS_SESSION_TOKEN'),
+        aws_host=host,
+        aws_region=getenv('AWS_REGION'),
+        aws_service='es',
+    )
+    es = Elasticsearch(
+        hosts=[{'host': host, 'port': 443}],
+        use_ssl=True,
+        verify_certs=True,
+        http_auth=auth,
+        connection_class=RequestsHttpConnection,
+    )
     return es
 
 
@@ -86,14 +90,15 @@ def get_cloudfront_records(bucket, key):
     marshalled_records = [
         {
             '_id': record[14],
-            'request_time': datetime.strptime(record[0]+record[1]+'+0000', "%Y-%m-%d%H:%M:%S%z"),
+            'request_time': datetime.strptime(record[0] + record[1] + '+0000', '%Y-%m-%d%H:%M:%S%z'),
             'ip_address': record[4],
             'file_name': basename(record[7]),
             'user_id': get_user_id(record[11]),
             'http_status': to_number(record[8]),
             'bytes_sent': to_number(record[3]),
         }
-        for record in records if not record[0].startswith('#') and record[5] == 'GET' and record[8] in ['200', '206']
+        for record in records
+        if not record[0].startswith('#') and record[5] == 'GET' and record[8] in ['200', '206']
     ]
     return marshalled_records
 
@@ -103,6 +108,6 @@ def lambda_handler(event, context):
     for record in event['Records']:
         bucket = record['s3']['bucket']['name']
         key = record['s3']['object']['key']
-        log.info('Processing file s3://{0}/{1}'.format(bucket, key))
+        log.info(f'Processing file s3://{bucket}/{key}')
         records = get_cloudfront_records(bucket, key)
         update_elasticsearch(records, config)
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,30 @@
+[project]
+requires-python = "==3.12"
+
+[tool.ruff]
+line-length = 120
+# The directories to consider when resolving first- vs. third-party imports.
+# See: https://docs.astral.sh/ruff/settings/#src
+src = ["**/src", "tests"]
+
+[tool.ruff.format]
+indent-style = "space"
+quote-style = "single"
+
+[tool.ruff.lint]
+extend-select = [
+    "I",   # isort: https://docs.astral.sh/ruff/rules/#isort-i
+
+    # TODO: Uncomment the following extensions and address their warnings:
+    #"UP",  # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    #"D",   # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d
+    #"ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
+    #"PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+case-sensitive = true
+lines-after-imports = 2
diff --git a/tests/test_ems_report.py b/tests/test_ems_report.py
@@ -2,19 +2,23 @@
 
 
 def test_get_category():
-    assert ems_report.get_category('foo-v1.png') \
-           == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
-    assert ems_report.get_category('foo-v1') \
-           == 'SENTINEL-1_INTERFEROGRAMS'
-    assert ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6.png') \
-           == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
-    assert ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6') \
-           == 'SENTINEL-1_INTERFEROGRAMS'
-    assert ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1.png') \
-           == 'ARIA_S1_GUNW_BROWSE'
-    assert ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1') \
-           == 'ARIA_S1_GUNW'
-    assert ems_report.get_category('foo-v4.png') \
-           == 'ARIA_S1_GUNW_BROWSE'
-    assert ems_report.get_category('foo-v4') \
-           == 'ARIA_S1_GUNW'
+    assert ems_report.get_category('foo-v1.png') == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
+    assert ems_report.get_category('foo-v1') == 'SENTINEL-1_INTERFEROGRAMS'
+    assert (
+        ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6.png')
+        == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
+    )
+    assert (
+        ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6')
+        == 'SENTINEL-1_INTERFEROGRAMS'
+    )
+    assert (
+        ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1.png')
+        == 'ARIA_S1_GUNW_BROWSE'
+    )
+    assert (
+        ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1')
+        == 'ARIA_S1_GUNW'
+    )
+    assert ems_report.get_category('foo-v4.png') == 'ARIA_S1_GUNW_BROWSE'
+    assert ems_report.get_category('foo-v4') == 'ARIA_S1_GUNW'