maintenance: (#18)

- bump dependencies
 - add python3.12 support
 - add flake8 linter
 - replace use of toml library with tomli or tomllib, depending on python version

.github/workflows/publish-package.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install poetry==1.7.1
+        pip install poetry==1.8.4
     - name: Build and publish
       env:
         PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v2
@@ -29,4 +29,4 @@ jobs:
         pip install tox
     - name: Test and run checks
       run: |
-        tox -e py,lint,typing
+        tox -e py,lint

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
@@ -12,8 +12,14 @@ repos:
       - id: isort
         args: [ --profile, black ]
 
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.13.0
+    hooks:
+      - id: mypy
+        exclude: '^tests/'
+
   - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         language: python

philter_lite/filters/__init__.py

@@ -1,10 +1,14 @@
 import os
 import re
+import sys
 import warnings
 from dataclasses import dataclass
 from typing import List, Optional, Pattern, Set
 
-import toml
+if sys.version_info < (3, 11):
+    import tomli as tomllib
+else:
+    import tomllib
 
 from philter_lite.filters import filter_db
 
@@ -136,8 +140,9 @@ def load_filters(filter_path) -> List[Filter]:
     """
     if not os.path.exists(filter_path):
         raise Exception("Filepath does not exist", filter_path)
-    with open(filter_path, "r") as fil_file:
-        return [filter_from_dict(x) for x in toml.loads(fil_file.read())["filters"]]
+    with open(filter_path, "rb") as fil_file:
+        filters_toml = tomllib.load(fil_file)
+    return [filter_from_dict(x) for x in filters_toml["filters"]]
 
 
 def _precompile(regex: str) -> Pattern[str]:

philter_lite/filters/filter_db.py

@@ -1,21 +1,26 @@
+import sys
 from importlib import resources
 from typing import Any, MutableMapping
 
-import toml
+if sys.version_info < (3, 11):
+    import tomli as tomllib
+else:
+    import tomllib
+
 
 from philter_lite import filters
 
 
 def load_regex_db() -> MutableMapping[str, Any]:
-    return toml.loads(resources.read_text(filters, "regex.toml"))
+    return tomllib.loads(resources.read_text(filters, "regex.toml"))
 
 
 def load_regex_context_db() -> MutableMapping[str, Any]:
-    return toml.loads(resources.read_text(filters, "regex_context.toml"))
+    return tomllib.loads(resources.read_text(filters, "regex_context.toml"))
 
 
 def load_set_db() -> MutableMapping[str, Any]:
-    return toml.loads(resources.read_text(filters, "set.toml"))
+    return tomllib.loads(resources.read_text(filters, "set.toml"))
 
 
 regex_db: MutableMapping[str, Any] = load_regex_db()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "philter-lite"
-version = "0.5.0"
+version = "0.5.1"
 description = "Open-source PHI-filtering software. A fork of philter-ucsf."
 readme = "README.md"
 authors = [
@@ -14,26 +14,27 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
 ]
 
 [tool.poetry.dependencies]
 python = "^3.9"
 chardet = "^5.2.0"
-nltk = "^3.8.1"
-xmltodict = "^0.13.0"
-toml = "^0.10.0"
+nltk = "^3.9.1"
+xmltodict = "^0.14.2"
+tomli = {version = "^2.1.0", python = "<3.11"}
 
-[tool.poetry.dev-dependencies]
-pre-commit = "^3.7.0"
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^4.0.1"
 pytest = "^8.1.1"
 
 [tool.poetry.scripts]
 philter_lite = 'philter_lite.main:main'
 
 [build-system]
-requires = ["poetry_core>=1.8.1"]
+requires = ["poetry_core>=1.9.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.black]

tests/test_package_metadata.py

@@ -1,15 +1,22 @@
 """Confirm that project metadata is set correctly."""
+
 import os
+import sys
 from typing import Any, Mapping
 
-import toml
+if sys.version_info < (3, 11):
+    import tomli as tomllib
+else:
+    import tomllib
+
 
 import philter_lite
 
 TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
 PYPROJECT_TOML_PATH = os.path.join(TESTS_DIR, "..", "pyproject.toml")
 
-pyproject_toml: Mapping[str, Any] = toml.load(PYPROJECT_TOML_PATH)
+with open(PYPROJECT_TOML_PATH, "rb") as pyproject_file:
+    pyproject_toml: Mapping[str, Any] = tomllib.load(pyproject_file)
 
 
 def test_version():

tests/test_philter.py

@@ -17,14 +17,14 @@ def test_filter_from_dict():
         "phi_type": "SOMETHING",
     }
 
-    filter = filter_from_dict(filter_dict)
+    my_filter = filter_from_dict(filter_dict)
 
-    assert filter.type == "regex"
-    assert filter.title == "test_city"
-    assert filter.data is not None
-    assert filter.exclude == "test_ex"
-    assert filter.phi_type == "SOMETHING"
-    assert isinstance(filter, filters.RegexFilter)
+    assert my_filter.type == "regex"
+    assert my_filter.title == "test_city"
+    assert my_filter.data is not None
+    assert my_filter.exclude == "test_ex"
+    assert my_filter.phi_type == "SOMETHING"
+    assert isinstance(my_filter, filters.RegexFilter)
 
     filter_dict = {
         "title": "Find Names 1",
@@ -36,15 +36,15 @@ def test_filter_from_dict():
         "phi_type": "Something",
     }
 
-    filter = filter_from_dict(filter_dict)
+    my_filter = filter_from_dict(filter_dict)
 
-    assert filter.type == "regex_context"
-    assert filter.title == "Find Names 1"
-    assert filter.data is not None
-    assert filter.exclude is True
-    assert filter.context == "right"
-    assert filter.context_filter == "Firstnames Blacklist"
-    assert isinstance(filter, filters.RegexContextFilter)
+    assert my_filter.type == "regex_context"
+    assert my_filter.title == "Find Names 1"
+    assert my_filter.data is not None
+    assert my_filter.exclude is True
+    assert my_filter.context == "right"
+    assert my_filter.context_filter == "Firstnames Blacklist"
+    assert isinstance(my_filter, filters.RegexContextFilter)
 
     filter_dict = {
         "title": "Whitelist 1",
@@ -55,14 +55,14 @@ def test_filter_from_dict():
         "phi_type": "Something",
     }
 
-    filter = filter_from_dict(filter_dict)
+    my_filter = filter_from_dict(filter_dict)
 
-    assert filter.type == "set"
-    assert filter.title == "Whitelist 1"
-    assert filter.data is not None
-    assert filter.exclude is False
-    assert filter.pos == set()
-    assert isinstance(filter, filters.SetFilter)
+    assert my_filter.type == "set"
+    assert my_filter.title == "Whitelist 1"
+    assert my_filter.data is not None
+    assert my_filter.exclude is False
+    assert my_filter.pos == set()
+    assert isinstance(my_filter, filters.SetFilter)
 
     filter_dict = {
         "title": "POS MATCHER",
@@ -72,13 +72,13 @@ def test_filter_from_dict():
         "phi_type": "OTHER",
     }
 
-    filter = filter_from_dict(filter_dict)
+    my_filter = filter_from_dict(filter_dict)
 
-    assert filter.type == "pos_matcher"
-    assert filter.title == "POS MATCHER"
-    assert filter.exclude is False
-    assert filter.pos == ["CD"]
-    assert isinstance(filter, filters.PosFilter)
+    assert my_filter.type == "pos_matcher"
+    assert my_filter.title == "POS MATCHER"
+    assert my_filter.exclude is False
+    assert my_filter.pos == ["CD"]
+    assert isinstance(my_filter, filters.PosFilter)
 
 
 def test_filter_from_dict_missing_phi_type():
@@ -90,8 +90,8 @@ def test_filter_from_dict_missing_phi_type():
         "notes": "test_notes",
     }
 
-    filter = filter_from_dict(filter_dict)
-    assert filter.phi_type == "OTHER"
+    my_filter = filter_from_dict(filter_dict)
+    assert my_filter.phi_type == "OTHER"
 
 
 def test_filter_from_dict_missing_file():

tox.ini

@@ -1,22 +1,15 @@
 [tox]
-envlist = typing,lint,py39,py310,py311
+envlist = lint,py39,py310,py311,py312
 isolated_build = True
 
 [testenv]
 deps =
     pytest
 commands_pre =
-    python -c 'import nltk; nltk.download("averaged_perceptron_tagger")'
+    python -c 'import nltk; nltk.download("averaged_perceptron_tagger_eng")'
 commands = python -m pytest {posargs}
 
 [lint]
 deps = pre-commit
 commands = pre-commit run --all-files
 skip_install = True
-
-
-[typing]
-deps =
-    types-toml
-    mypy
-commands = mypy philter_lite/