From 355e44b8a950e575626abaa85ca967a740a83281 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 15:07:56 +0200 Subject: [PATCH 01/56] test: add reproducibility tests for BH models --- tests/test_examples/test_brock_hommes.py | 198 +++++++++++------------ 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/tests/test_examples/test_brock_hommes.py b/tests/test_examples/test_brock_hommes.py index ee035466..9cd4e372 100644 --- a/tests/test_examples/test_brock_hommes.py +++ b/tests/test_examples/test_brock_hommes.py @@ -142,105 +142,105 @@ def test_bh4_model() -> None: seed = 42 expected_output = np.array( [ - [0.08922630308955377], - [0.05634330307256878], - [0.06414066791141397], - [0.10508080199462444], - [0.06465297620368961], - [0.03484719426768868], - [0.08615989071274688], - [0.0913785448619035], - [0.045053375926644455], - [0.0518594284576269], - [0.01784820234069612], - [-0.0067186064600018615], - [0.0044211074099892765], - [-0.07247632349091557], - [-0.12025482695843324], - [-0.10683951117309803], - [-0.11411574029353017], - [-0.06706413981572737], - [-0.08159742266533461], - [-0.11298336100607148], - [-0.02118343228750438], - [-0.02181210940529453], - [-0.012505333335104242], - [-0.06493466919203203], - [-0.06777317743401062], - [-0.04278994714826678], - [-0.07482654477671165], - [-0.037818804665372116], - [-0.04933496149096962], - [-0.046094271127451104], - [-0.05581557283042439], - [0.034347731103219224], - [0.02512761448471606], - [-0.024599348685024267], - [0.014483613538501935], - [-0.03749490281067554], - [-0.018848070441442842], - [-0.0903445511592709], - [-0.11689354629214414], - [-0.07402552988455961], - [-0.0213222075208419], - [-0.006964839502735555], - [-0.009134785382048373], - [-0.01832020700387496], - [-0.07147812371256342], - [-0.07928009011168288], - [-0.07352800331763301], - [-0.009140520210023257], - [0.008546944049819353], - [-0.06352846392417359], - [-0.03278851202689395], - [-0.0374368443569739], - [-0.052935038718777555], - [-0.012897736194776518], - [0.03266695765916192], - [0.06049519104165061], - [0.00938186044487789], - [-0.0067468712667628295], - [0.008106506614296927], - [0.044569547984167394], - [0.012741617482752196], - [0.0008699617587074661], - [-0.04344780597100196], - [-0.07846817350512286], - [-0.023083526873630755], - [0.038777720968499366], - [0.02531720124286847], - [0.05708022146835147], - [0.0546597682993881], - [0.012409093566453908], - [0.02209176221801424], - [0.07646934910751865], - [0.05285043719427516], - [0.09823849499036635], - [-0.034478225021676856], - [0.005964825849260701], - [0.00840848077394073], - [-0.005946308824872809], - [-0.0007876637933463152], - [-0.07916013182054467], - [-0.06531819165292246], - [-0.030998771363946246], - [0.037675659019960714], - [0.007016669311064504], - [-0.02775270057141131], - [-0.03985857414739906], - [0.00829644879270901], - [0.019749784337136797], - [-0.007020082438403985], - [0.014910909631843444], - [0.014652643490741945], - [0.04854568430373574], - [0.006634050123336816], - [-0.009202535783939553], - [-0.022243503584838255], - [-0.0736884382963828], - [-0.040546552819254726], - [-0.017187399613289116], - [-0.011276721036127601], + [0.0892263], + [0.0563433], + [0.06414067], + [0.1050808], + [0.06465298], + [0.03484719], + [0.08615989], + [0.09137854], + [0.04505338], + [0.05185943], + [0.0178482], + [-0.00671861], + [0.00442111], + [-0.07247632], + [-0.12025483], + [-0.10683951], + [-0.11411574], + [-0.06706414], + [-0.08159742], + [-0.11298336], + [-0.02118343], + [-0.02181211], + [-0.01250533], + [-0.06493467], + [-0.06777318], + [-0.04278995], + [-0.07482654], + [-0.0378188], + [-0.04933496], + [-0.04609427], + [-0.05581557], + [0.03434773], + [0.02512761], + [-0.02459935], + [0.01448361], + [-0.0374949], + [-0.01884807], + [-0.09034455], + [-0.11689355], + [-0.07402553], + [-0.02132221], + [-0.00696484], + [-0.00913479], + [-0.01832021], + [-0.07147812], + [-0.07928009], + [-0.073528], + [-0.00914052], + [0.00854694], + [-0.06352846], + [-0.03278851], + [-0.03743684], + [-0.05293504], + [-0.01289774], + [0.03266696], + [0.06049519], + [0.00938186], + [-0.00674687], + [0.00810651], + [0.04456955], + [0.01274162], + [0.00086996], + [-0.04344781], + [-0.07846817], + [-0.02308353], + [0.03877772], + [0.0253172], + [0.05708022], + [0.05465977], + [0.01240909], + [0.02209176], + [0.07646935], + [0.05285044], + [0.09823849], + [-0.03447823], + [0.00596483], + [0.00840848], + [-0.00594631], + [-0.00078766], + [-0.07916013], + [-0.06531819], + [-0.03099877], + [0.03767566], + [0.00701667], + [-0.0277527], + [-0.03985857], + [0.00829645], + [0.01974978], + [-0.00702008], + [0.01491091], + [0.01465264], + [0.04854568], + [0.00663405], + [-0.00920254], + [-0.0222435], + [-0.07368844], + [-0.04054655], + [-0.0171874], + [-0.01127672], [0.0], ] ) From cafc3383da62d0e50b8f842649f1a8c63c5d5629 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 17:30:22 +0200 Subject: [PATCH 02/56] dev: remove isort usage throughout the repository --- .github/workflows/lint.yml | 1 - CONTRIBUTING.md | 4 +--- Makefile | 22 ++-------------------- tox.ini | 13 +------------ 4 files changed, 4 insertions(+), 36 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 93e823e5..91db5a12 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,6 @@ jobs: - name: Code style check run: | tox -e black-check - tox -e isort-check tox -e flake8 tox -e vulture tox -e darglint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16e9db09..8e3aeb5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,16 +71,14 @@ make safety make bandit ``` -- To apply [`black`](https://black.readthedocs.io/en/stable/) and [`isort`](https://pycqa.github.io/isort/index.html) code formatters: +- To apply [`black`](https://black.readthedocs.io/en/stable/) code formatter: ``` make black -make isort ``` - whereas, to only check compliance: ``` make black-check -make isort-check ``` - To run tests: `make test`. diff --git a/Makefile b/Makefile index 9ba7a863..c512844b 100644 --- a/Makefile +++ b/Makefile @@ -63,10 +63,10 @@ clean-test: ## remove test and coverage artifacts rm -fr coverage.xml PHONY.: lint-all -lint-all: black isort flake8 static bandit safety vulture darglint pylint ## run all linters +lint-all: black flake8 static bandit safety vulture darglint pylint ## run all linters PHONY.: lint-all-files -lint-all-files: black-files isort-files flake8-files static-files bandit-files vulture-files darglint-files pylint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") +lint-all-files: black-files flake8-files static-files bandit-files vulture-files darglint-files pylint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") PHONY.: flake8 flake8: ## check style with flake8 @@ -86,24 +86,6 @@ static-files: ## static type checking with mypy for specific files (specified wi $(call check_defined, files) mypy $(files) -PHONY.: isort -isort: ## sort import statements with isort - isort black_it tests scripts examples - -PHONY.: isort-files -isort-files: ## sort import statements with isort for specific files (specified with files="file1 file2 somedir ...") - $(call check_defined, files) - isort $(files) - -PHONY.: isort-check -isort-check: ## check import statements order with isort - isort --check-only black_it tests scripts examples - -PHONY.: isort-check-files -isort-check-files: ## check import statements order with isort for specific files (specified with files="file1 file2 somedir ...") - $(call check_defined, files) - isort --check-only $(files) - PHONY.: black black: ## apply black formatting black black_it tests scripts examples diff --git a/tox.ini b/tox.ini index d7db8153..53535eca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = True -envlist = bandit, safety, check-copyright, black-check, isort-check, vulture, flake8, mypy, pylint, py3{8,9,10}, docs +envlist = bandit, safety, check-copyright, black-check, vulture, flake8, mypy, pylint, py3{8,9,10}, docs [tox:.package] # note tox will use the same python version as under what tox is installed to package @@ -48,7 +48,6 @@ deps = flake8-bugbear==23.3.12 flake8-docstrings==1.7.0 flake8-eradicate==1.5.0 - flake8-isort==6.0.0 pydocstyle==6.3.0 commands = flake8 black_it tests scripts examples @@ -80,16 +79,6 @@ skip_install = True deps = black==23.7.0 commands = black black_it tests scripts examples --check --verbose -[testenv:isort] -skip_install = True -deps = isort==5.12.0 -commands = isort black_it tests scripts examples - -[testenv:isort-check] -skip_install = True -deps = isort==5.12.0 -commands = isort --check-only black_it tests scripts examples - [testenv:bandit] skipsdist = True skip_install = True From 1e31ced20f3dba6e627db3f9694494e861f5e23e Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 17:33:18 +0200 Subject: [PATCH 03/56] dev: remove isort from pyproject.toml and poetry.lock --- poetry.lock | 44 +++++++++++++++++++++++++------------------- pyproject.toml | 2 -- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index e0af0390..1e5fdb93 100644 --- a/poetry.lock +++ b/poetry.lock @@ -659,6 +659,7 @@ files = [ {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"}, {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"}, {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"}, + {file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"}, {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"}, @@ -667,6 +668,7 @@ files = [ {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"}, {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"}, {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"}, + {file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"}, {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"}, @@ -675,6 +677,7 @@ files = [ {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"}, {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"}, {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"}, + {file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"}, {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"}, @@ -683,6 +686,7 @@ files = [ {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"}, {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"}, {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"}, + {file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"}, {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"}, @@ -1191,24 +1195,6 @@ attrs = "*" eradicate = ">=2.0,<3.0" flake8 = ">5" -[[package]] -name = "flake8-isort" -version = "6.0.0" -description = "flake8 plugin that integrates isort ." -optional = false -python-versions = ">=3.7" -files = [ - {file = "flake8-isort-6.0.0.tar.gz", hash = "sha256:537f453a660d7e903f602ecfa36136b140de279df58d02eb1b6a0c84e83c528c"}, - {file = "flake8_isort-6.0.0-py3-none-any.whl", hash = "sha256:aa0cac02a62c7739e370ce6b9c31743edac904bae4b157274511fc8a19c75bbc"}, -] - -[package.dependencies] -flake8 = "*" -isort = ">=5.0.0,<6" - -[package.extras] -test = ["pytest"] - [[package]] name = "fonttools" version = "4.42.1" @@ -2552,6 +2538,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -4069,6 +4065,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4076,8 +4073,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4094,6 +4098,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4101,6 +4106,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -5815,4 +5821,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "d2bfd21bed2cd4ed3aefe9fedd6af6543f9ddcc616316230d441e581dba7f3ae" +content-hash = "583ec2f6d97216eb3bd813386569d324a57f0b2008875da41658474396485915" diff --git a/pyproject.toml b/pyproject.toml index 82ad12b6..b30b77b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,10 +62,8 @@ flake8 = "^5.0.4" flake8-bugbear = "^23.3.12" flake8-docstrings = "^1.7.0" flake8-eradicate = "^1.5.0" -flake8-isort = "^6.0.0" hypothesis = "^6.82.6" ipython = "^8.3.0" -isort = "^5.12.0" jupyter = "^1.0.0" Markdown = "^3.4.4" markdown-include = "^0.8.1" From df7c051e4e841613b18b743892140ac44ff0f4e2 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 17:43:12 +0200 Subject: [PATCH 04/56] dev: remove pylint usage throughout the repository --- .github/workflows/lint.yml | 2 -- .pylintrc | 50 -------------------------------------- CONTRIBUTING.md | 1 - Makefile | 13 ++-------- tox.ini | 12 +-------- 5 files changed, 3 insertions(+), 75 deletions(-) delete mode 100644 .pylintrc diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 91db5a12..458b2417 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,8 +25,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: pip install tox - - name: Pylint - run: tox -e pylint - name: Code style check run: | tox -e black-check diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 9e22e3d5..00000000 --- a/.pylintrc +++ /dev/null @@ -1,50 +0,0 @@ -[MASTER] -ignore-patterns= -ignore= -ignore-paths=^(.*/|)examples/models/.*$ -load-plugins=pylint.extensions.no_self_use - -[MESSAGES CONTROL] -disable=C0103,C0301,R0801,W1514,W1505 - -# See here for more options: https://www.codeac.io/documentation/pylint-configuration.html - -[IMPORTS] -ignored-modules= -known-third-party=xgboost - -[DESIGN] -# min-public-methods=1 -# max-public-methods=36 -# max-returns=10 -# max-bool-expr=7 -max-args=7 -max-locals=20 -# max-statements=80 -# max-parents=10 -# max-branches=24 -# max-attributes=8 - -[REFACTORING] -# max-nested-blocks=6 - -[SPELLING] -# uncomment to enable -# spelling-dict=en_US - -# List of comma separated words that should not be checked. -spelling-ignore-words=nocover,pragma,params,noqa,kwargs,str,async,json,boolean,config,pytest,args,url,tx,jsonschema,traceback,api,nosec - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=10 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8e3aeb5b..4693d967 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,6 @@ We have various commands which are helpful during development. ``` make lint make static -make pylint make safety make bandit ``` diff --git a/Makefile b/Makefile index c512844b..0d154f9e 100644 --- a/Makefile +++ b/Makefile @@ -63,10 +63,10 @@ clean-test: ## remove test and coverage artifacts rm -fr coverage.xml PHONY.: lint-all -lint-all: black flake8 static bandit safety vulture darglint pylint ## run all linters +lint-all: black flake8 static bandit safety vulture darglint ## run all linters PHONY.: lint-all-files -lint-all-files: black-files flake8-files static-files bandit-files vulture-files darglint-files pylint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") +lint-all-files: black-files flake8-files static-files bandit-files vulture-files darglint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") PHONY.: flake8 flake8: ## check style with flake8 @@ -116,15 +116,6 @@ PHONY.: safety safety: ## run safety safety check -PHONY.: safety-files -pylint: ## run pylint - pylint black_it tests scripts examples - -PHONY.: pylint-files -pylint-files: ## run pylint for specific files (specified with files="file1 file2 somedir ...") - $(call check_defined, files) - pylint $(files) - PHONY.: vulture vulture: ## run vulture vulture black_it scripts/whitelists/package_whitelist.py diff --git a/tox.ini b/tox.ini index 53535eca..0c2392ad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = True -envlist = bandit, safety, check-copyright, black-check, vulture, flake8, mypy, pylint, py3{8,9,10}, docs +envlist = bandit, safety, check-copyright, black-check, vulture, flake8, mypy, py3{8,9,10}, docs [tox:.package] # note tox will use the same python version as under what tox is installed to package @@ -59,16 +59,6 @@ deps = commands = mypy black_it tests scripts examples -[testenv:pylint] -skipdist = True -deps = - pylint==2.17.5 - pytest==7.4.0 - hypothesis[numpy]==6.82.6 - mesa==2.1.1 - mistletoe==1.2.0 -commands = pylint black_it tests scripts examples - [testenv:black] skip_install = True deps = black==23.7.0 From e5940c1d2074a52a7c080ccfb412e6986e1d1891 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 17:52:32 +0200 Subject: [PATCH 05/56] fix: remove '# pylint: disable' directives --- black_it/calibrator.py | 8 ++-- black_it/plot/plot_descriptive_statistics.py | 2 +- black_it/plot/plot_results.py | 37 ++++++------------- black_it/samplers/cors.py | 6 +-- black_it/samplers/gaussian_process.py | 4 +- black_it/samplers/halton.py | 2 +- black_it/samplers/particle_swarm.py | 4 +- black_it/samplers/random_forest.py | 2 +- black_it/samplers/xgboost.py | 4 +- black_it/schedulers/rl/rl_scheduler.py | 10 ++--- black_it/search_space.py | 2 +- black_it/utils/base.py | 2 +- black_it/utils/json_pandas_checkpointing.py | 2 +- black_it/utils/sqlite3_checkpointing.py | 10 ++--- black_it/utils/time_series.py | 4 +- examples/models/economics/boltzmann_wealth.py | 4 +- examples/particle_swarm_sampler.ipynb | 2 +- scripts/whitelists/examples_whitelist.py | 1 - scripts/whitelists/package_whitelist.py | 1 - scripts/whitelists/tests_whitelist.py | 1 - tests/test_calibrator.py | 6 +-- tests/test_examples/base.py | 2 +- tests/test_examples/test_docker_sir.py | 4 +- tests/test_examples/test_main.py | 4 +- tests/test_losses/test_gsl.py | 4 +- tests/test_plot/base.py | 2 +- tests/test_plot/test_plot_results.py | 22 ++++------- tests/test_samplers/test_base.py | 4 +- tests/test_samplers/test_gaussian_process.py | 6 +-- tests/test_samplers/test_xgboost.py | 4 +- .../test_pandas_json_checkpointing.py | 4 +- tests/test_utils/test_seedable.py | 10 +---- .../test_utils/test_sqlite3_checkpointing.py | 2 +- tests/utils/base.py | 2 +- tests/utils/docs.py | 6 +-- tests/utils/strategies.py | 2 +- 36 files changed, 74 insertions(+), 118 deletions(-) diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 99613f3a..41bb89b9 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -41,12 +41,12 @@ from black_it.utils.seedable import BaseSeedable -class Calibrator(BaseSeedable): # pylint: disable=too-many-instance-attributes +class Calibrator(BaseSeedable): """The class used to perform a calibration.""" STATE_VERSION = 0 - def __init__( # pylint: disable=too-many-arguments + def __init__( self, loss_function: BaseLoss, real_data: NDArray[np.float64], @@ -214,7 +214,7 @@ def set_samplers(self, samplers: List[BaseSampler]) -> None: """ # overwrite the list of samplers - self.scheduler._samplers = tuple(samplers) # pylint: disable=protected-access + self.scheduler._samplers = tuple(samplers) self.update_samplers_id_table(samplers) def set_scheduler(self, scheduler: BaseScheduler) -> None: @@ -236,7 +236,7 @@ def update_samplers_id_table(self, samplers: Sequence[BaseSampler]) -> None: sampler_id = sampler_id + 1 @classmethod - def restore_from_checkpoint( # pylint: disable=too-many-locals + def restore_from_checkpoint( cls, checkpoint_path: str, model: Callable ) -> "Calibrator": """ diff --git a/black_it/plot/plot_descriptive_statistics.py b/black_it/plot/plot_descriptive_statistics.py index d2ffa87e..e018ae38 100644 --- a/black_it/plot/plot_descriptive_statistics.py +++ b/black_it/plot/plot_descriptive_statistics.py @@ -23,7 +23,7 @@ import statsmodels.api as sm -def ts_stats(ts: List[float]) -> None: # pylint: disable=too-many-locals +def ts_stats(ts: List[float]) -> None: """Show TS graphical descriptive statistics.""" color = "darkslateblue" alpha = 0.8 diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 772c2eeb..0819083c 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -40,11 +40,7 @@ def _get_samplers_id_table(saving_folder: Union[str, os.PathLike]) -> Dict[str, with open(os.path.join(saving_folder, "samplers_pickled.pickle"), "rb") as f: method_list = pickle.load(f) # nosec B301 - samplers_id_table = ( - Calibrator._construct_samplers_id_table( # pylint: disable=protected-access - method_list - ) - ) + samplers_id_table = Calibrator._construct_samplers_id_table(method_list) return samplers_id_table @@ -89,9 +85,7 @@ def plot_convergence(saving_folder: Union[str, os.PathLike]) -> None: ) g = sns.lineplot( - x=np.arange( - max(df["batch_num_samp"]) + 1 # pylint: disable=unsubscriptable-object - ), + x=np.arange(max(df["batch_num_samp"]) + 1), y=losses_cummin, color="black", ls="--", @@ -99,7 +93,7 @@ def plot_convergence(saving_folder: Union[str, os.PathLike]) -> None: label="min loss", ) - ids = df["method_samp"].unique() # pylint: disable=unsubscriptable-object + ids = df["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) handles, labels = g.get_legend_handles_labels() @@ -134,7 +128,7 @@ def plot_losses(saving_folder: Union[str, os.PathLike]) -> None: }, ) - g._legend.set_bbox_to_anchor((0.8, 0.5)) # pylint: disable=protected-access + g._legend.set_bbox_to_anchor((0.8, 0.5)) def plot_sampling(saving_folder: Union[str, os.PathLike]) -> None: @@ -160,12 +154,12 @@ def plot_sampling(saving_folder: Union[str, os.PathLike]) -> None: "fill": False, }, ) - ids = df["method_samp"].unique() # pylint: disable=unsubscriptable-object + ids = df["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) # take legend of the plot in the last row and first column, to be sure it's a scatter plot handles, _ = g.axes[-1][0].get_legend_handles_labels() - g._legend.remove() # pylint: disable=protected-access + g._legend.remove() plt.legend(loc=2, handles=handles, labels=sampler_names, bbox_to_anchor=(0.0, 1.8)) @@ -184,9 +178,7 @@ def plot_losses_method_num( if method_num not in set(df["method_samp"]): raise ValueError(f"Samplers with method_num = {method_num} was never used") - df = df.loc[ - df["method_samp"] == method_num # pylint: disable=unsubscriptable-object - ] + df = df.loc[df["method_samp"] == method_num] num_params = sum("params_samp_" in c_str for c_str in df.columns) @@ -206,7 +198,7 @@ def plot_losses_method_num( }, ) - g._legend.set_bbox_to_anchor((0.8, 0.5)) # pylint: disable=protected-access + g._legend.set_bbox_to_anchor((0.8, 0.5)) def plot_losses_interact(saving_folder: Union[str, os.PathLike]) -> None: @@ -252,14 +244,9 @@ def plot_sampling_batch_nums( df["filter_bns"] = filter_bns - df_ = df.loc[ - df["filter_bns"] # pylint: disable=singleton-comparison,unsubscriptable-object - == True # noqa - ] + df_ = df.loc[df["filter_bns"] == True] # noqa - num_params = sum( - "params_samp_" in c_str for c_str in df.columns # pylint: disable=no-member - ) + num_params = sum("params_samp_" in c_str for c_str in df.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] @@ -274,12 +261,12 @@ def plot_sampling_batch_nums( palette="tab10", ) - ids = df["method_samp"].unique() # pylint: disable=unsubscriptable-object + ids = df["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) # take legend of the plot in the last row and first column, to be sure it's a scatter plot handles, _ = g.axes[-1][0].get_legend_handles_labels() - g._legend.remove() # pylint: disable=protected-access + g._legend.remove() plt.legend(loc=2, handles=handles, labels=sampler_names, bbox_to_anchor=(0.0, 1.8)) diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index f32dea66..929ab9ff 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -121,10 +121,10 @@ def fit(x: float) -> float: return fit -class CORSSampler(BaseSampler): # pylint: disable=too-many-instance-attributes +class CORSSampler(BaseSampler): """Implement the modified CORS sampler.""" - def __init__( # pylint: disable=too-many-arguments + def __init__( self, batch_size: int, max_samples: int, @@ -211,7 +211,7 @@ def sample_batch( "fun": lambda x, localk=k: np.linalg.norm( np.subtract(x, current_points[localk]) ) - - r, # noqa: B023 # pylint: disable=cell-var-from-loop + - r, # noqa: B023 } for k in range(nb_seed_points + j) ] diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index dfac04c6..027fdc4a 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -22,7 +22,7 @@ import numpy as np from numpy.typing import NDArray -from scipy.special import erfc # pylint: disable=no-name-in-module +from scipy.special import erfc from sklearn.gaussian_process import GaussianProcessRegressor, kernels from black_it.samplers.surrogate import MLSurrogateSampler @@ -48,7 +48,7 @@ class GaussianProcessSampler(MLSurrogateSampler): Note: this class is a wrapper of the GaussianProcessRegressor model of the scikit-learn package. """ - def __init__( # pylint: disable=too-many-arguments + def __init__( self, batch_size: int, random_state: Optional[int] = None, diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index 2c455716..14d835f0 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -161,7 +161,7 @@ def __next__(self) -> int: return self._candidate -class _CachedPrimesCalculator: # pylint: disable=too-few-public-methods +class _CachedPrimesCalculator: """Utility class to compute and cache the first n prime numbers.""" def __init__(self) -> None: diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index 4e06a062..c348f313 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -25,9 +25,7 @@ from black_it.utils.base import _assert, digitize_data, positive_float -class ParticleSwarmSampler( - BaseSampler -): # pylint: disable=(too-many-instance-attributes) +class ParticleSwarmSampler(BaseSampler): """ Implementation of a particle swarm sampler. diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index a87f180b..89c9a9f6 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -28,7 +28,7 @@ class RandomForestSampler(MLSurrogateSampler): """This class implements random forest sampling.""" - def __init__( # pylint: disable=too-many-arguments + def __init__( self, batch_size: int, random_state: Optional[int] = None, diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 1bcb378d..00b357c4 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -32,7 +32,7 @@ class XGBoostSampler(MLSurrogateSampler): """This class implements xgboost sampling.""" - def __init__( # pylint: disable=too-many-arguments + def __init__( self, batch_size: int, random_state: Optional[int] = None, @@ -124,7 +124,7 @@ def _clip_losses(y: NDArray[np.float64]) -> NDArray[np.float64]: def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: """Fit a xgboost surrogate model.""" # prepare data - y = self._clip_losses(y) # pylint: disable=W0212 + y = self._clip_losses(y) _ = xgb.DMatrix(data=X, label=y) # train surrogate diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index e7400b64..f5ffc0e1 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -29,7 +29,7 @@ from black_it.schedulers.rl.envs.base import CalibrationEnv -class RLScheduler(BaseScheduler): # pylint: disable=too-many-instance-attributes +class RLScheduler(BaseScheduler): """ This class implement a RL-based scheduler. @@ -54,8 +54,8 @@ def __init__( super().__init__(new_samplers, random_state) - self._in_queue: Queue = self._env._out_queue # pylint: disable=protected-access - self._out_queue: Queue = self._env._in_queue # pylint: disable=protected-access + self._in_queue: Queue = self._env._out_queue + self._out_queue: Queue = self._env._in_queue self._best_param: Optional[float] = None self._best_loss: Optional[float] = None @@ -143,9 +143,7 @@ def update( if self._best_loss is None: self._best_loss = best_new_loss self._best_param = new_params[np.argmin(new_losses)] - self._env._curr_best_loss = ( # pylint: disable=protected-access - best_new_loss - ) + self._env._curr_best_loss = best_new_loss return if best_new_loss < cast(float, self._best_loss): self._best_loss = best_new_loss diff --git a/black_it/search_space.py b/black_it/search_space.py index 13a3c1e3..0e2694e5 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -22,7 +22,7 @@ from numpy.typing import NDArray -class SearchSpace: # pylint: disable=too-few-public-methods +class SearchSpace: """A class that contains information on the search grid of explorable parameters.""" def __init__( diff --git a/black_it/utils/base.py b/black_it/utils/base.py index 34372d1a..7e85c607 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -32,7 +32,7 @@ def _assert( ) -> None: """Check condition; if false, raise exception with the provided error message.""" if not condition: - raise exception_class(error_message) # pylint: disable=broad-exception-raised + raise exception_class(error_message) def check_arg(condition: bool, message: str) -> None: diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index 88129c52..b6b9e76e 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -93,7 +93,7 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - ) -def save_calibrator_state( # pylint: disable=too-many-arguments,too-many-locals +def save_calibrator_state( checkpoint_path: PathLike, parameters_bounds: NDArray[np.float64], parameters_precision: NDArray[np.float64], diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index 6ae2a3b1..9ddac27a 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -165,7 +165,7 @@ class gz_ndarray(NDArray): """ -def load_calibrator_state( # pylint: disable=too-many-locals +def load_calibrator_state( checkpoint_path: PathLike, ) -> Tuple: """ @@ -192,7 +192,7 @@ def load_calibrator_state( # pylint: disable=too-many-locals SQL_LOAD_USER_VERSION ).fetchone()[0] if checkpoint_schema_version != SCHEMA_VERSION: - raise Exception( # pylint: disable=broad-exception-raised + raise Exception( f"The checkpoint you want to load has been generated with another version of the code:\n" f"\tCheckpoint schema version: {checkpoint_schema_version}" f"\tSchema version of the current code: {SCHEMA_VERSION}" @@ -247,13 +247,13 @@ def load_calibrator_state( # pylint: disable=too-many-locals method_samp, ) - except BaseException as err: # pylint: disable=broad-except + except BaseException as err: raise err from err finally: connection.close() -def save_calibrator_state( # pylint: disable=too-many-arguments,too-many-locals +def save_calibrator_state( checkpoint_path: PathLike, parameters_bounds: NDArray[np.float64], parameters_precision: NDArray[np.float64], @@ -348,7 +348,7 @@ def save_calibrator_state( # pylint: disable=too-many-arguments,too-many-locals connection.commit() - except BaseException as err: # pylint: disable=broad-except + except BaseException as err: connection.rollback() raise err from err finally: diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index 437a36a9..a9de3b67 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -54,7 +54,7 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: # first 4 moments and auto-correlations of the time series avg_vec_mom[0] = np.mean(time_series) avg_vec_mom[1] = np.std(time_series) - s, k = skew(time_series), kurtosis(time_series) # pylint: disable=invalid-name + s, k = skew(time_series), kurtosis(time_series) avg_vec_mom[2] = np.sign(s) * np.power(abs(s), 1.0 / 3.0) avg_vec_mom[3] = np.sign(k) * np.power(abs(k), 1.0 / 4.0) @@ -70,7 +70,7 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: avg_vec_mom[9] = np.mean(abs_diff) avg_vec_mom[10] = np.std(abs_diff) - s, k = skew(abs_diff), kurtosis(abs_diff) # pylint: disable=invalid-name + s, k = skew(abs_diff), kurtosis(abs_diff) avg_vec_mom[11] = np.sign(s) * np.power(abs(s), 1.0 / 3.0) avg_vec_mom[12] = np.sign(k) * np.power(abs(k), 1.0 / 4.0) diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index 22c33133..f2ec7971 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -41,7 +41,7 @@ class BoltzmannWealthModel(Model): highly skewed distribution of wealth. """ - def __init__( # pylint: disable=super-init-not-called + def __init__( self, num_agents: int = 100, width: int = 10, @@ -76,7 +76,7 @@ def step(self) -> None: # collect data self.datacollector.collect(self) - def run_model(self, n: int) -> None: # pylint: disable=arguments-differ + def run_model(self, n: int) -> None: """Run the model.""" for _ in range(n): self.step() diff --git a/examples/particle_swarm_sampler.ipynb b/examples/particle_swarm_sampler.ipynb index 524b65c0..571334ad 100644 --- a/examples/particle_swarm_sampler.ipynb +++ b/examples/particle_swarm_sampler.ipynb @@ -111,7 +111,7 @@ "output_type": "stream", "text": [ "/Users/aldoglielmo/DRTA_code/black-it/black_it/utils/time_series.py:54: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", - " s, k = skew(time_series), kurtosis(time_series) # pylint: disable=invalid-name\n", + " s, k = skew(time_series), kurtosis(time_series)\n", "/Users/aldoglielmo/miniconda3/envs/py39_2/lib/python3.9/site-packages/statsmodels/tsa/stattools.py:681: RuntimeWarning: invalid value encountered in divide\n", " acf = avf[: nlags + 1] / avf[0]\n" ] diff --git a/scripts/whitelists/examples_whitelist.py b/scripts/whitelists/examples_whitelist.py index 1261034a..0c5a68f8 100644 --- a/scripts/whitelists/examples_whitelist.py +++ b/scripts/whitelists/examples_whitelist.py @@ -1,6 +1,5 @@ # flake8: noqa # type: ignore -# pylint: skip-file num_cores # unused variable (examples/docker-sir.py:21) losses # unused variable (examples/docker-sir.py:56) num_cores # unused variable (examples/main.py:22) diff --git a/scripts/whitelists/package_whitelist.py b/scripts/whitelists/package_whitelist.py index 1a82b9e6..ebd99ee3 100644 --- a/scripts/whitelists/package_whitelist.py +++ b/scripts/whitelists/package_whitelist.py @@ -1,6 +1,5 @@ # flake8: noqa # type: ignore -# pylint: skip-file _.set_samplers # unused method (black_it/calibrator.py:180) _.set_scheduler # unused method (black_it/calibrator.py:220) _.restore_from_checkpoint # unused method (black_it/calibrator.py:203) diff --git a/scripts/whitelists/tests_whitelist.py b/scripts/whitelists/tests_whitelist.py index 41077cea..a5f2cd92 100644 --- a/scripts/whitelists/tests_whitelist.py +++ b/scripts/whitelists/tests_whitelist.py @@ -1,6 +1,5 @@ # flake8: noqa # type: ignore -# pylint: skip-file TEST_DIR # unused variable (tests/conftest.py:17) _.__abstractmethods__ # unused attribute (tests/test_samplers/test_base.py:14) code_blocks # unused variable (tests/utils/docs.py:77) diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index 59120ceb..65585398 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -40,7 +40,7 @@ from examples.models.simple_models import NormalMV -class TestCalibrate: # pylint: disable=too-many-instance-attributes,attribute-defined-outside-init +class TestCalibrate: """Test the Calibrator.calibrate method.""" expected_params = np.array( @@ -171,7 +171,7 @@ def test_calibrator_calibrate(self, n_jobs: int) -> None: params, losses = cal.calibrate(14) - # TODO: this is a temporary workaround to make tests to run also on Windows. # pylint: disable=fixme + # TODO: this is a temporary workaround to make tests to run also on Windows. # See: https://github.com/bancaditalia/black-it/issues/49 print(params.tolist()) print(losses.tolist()) @@ -258,7 +258,7 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N if key == "scheduler": t1 = type(vars_cal["scheduler"]) t2 = type(cal_restored.scheduler) - assert t1 == t2 # noqa, pylint: disable=unidiomatic-typecheck + assert t1 == t2 # noqa elif key == "loss_function": assert ( type(vars_cal["loss_function"]).__name__ diff --git a/tests/test_examples/base.py b/tests/test_examples/base.py index 02a94e5f..900a0f18 100644 --- a/tests/test_examples/base.py +++ b/tests/test_examples/base.py @@ -24,7 +24,7 @@ from tests.utils.base import PopenResult, run_process -class BaseMainExampleTestClass: # pylint: disable=too-few-public-methods +class BaseMainExampleTestClass: """Base test class for the main example.""" script_path: Path diff --git a/tests/test_examples/test_docker_sir.py b/tests/test_examples/test_docker_sir.py index c9c229ff..b78f69be 100644 --- a/tests/test_examples/test_docker_sir.py +++ b/tests/test_examples/test_docker_sir.py @@ -27,9 +27,7 @@ @pytest.mark.e2e @requires_docker @skip_on_windows -class TestDockerSirMainExample( # pylint: disable=too-few-public-methods - BaseMainExampleTestClass -): +class TestDockerSirMainExample(BaseMainExampleTestClass): """Test that example docker-sir can be run successfully.""" script_path = EXAMPLE_DOCKER_SIR_SCRIPT_PATH diff --git a/tests/test_examples/test_main.py b/tests/test_examples/test_main.py index c3a6dc41..b7e9cb28 100644 --- a/tests/test_examples/test_main.py +++ b/tests/test_examples/test_main.py @@ -26,9 +26,7 @@ BEST_PARAMETERS_STR = "Best parameters found: [0.19 0.19 0.74]" -class TestMainExample( # pylint: disable=too-few-public-methods - BaseMainExampleTestClass -): +class TestMainExample(BaseMainExampleTestClass): """Test that example main can be run successfully.""" script_path = EXAMPLE_MAIN_SCRIPT_PATH diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index 5d539db8..0be32b40 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -47,7 +47,7 @@ def fail_check(left: float, right: float) -> bool: return True -class TestDiscretize: # pylint: disable=no-self-use +class TestDiscretize: """Test the 'discretize' function.""" @given(discretize_args()) @@ -82,7 +82,7 @@ def test_ordering_is_preserved(self, args: Tuple) -> None: assert is_sorted(decreasing_actual, reverse=True) -class TestGetWords: # pylint: disable=no-self-use,too-few-public-methods +class TestGetWords: """Test the 'get_words' function.""" @given(get_words_args()) diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index b4d1e46a..4f0de032 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -24,7 +24,7 @@ from matplotlib.testing.compare import compare_images -class BasePlotTest: # pylint: disable=too-few-public-methods +class BasePlotTest: """Base test class for plotting functions.""" plotting_function: Callable diff --git a/tests/test_plot/test_plot_results.py b/tests/test_plot/test_plot_results.py index 3b56b26f..80612073 100644 --- a/tests/test_plot/test_plot_results.py +++ b/tests/test_plot/test_plot_results.py @@ -35,9 +35,7 @@ @skip_on_windows() -class TestPlotConvergence( - BasePlotResultsTest -): # pylint: disable=too-few-public-methods +class TestPlotConvergence(BasePlotResultsTest): """Test the 'plot_convergence' function.""" plotting_function = plot_convergence @@ -47,7 +45,7 @@ class TestPlotConvergence( @skip_on_windows() -class TestPlotLosses(BasePlotResultsTest): # pylint: disable=too-few-public-methods +class TestPlotLosses(BasePlotResultsTest): """Test the 'plot_losses' function.""" plotting_function = plot_losses @@ -56,7 +54,7 @@ class TestPlotLosses(BasePlotResultsTest): # pylint: disable=too-few-public-met @skip_on_windows() -class TestPlotSampling(BasePlotResultsTest): # pylint: disable=too-few-public-methods +class TestPlotSampling(BasePlotResultsTest): """Test the 'plot_sampling' function.""" plotting_function = plot_sampling @@ -65,9 +63,7 @@ class TestPlotSampling(BasePlotResultsTest): # pylint: disable=too-few-public-m @skip_on_windows() -class TestPlotLossesMethodNum( - BasePlotResultsTest -): # pylint: disable=too-few-public-methods +class TestPlotLossesMethodNum(BasePlotResultsTest): """Test the 'plot_losses_method_num' function.""" plotting_function = plot_losses_method_num @@ -80,9 +76,7 @@ def test_run(self) -> None: """Run the test.""" @pytest.mark.parametrize("method_num", list(range(5))) - def test_run_by_method_num( - self, method_num: int - ) -> None: # pylint: disable=arguments-differ + def test_run_by_method_num(self, method_num: int) -> None: """Run the test for all method numbers.""" self.expected_image = ( PLOT_DIR / f"plot_losses_method_num_{method_num}-expected.png" @@ -98,7 +92,7 @@ def teardown(self) -> None: @skip_on_windows() -class TestPlotBatchNums(BasePlotResultsTest): # pylint: disable=too-few-public-methods +class TestPlotBatchNums(BasePlotResultsTest): """Test the 'plot_sampling_batch_nums' function.""" plotting_function = plot_sampling_batch_nums @@ -111,9 +105,7 @@ def test_run(self) -> None: """Run the test.""" @pytest.mark.parametrize("batch_num", list(range(0, 13, 3))) - def test_run_by_batch_num( - self, batch_num: int - ) -> None: # pylint: disable=arguments-differ + def test_run_by_batch_num(self, batch_num: int) -> None: """Run the test for all method numbers.""" self.expected_image = ( PLOT_DIR / f"plot_sampling_batch_nums_{batch_num:03d}-expected.png" diff --git a/tests/test_samplers/test_base.py b/tests/test_samplers/test_base.py index 6bac0bbe..d8df1104 100644 --- a/tests/test_samplers/test_base.py +++ b/tests/test_samplers/test_base.py @@ -30,13 +30,13 @@ def test_find_and_get_duplicates() -> None: BaseSampler.__abstractmethods__ = frozenset() - sampler = BaseSampler(batch_size=1) # type: ignore # pylint: disable=abstract-class-instantiated + sampler = BaseSampler(batch_size=1) # type: ignore duplicates = sampler.find_and_get_duplicates(new_points, existing_points) assert duplicates == [0, 1, 2, 4, 5] -class TestSetRandomState: # pylint: disable=attribute-defined-outside-init,too-many-instance-attributes +class TestSetRandomState: """Test 'BaseSampler.random_state' setter.""" class MyCustomSampler(BaseSampler): diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 5965bec6..625d0429 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -24,7 +24,7 @@ from black_it.search_space import SearchSpace -class TestGaussianProcess2D: # pylint: disable=attribute-defined-outside-init +class TestGaussianProcess2D: """Test GaussianProcess sampling.""" def setup(self) -> None: @@ -111,9 +111,7 @@ def test_gaussian_process_sample_warning_too_large_dataset() -> None: ( xys, losses, - ) = TestGaussianProcess2D._construct_fake_grid( # pylint: disable=protected-access - n=23 - ) + ) = TestGaussianProcess2D._construct_fake_grid(n=23) with pytest.warns( RuntimeWarning, match="Standard GP evaluations can be expensive " diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index ae34ee00..54a96644 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -25,7 +25,7 @@ from black_it.search_space import SearchSpace from examples.models.economics.brock_hommes import BH4 -# TODO: this is a temporary workaround to make tests to run also on Windows and Mac. # pylint: disable=fixme +# TODO: this is a temporary workaround to make tests to run also on Windows and Mac. # See: https://github.com/bancaditalia/black-it/issues/49 if sys.platform == "win32": expected_params = np.array([[0.24, 0.26], [0.19, 0.11], [0.13, 0.22], [0.11, 0.05]]) @@ -126,7 +126,7 @@ def test_clip_losses() -> None: # verify that _clip_losses works as expected y = np.array([0.0, -1e40, 1e40]) - y2 = xgboost._clip_losses(y) # pylint: disable=W0212 + y2 = xgboost._clip_losses(y) assert ( y2 == np.array([0.0, MIN_FLOAT32 + EPS_FLOAT32, MAX_FLOAT32 - EPS_FLOAT32]) diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index dd57c70f..cd4b4ff5 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -25,9 +25,7 @@ ) -def test_save_and_load_calibrator_state() -> ( # pylint: disable=too-many-locals,too-many-statements - None -): +def test_save_and_load_calibrator_state() -> None: """Test the 'save_calibrator_state' and 'load_calibrator_state' functions.""" parameters_bounds = np.array([[0, 1], [0, 1]]).T parameters_precision = np.array([0.01, 0.01]) diff --git a/tests/test_utils/test_seedable.py b/tests/test_utils/test_seedable.py index c877f822..09d91470 100644 --- a/tests/test_utils/test_seedable.py +++ b/tests/test_utils/test_seedable.py @@ -78,16 +78,10 @@ def test_get_random_seed(random_seed: int) -> None: nb_iterations = 1000 seedable.random_state = random_seed - expected_values_1 = [ - seedable._get_random_seed() # pylint: disable=protected-access - for _ in range(nb_iterations) - ] + expected_values_1 = [seedable._get_random_seed() for _ in range(nb_iterations)] seedable.random_state = random_seed - expected_values_2 = [ - seedable._get_random_seed() # pylint: disable=protected-access - for _ in range(nb_iterations) - ] + expected_values_2 = [seedable._get_random_seed() for _ in range(nb_iterations)] assert expected_values_1 == expected_values_2 assert all(0 <= value <= 2**32 - 1 for value in expected_values_1) diff --git a/tests/test_utils/test_sqlite3_checkpointing.py b/tests/test_utils/test_sqlite3_checkpointing.py index da8ba3cc..e36eda25 100644 --- a/tests/test_utils/test_sqlite3_checkpointing.py +++ b/tests/test_utils/test_sqlite3_checkpointing.py @@ -29,7 +29,7 @@ ) -def test_sqlite3_checkpointing() -> None: # pylint: disable=too-many-locals +def test_sqlite3_checkpointing() -> None: """Test SQLite3 checkpointing.""" parameters_bounds = np.array([1.0, 2.0, 3.0]) parameters_precision = np.array([0.1, 0.1, 0.1]) diff --git a/tests/utils/base.py b/tests/utils/base.py index aa343367..4dcf954f 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -53,7 +53,7 @@ def run_process( Raises: RuntimeError: if the process cannot be stopped. """ - process = subprocess.Popen( # pylint: disable=consider-using-with # nosec B603 + process = subprocess.Popen( # nosec B603 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/tests/utils/docs.py b/tests/utils/docs.py index cfcf2592..33c958da 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -54,7 +54,7 @@ def code_block_extractor(child_dict: Dict) -> str: return child_dict["children"][0]["content"] -class BaseTestMarkdownDocs: # pylint: disable=too-few-public-methods +class BaseTestMarkdownDocs: """Base test class for testing Markdown documents.""" DOC_PATH: Path @@ -104,7 +104,5 @@ def test_python_blocks(self, *mocks: Any) -> None: globals_, locals_ = self.globals, self.locals for python_code_block in python_blocks: - exec( # nosec # pylint: disable=exec-used - python_code_block, globals_, locals_ - ) + exec(python_code_block, globals_, locals_) # nosec self._assert(locals_, *mocks) diff --git a/tests/utils/strategies.py b/tests/utils/strategies.py index 4d792c55..0d2cb480 100644 --- a/tests/utils/strategies.py +++ b/tests/utils/strategies.py @@ -15,7 +15,7 @@ # along with this program. If not, see . """Hypothesis strategies for testing.""" -from typing import Callable, Tuple # pylint: disable=unused-import +from typing import Callable, Tuple import hypothesis.extra.numpy import hypothesis.strategies as st From 70e1c6ba1ca448ba589e293d9d918da647f327ee Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 17:54:16 +0200 Subject: [PATCH 06/56] dev: remove pylint from pyproject.toml and poetry.lock --- poetry.lock | 215 +------------------------------------------------ pyproject.toml | 1 - 2 files changed, 1 insertion(+), 215 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1e5fdb93..4557d6fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -103,22 +103,6 @@ files = [ [package.dependencies] python-dateutil = ">=2.7.0" -[[package]] -name = "astroid" -version = "2.15.6" -description = "An abstract syntax tree for Python with inference support." -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "astroid-2.15.6-py3-none-any.whl", hash = "sha256:389656ca57b6108f939cf5d2f9a2a825a3be50ba9d589670f393236e0a03b91c"}, - {file = "astroid-2.15.6.tar.gz", hash = "sha256:903f024859b7c7687d7a7f3a3f73b17301f8e42dfd9cc9df9d4418172d3e2dbd"}, -] - -[package.dependencies] -lazy-object-proxy = ">=1.4.0" -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} -wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} - [[package]] name = "asttokens" version = "2.2.1" @@ -989,20 +973,6 @@ files = [ {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] -[[package]] -name = "dill" -version = "0.3.7" -description = "serialize all of Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e"}, - {file = "dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] - [[package]] name = "distlib" version = "0.3.7" @@ -1698,23 +1668,6 @@ files = [ [package.dependencies] arrow = ">=0.15.0" -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - [[package]] name = "jaraco-classes" version = "3.3.0" @@ -2284,51 +2237,6 @@ files = [ {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"}, ] -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -optional = false -python-versions = ">=3.7" -files = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] - [[package]] name = "lxml" version = "4.9.3" @@ -3768,32 +3676,6 @@ files = [ [package.extras] plugins = ["importlib-metadata"] -[[package]] -name = "pylint" -version = "2.17.5" -description = "python code static checker" -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "pylint-2.17.5-py3-none-any.whl", hash = "sha256:73995fb8216d3bed149c8d51bba25b2c52a8251a2c8ac846ec668ce38fab5413"}, - {file = "pylint-2.17.5.tar.gz", hash = "sha256:f7b601cbc06fef7e62a754e2b41294c2aa31f1cb659624b9a85bcba29eaf8252"}, -] - -[package.dependencies] -astroid = ">=2.15.6,<=2.17.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = {version = ">=0.2", markers = "python_version < \"3.11\""} -isort = ">=4.2.5,<6" -mccabe = ">=0.6,<0.8" -platformdirs = ">=2.2.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} - -[package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] - [[package]] name = "pymdown-extensions" version = "10.1" @@ -5275,17 +5157,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "tomlkit" -version = "0.12.1" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomlkit-0.12.1-py3-none-any.whl", hash = "sha256:712cbd236609acc6a3e2e97253dfc52d4c2082982a88f61b640ecf0817eab899"}, - {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, -] - [[package]] name = "tornado" version = "6.3.3" @@ -5681,90 +5552,6 @@ files = [ {file = "widgetsnbextension-4.0.8.tar.gz", hash = "sha256:9ec291ba87c2dfad42c3d5b6f68713fa18be1acd7476569516b2431682315c17"}, ] -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] - [[package]] name = "xgboost" version = "1.7.6" @@ -5821,4 +5608,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "583ec2f6d97216eb3bd813386569d324a57f0b2008875da41658474396485915" +content-hash = "1002e7eab554452d88c573169ff3bf380a98bd81a1006463924d7dfb76b84b32" diff --git a/pyproject.toml b/pyproject.toml index b30b77b2..a4fcca39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,6 @@ mypy = "^1.5.1" nbmake = "^1.4.3" ndlib = "^5.1.1" pydocstyle = "^6.3.0" -pylint = "^2.17.5" pymdown-extensions = "^10.1" pytest = "^7.4.0" pytest-cov = "^4.1.0" From a73c4d5f7e61b9b863a99a241407bbdb64b26e37 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 18:21:20 +0200 Subject: [PATCH 07/56] dev: remove flake8 usage throughout the repository --- .github/workflows/lint.yml | 1 - Makefile | 13 ++-------- scripts/list-poetry-dependencies.sh | 2 +- scripts/whitelists/examples_whitelist.py | 1 - scripts/whitelists/package_whitelist.py | 1 - scripts/whitelists/tests_whitelist.py | 1 - setup.cfg | 30 ------------------------ tox.ini | 13 +--------- 8 files changed, 4 insertions(+), 58 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 458b2417..79f13af4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,7 +28,6 @@ jobs: - name: Code style check run: | tox -e black-check - tox -e flake8 tox -e vulture tox -e darglint - name: Static type check diff --git a/Makefile b/Makefile index 0d154f9e..86d1b5c8 100644 --- a/Makefile +++ b/Makefile @@ -63,19 +63,10 @@ clean-test: ## remove test and coverage artifacts rm -fr coverage.xml PHONY.: lint-all -lint-all: black flake8 static bandit safety vulture darglint ## run all linters +lint-all: black static bandit safety vulture darglint ## run all linters PHONY.: lint-all-files -lint-all-files: black-files flake8-files static-files bandit-files vulture-files darglint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") - -PHONY.: flake8 -flake8: ## check style with flake8 - flake8 black_it tests scripts examples - -PHONY.: flake8-files -flake8-files: ## check style with flake8 for specific files (specified with files="file1 file2 somedir ...") - $(call check_defined, files) - flake8 $(files) +lint-all-files: black-files static-files bandit-files vulture-files darglint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") PHONY.: static static: ## static type checking with mypy diff --git a/scripts/list-poetry-dependencies.sh b/scripts/list-poetry-dependencies.sh index 0cbe093f..ae3eee56 100755 --- a/scripts/list-poetry-dependencies.sh +++ b/scripts/list-poetry-dependencies.sh @@ -17,7 +17,7 @@ With args, considers each argument as a search pattern to select requirements. EXAMPLES: ./$(basename "$0") -h # prints the usage ./$(basename "$0") --all # exports all the packages from poetry.lock - ./$(basename "$0") flake8 numpy # exports the packages needed by flake8 and numpy + ./$(basename "$0") black numpy # exports the packages needed by black and numpy " if [ $# == 0 ] || ([ $# == 1 ] && ([ "$1" == "-h" ] || [ "$1" == "--help" ])); diff --git a/scripts/whitelists/examples_whitelist.py b/scripts/whitelists/examples_whitelist.py index 0c5a68f8..5422b7de 100644 --- a/scripts/whitelists/examples_whitelist.py +++ b/scripts/whitelists/examples_whitelist.py @@ -1,4 +1,3 @@ -# flake8: noqa # type: ignore num_cores # unused variable (examples/docker-sir.py:21) losses # unused variable (examples/docker-sir.py:56) diff --git a/scripts/whitelists/package_whitelist.py b/scripts/whitelists/package_whitelist.py index ebd99ee3..2e08a1d9 100644 --- a/scripts/whitelists/package_whitelist.py +++ b/scripts/whitelists/package_whitelist.py @@ -1,4 +1,3 @@ -# flake8: noqa # type: ignore _.set_samplers # unused method (black_it/calibrator.py:180) _.set_scheduler # unused method (black_it/calibrator.py:220) diff --git a/scripts/whitelists/tests_whitelist.py b/scripts/whitelists/tests_whitelist.py index a5f2cd92..f09dc230 100644 --- a/scripts/whitelists/tests_whitelist.py +++ b/scripts/whitelists/tests_whitelist.py @@ -1,4 +1,3 @@ -# flake8: noqa # type: ignore TEST_DIR # unused variable (tests/conftest.py:17) _.__abstractmethods__ # unused attribute (tests/test_samplers/test_base.py:14) diff --git a/setup.cfg b/setup.cfg index 186c8678..23454144 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,36 +7,6 @@ test = pytest [metadata] license_file = LICENSE -[flake8] -ignore = W503,E203 -exclude = - .tox, - .git, - __pycache__, - build, - dist, - tests/fixtures/plots/*, - *.md, - *.pyc, - *.egg-info, - .cache, - .eggs, - scripts/__init__.py - scripts/whitelist.py - scripts/whitelists/__init__.py - examples/models -max-complexity = 10 -max-line-length = 120 - -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -ensure_newline_before_comments=True -known_third_party=xgboost - [black] exclude = "scripts/whitelists/" diff --git a/tox.ini b/tox.ini index 0c2392ad..55085694 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = True -envlist = bandit, safety, check-copyright, black-check, vulture, flake8, mypy, py3{8,9,10}, docs +envlist = bandit, safety, check-copyright, black-check, vulture, mypy, py3{8,9,10}, docs [tox:.package] # note tox will use the same python version as under what tox is installed to package @@ -41,17 +41,6 @@ deps = nbmake==1.4.3 commands = pytest examples/tests_on_toy_model.ipynb --nbmake --nbmake-timeout=300 -[testenv:flake8] -skip_install = True -deps = - flake8==5.0.4 - flake8-bugbear==23.3.12 - flake8-docstrings==1.7.0 - flake8-eradicate==1.5.0 - pydocstyle==6.3.0 -commands = - flake8 black_it tests scripts examples - [testenv:mypy] deps = mypy==1.5.1 From 4c1441eebdde60ea0d445f457b541bfa6abe748b Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Thu, 24 Aug 2023 18:23:02 +0200 Subject: [PATCH 08/56] dev: remove flake8-* packages from pyproject.toml and poetry.lock --- poetry.lock | 117 ++----------------------------------------------- pyproject.toml | 4 -- 2 files changed, 4 insertions(+), 117 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4557d6fc..5718f0ca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1032,17 +1032,6 @@ networkx = "*" numpy = "*" tqdm = "*" -[[package]] -name = "eradicate" -version = "2.3.0" -description = "Removes commented-out code." -optional = false -python-versions = "*" -files = [ - {file = "eradicate-2.3.0-py3-none-any.whl", hash = "sha256:2b29b3dd27171f209e4ddd8204b70c02f0682ae95eecb353f10e8d72b149c63e"}, - {file = "eradicate-2.3.0.tar.gz", hash = "sha256:06df115be3b87d0fc1c483db22a2ebb12bcf40585722810d809cc770f5031c37"}, -] - [[package]] name = "exceptiongroup" version = "1.1.3" @@ -1100,71 +1089,6 @@ files = [ docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] -[[package]] -name = "flake8" -version = "5.0.4" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, - {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.9.0,<2.10.0" -pyflakes = ">=2.5.0,<2.6.0" - -[[package]] -name = "flake8-bugbear" -version = "23.3.12" -description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." -optional = false -python-versions = ">=3.7" -files = [ - {file = "flake8-bugbear-23.3.12.tar.gz", hash = "sha256:e3e7f74c8a49ad3794a7183353026dabd68c74030d5f46571f84c1fb0eb79363"}, - {file = "flake8_bugbear-23.3.12-py3-none-any.whl", hash = "sha256:beb5c7efcd7ccc2039ef66a77bb8db925e7be3531ff1cb4d0b7030d0e2113d72"}, -] - -[package.dependencies] -attrs = ">=19.2.0" -flake8 = ">=3.0.0" - -[package.extras] -dev = ["coverage", "hypothesis", "hypothesmith (>=0.2)", "pre-commit", "pytest", "tox"] - -[[package]] -name = "flake8-docstrings" -version = "1.7.0" -description = "Extension for flake8 which uses pydocstyle to check docstrings" -optional = false -python-versions = ">=3.7" -files = [ - {file = "flake8_docstrings-1.7.0-py2.py3-none-any.whl", hash = "sha256:51f2344026da083fc084166a9353f5082b01f72901df422f74b4d953ae88ac75"}, - {file = "flake8_docstrings-1.7.0.tar.gz", hash = "sha256:4c8cc748dc16e6869728699e5d0d685da9a10b0ea718e090b1ba088e67a941af"}, -] - -[package.dependencies] -flake8 = ">=3" -pydocstyle = ">=2.1" - -[[package]] -name = "flake8-eradicate" -version = "1.5.0" -description = "Flake8 plugin to find commented out code" -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "flake8_eradicate-1.5.0-py3-none-any.whl", hash = "sha256:18acc922ad7de623f5247c7d5595da068525ec5437dd53b22ec2259b96ce9d22"}, - {file = "flake8_eradicate-1.5.0.tar.gz", hash = "sha256:aee636cb9ecb5594a7cd92d67ad73eb69909e5cc7bd81710cf9d00970f3983a6"}, -] - -[package.dependencies] -attrs = "*" -eradicate = ">=2.0,<3.0" -flake8 = ">5" - [[package]] name = "fonttools" version = "4.42.1" @@ -2584,17 +2508,6 @@ files = [ [package.dependencies] traitlets = "*" -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -2644,13 +2557,13 @@ docs = ["ipykernel", "ipython", "nbsphinx", "pydata-sphinx-theme", "seaborn", "s [[package]] name = "mistletoe" -version = "1.2.0" +version = "1.2.1" description = "A fast, extensible Markdown parser in pure Python." optional = false python-versions = "~=3.5" files = [ - {file = "mistletoe-1.2.0-py3-none-any.whl", hash = "sha256:637a87afacb5eb033b4e47faa059f6c45386e911c7f3defff4e854e061759294"}, - {file = "mistletoe-1.2.0.tar.gz", hash = "sha256:3ad979a94207792456573604f57df36a22fb1a4c81d7f9d1b9f3eca88608a0ab"}, + {file = "mistletoe-1.2.1-py3-none-any.whl", hash = "sha256:620563ac06380ce0629b4a2afa2f2ab797ffac3efdcccaf2362a7266600e6dcc"}, + {file = "mistletoe-1.2.1.tar.gz", hash = "sha256:7d0c1ab3747047d169f9fc4b925d1cba3f5c13eaf0b90c365b72e47e59d00a02"}, ] [[package]] @@ -3612,17 +3525,6 @@ files = [ {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, ] -[[package]] -name = "pycodestyle" -version = "2.9.1" -description = "Python style guide checker" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"}, - {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, -] - [[package]] name = "pycparser" version = "2.21" @@ -3651,17 +3553,6 @@ snowballstemmer = ">=2.2.0" [package.extras] toml = ["tomli (>=1.2.3)"] -[[package]] -name = "pyflakes" -version = "2.5.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, - {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, -] - [[package]] name = "pygments" version = "2.16.1" @@ -5608,4 +5499,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "1002e7eab554452d88c573169ff3bf380a98bd81a1006463924d7dfb76b84b32" +content-hash = "deeeb2a3a4d34749209f71965e2e4a8e52200f92510d9f848ab4eab4d334aade" diff --git a/pyproject.toml b/pyproject.toml index a4fcca39..e89a9e0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,10 +58,6 @@ bandit = "^1.7.5" black = "^23.7.0" codecov = "^2.1.13" darglint = "^1.8.1" -flake8 = "^5.0.4" -flake8-bugbear = "^23.3.12" -flake8-docstrings = "^1.7.0" -flake8-eradicate = "^1.5.0" hypothesis = "^6.82.6" ipython = "^8.3.0" jupyter = "^1.0.0" From a8a96f982fa25820925316176cd5339611f12987 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 09:49:31 +0200 Subject: [PATCH 09/56] dev: add ruff to pyproject.toml as dev dependency --- poetry.lock | 28 +++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 5718f0ca..05c905eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4492,6 +4492,32 @@ files = [ {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, ] +[[package]] +name = "ruff" +version = "0.0.290" +description = "An extremely fast Python linter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.0.290-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:0e2b09ac4213b11a3520221083866a5816616f3ae9da123037b8ab275066fbac"}, + {file = "ruff-0.0.290-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:4ca6285aa77b3d966be32c9a3cd531655b3d4a0171e1f9bf26d66d0372186767"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35e3550d1d9f2157b0fcc77670f7bb59154f223bff281766e61bdd1dd854e0c5"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d748c8bd97874f5751aed73e8dde379ce32d16338123d07c18b25c9a2796574a"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:982af5ec67cecd099e2ef5e238650407fb40d56304910102d054c109f390bf3c"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bbd37352cea4ee007c48a44c9bc45a21f7ba70a57edfe46842e346651e2b995a"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d9be6351b7889462912e0b8185a260c0219c35dfd920fb490c7f256f1d8313e"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75cdc7fe32dcf33b7cec306707552dda54632ac29402775b9e212a3c16aad5e6"}, + {file = "ruff-0.0.290-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb07f37f7aecdbbc91d759c0c09870ce0fb3eed4025eebedf9c4b98c69abd527"}, + {file = "ruff-0.0.290-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2ab41bc0ba359d3f715fc7b705bdeef19c0461351306b70a4e247f836b9350ed"}, + {file = "ruff-0.0.290-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:150bf8050214cea5b990945b66433bf9a5e0cef395c9bc0f50569e7de7540c86"}, + {file = "ruff-0.0.290-py3-none-musllinux_1_2_i686.whl", hash = "sha256:75386ebc15fe5467248c039f5bf6a0cfe7bfc619ffbb8cd62406cd8811815fca"}, + {file = "ruff-0.0.290-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ac93eadf07bc4ab4c48d8bb4e427bf0f58f3a9c578862eb85d99d704669f5da0"}, + {file = "ruff-0.0.290-py3-none-win32.whl", hash = "sha256:461fbd1fb9ca806d4e3d5c745a30e185f7cf3ca77293cdc17abb2f2a990ad3f7"}, + {file = "ruff-0.0.290-py3-none-win_amd64.whl", hash = "sha256:f1f49f5ec967fd5778813780b12a5650ab0ebcb9ddcca28d642c689b36920796"}, + {file = "ruff-0.0.290-py3-none-win_arm64.whl", hash = "sha256:ae5a92dfbdf1f0c689433c223f8dac0782c2b2584bd502dfdbc76475669f1ba1"}, + {file = "ruff-0.0.290.tar.gz", hash = "sha256:949fecbc5467bb11b8db810a7fa53c7e02633856ee6bd1302b2f43adcd71b88d"}, +] + [[package]] name = "safety" version = "2.4.0b1" @@ -5499,4 +5525,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "deeeb2a3a4d34749209f71965e2e4a8e52200f92510d9f848ab4eab4d334aade" +content-hash = "2b2cec2b629d685f163b1fcc747919eb39a9339241758640652d5faaa7511d82" diff --git a/pyproject.toml b/pyproject.toml index e89a9e0d..737ba15e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ pymdown-extensions = "^10.1" pytest = "^7.4.0" pytest-cov = "^4.1.0" pytest-randomly = "^3.15.0" +ruff = "^0.0.290" safety = "^2.3.5" tox = "^4.4.7" twine = "^4.0.2" From f5ba6176e9b6b7e36ec4e49b4b8099667c95ec28 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 10:21:47 +0200 Subject: [PATCH 10/56] dev: fix 'make lint' with 'make lint-all' --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4693d967..1f7b297e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ We have various commands which are helpful during development. - For linting and static analysis use: ``` -make lint +make lint-all make static make safety make bandit From 92be0dde0b731896726298ceaa54ed7126314079 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 09:51:01 +0200 Subject: [PATCH 11/56] dev: add .ruff.toml configuration file We preferred a configuration based on `.ruff.toml` in order to not bloat pyproject.toml. We started from the configuration example, which is the default configuration, reported on the documentation website: https://github.com/astral-sh/ruff/blob/v0.0.285/docs/configuration.md#using-pyprojecttoml: ``` [tool.ruff] \# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default. \# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or \# McCabe complexity (`C901`) by default. select = ["E", "F"] ignore = [] \# Allow autofix for all enabled rules (when `--fix`) is provided. fixable = ["ALL"] unfixable = [] \# Exclude a variety of commonly ignored directories. exclude = [ ".bzr", ".direnv", ".eggs", ".git", ".git-rewrite", ".hg", ".mypy_cache", ".nox", ".pants.d", ".pytype", ".ruff_cache", ".svn", ".tox", ".venv", "__pypackages__", "_build", "buck-out", "build", "dist", "node_modules", "venv", ] per-file-ignores = {} \ Same as Black. line-length = 88 \# Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" \# Assume Python 3.8 target-version = "py38" ``` Then, we applied the following changes: - removed comments - disable "ALL" checks: we add one check at a time in the following commits. - ignore "E203" (Whitespace before ':') - ignore files in 'scripts/whitelists/' - increase line-length from 88 to 120 - add mccabe check and pydocstyle convention "google" - add preview = true because of the warning: Selection of nursery rule `E203` without the `--preview` flag is deprecated. - comment out 'fixable'/'unfixable' setting, we use the default settings. --- .ruff.toml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .ruff.toml diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000..8a6fda5a --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,51 @@ +select = [] +ignore = ["E203"] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +#fixable = ["ALL"] +#unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "scripts/whitelists/*" +] + +# Same as Black. +line-length = 120 + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Whether to enable preview mode. When preview mode is enabled, Ruff will use unstable rules and fixes. +preview = true + +# Assume Python 3.8 +target-version = "py38" + +[mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 + +[pydocstyle] +convention = "google" From c93729eb95d7bd1acfe04a739a7bbdc83da49a6d Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 10:21:14 +0200 Subject: [PATCH 12/56] dev: add ruff in tox, Makefile, and in contributing guidelines --- .ruff.toml | 2 ++ .spelling | 2 ++ CONTRIBUTING.md | 5 +++++ Makefile | 23 +++++++++++++++++++++-- tox.ini | 10 ++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 8a6fda5a..612c4790 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -5,6 +5,8 @@ ignore = ["E203"] #fixable = ["ALL"] #unfixable = [] +include = ["black_it/**/*.py", "tests/**/*.py", "scripts/**/*.py", "examples/**/*.py"] + # Exclude a variety of commonly ignored directories. exclude = [ ".bzr", diff --git a/.spelling b/.spelling index d24e01f4..ba0e91aa 100644 --- a/.spelling +++ b/.spelling @@ -72,3 +72,5 @@ AldoGl marcofavoritobi codecoverage Codecov + - CONTRIBUTING.md +linter diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f7b297e..3f52e803 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,11 @@ make black make black-check ``` +- To apply [`ruff`](https://beta.ruff.rs/docs/) linter: +``` +make ruff-check +``` + - To run tests: `make test`. - For testing `black_it.{SUBMODULE}` with `tests/test_{TESTMODULE}` use: diff --git a/Makefile b/Makefile index 86d1b5c8..962ce277 100644 --- a/Makefile +++ b/Makefile @@ -63,10 +63,10 @@ clean-test: ## remove test and coverage artifacts rm -fr coverage.xml PHONY.: lint-all -lint-all: black static bandit safety vulture darglint ## run all linters +lint-all: black ruff static bandit safety vulture darglint ## run all linters PHONY.: lint-all-files -lint-all-files: black-files static-files bandit-files vulture-files darglint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") +lint-all-files: black-files ruff-files static-files bandit-files vulture-files darglint-files ## run all linters for specific files (specified with files="file1 file2 somedir ...") PHONY.: static static: ## static type checking with mypy @@ -90,10 +90,29 @@ PHONY.: black-check black-check: ## check black formatting black --check --verbose black_it tests scripts examples +PHONY.: black-check-files black-check-files: ## check black formatting for specific files (specified with files="file1 file2 somedir ...") $(call check_defined, files) black --check --verbose $(files) +PHONY.: ruff +ruff: ## run ruff linter + ruff check --fix . + +PHONY.: ruff-files +ruff-files: ## run ruff linter for specific files (specified with files="file1 file2 somedir ...") + $(call check_defined, files) + ruff check --fix $(files) + +PHONY.: ruff-check +ruff-check: ## check ruff linter rules + ruff check . + +PHONY.: ruff-check-files +ruff-check-files: ## check ruff linter rules for specific files (specified with files="file1 file2 somedir ...") + $(call check_defined, files) + ruff check $(files) + PHONY.: bandit bandit: ## run bandit bandit --configfile .bandit.yaml --recursive black_it tests scripts examples diff --git a/tox.ini b/tox.ini index 55085694..110efe60 100644 --- a/tox.ini +++ b/tox.ini @@ -58,6 +58,16 @@ skip_install = True deps = black==23.7.0 commands = black black_it tests scripts examples --check --verbose +[testenv:ruff] +skip_install = True +deps = ruff==0.0.290 +commands = ruff check --fix . + +[testenv:ruff-check] +skip_install = True +deps = ruff==0.0.290 +commands = ruff check . + [testenv:bandit] skipsdist = True skip_install = True From 477cfa560ad9d9f9df624a269c5116745d0c86b2 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 10:54:30 +0200 Subject: [PATCH 13/56] lint: fix pyflakes errors To reproduce the errors: ``` ruff check --select "F" black_it tests examples scripts ``` This will first give the following error: ``` examples/models/economics/brock_hommes.py:62:5: F841 [*] Local variable `pStar` is assigned to but never used ``` once fixed, running the same command again will give the following error: ``` examples/models/economics/brock_hommes.py:54:5: F841 [*] Local variable `div` is assigned to but never used ``` Once fixed the last error, pyflakes does not detect any other issue. --- .ruff.toml | 2 +- examples/models/economics/brock_hommes.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 612c4790..b01167fc 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = [] +select = ["F"] ignore = ["E203"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index 64001055..53f126a6 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -51,7 +51,6 @@ def BH2(theta: Sequence[float], N: int, seed: int): beta = 3.6 sigma = 1.0 a = 1.0 - div = 0.1 divEpsMin = 0 # -0.05 divEpsMax = 0 # 0.05 C = 1.0 @@ -59,7 +58,6 @@ def BH2(theta: Sequence[float], N: int, seed: int): x_lag2 = 0.10 x_lag1 = 0.10 - pStar = div / (R - 1) bsa = beta / (a * (sigma**2)) x = np.zeros(N + 2) From d8c9673f58afc7486380c6a7bf8a1e0b59afba68 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 12:02:21 +0200 Subject: [PATCH 14/56] lint: fix pep8-naming errors To reproduce the errors: ``` ruff check --select "N" black_it tests examples scripts ``` Some of the errors were fixed; other were ignored using the directives '# noqa: NXXX'. We decided to ignore those errors that would have required invasive changes throughout the codebase. For example, the model functions in examples/models have a positional parameter named 'N', which should be lowercase, but in other parts of the code the model functions were called with "N" as keyworded parameter. Hence, renaming N to n would have required a change to the function calls, replacing 'N=' with 'n='. The same strategy was applied for function names that were used in many parts of the project. Instead, many narrow-scoped variables were modified properly as suggested by the Ruff errors. --- .ruff.toml | 2 +- black_it/calibrator.py | 6 +- black_it/loss_functions/fourier.py | 4 +- black_it/loss_functions/gsl_div.py | 14 ++-- black_it/loss_functions/likelihood.py | 36 ++++----- black_it/loss_functions/msm.py | 4 +- black_it/plot/plot_descriptive_statistics.py | 52 ++++++------- black_it/samplers/cors.py | 18 +++-- black_it/samplers/gaussian_process.py | 16 ++-- black_it/samplers/random_forest.py | 12 +-- black_it/samplers/surrogate.py | 4 +- black_it/samplers/xgboost.py | 4 +- black_it/utils/json_pandas_checkpointing.py | 4 +- black_it/utils/sqlite3_checkpointing.py | 10 +-- black_it/utils/time_series.py | 4 +- examples/models/economics/boltzmann_wealth.py | 6 +- examples/models/economics/brock_hommes.py | 18 ++--- examples/models/forest_fire/forest_fire.py | 5 +- examples/models/simple_models.py | 76 ++++++++++++++----- examples/models/sir/sir_docker.py | 4 +- examples/models/sir/sir_python.py | 36 ++++----- tests/test_losses/test_likelihood.py | 8 +- tests/test_utils/test_base.py | 6 +- .../test_pandas_json_checkpointing.py | 4 +- .../test_utils/test_sqlite3_checkpointing.py | 4 +- 25 files changed, 201 insertions(+), 156 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index b01167fc..dd20c879 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F"] +select = ["F","W","E","C90","I","N"] ignore = ["E203"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 41bb89b9..b245b66e 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -254,8 +254,8 @@ def restore_from_checkpoint( parameters_precision, real_data, ensemble_size, - N, - _D, + sim_length, + _d, convergence_precision, verbose, saving_file, @@ -290,7 +290,7 @@ def restore_from_checkpoint( parameters_precision, ensemble_size, scheduler=scheduler, - sim_length=N, + sim_length=sim_length, convergence_precision=convergence_precision, verbose=verbose, saving_folder=saving_file, diff --git a/black_it/loss_functions/fourier.py b/black_it/loss_functions/fourier.py index 9307574d..5924508a 100644 --- a/black_it/loss_functions/fourier.py +++ b/black_it/loss_functions/fourier.py @@ -132,7 +132,7 @@ def compute_loss_1d( The computed loss over the specific coordinate. """ f_real_data = np.fft.rfft(real_data, axis=0) - N = f_real_data.shape[0] + ts_length = f_real_data.shape[0] f_real_data = self.frequency_filter(f_real_data, self.f) # computer mean fft transform of simulated ensemble f_sim_data = [] @@ -143,6 +143,6 @@ def compute_loss_1d( f_sim_data = np.array(f_sim_data).mean(0) - loss_1d = np.sqrt(np.sum((abs(f_sim_data - f_real_data)) ** 2) / N) + loss_1d = np.sqrt(np.sum((abs(f_sim_data - f_real_data)) ** 2) / ts_length) return loss_1d diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 5a65335c..7ba94ae8 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -118,16 +118,16 @@ def compute_loss_1d( Returns: the GSL loss """ - N = len(real_data) + ts_length = len(real_data) ensemble_size = sim_data_ensemble.shape[0] if self.nb_values is None: - nb_values = int((N - 1) / 2.0) + nb_values = int((ts_length - 1) / 2.0) else: nb_values = self.nb_values if self.nb_word_lengths is None: - nb_word_lengths = int((N - 1) / 2.0) + nb_word_lengths = int((ts_length - 1) / 2.0) else: nb_word_lengths = self.nb_word_lengths @@ -149,7 +149,7 @@ def compute_loss_1d( ) loss = self.gsl_div_1d_1_sample( - sim_xd, obs_xd, nb_word_lengths, nb_values, N + sim_xd, obs_xd, nb_word_lengths, nb_values, ts_length ) gsl_loss += loss @@ -162,7 +162,7 @@ def gsl_div_1d_1_sample( obs_xd: NDArray, nb_word_lengths: int, nb_values: int, - N: int, + ts_length: int, ) -> float: """Compute the GSL-div for a single realisation of the simulated data. @@ -171,7 +171,7 @@ def gsl_div_1d_1_sample( obs_xd: discretised real series nb_word_lengths: the number of word length to consider nb_values: number of values the digitised series can take - N: the length of real and simulated series + ts_length: the length of real and simulated series Returns: the computed loss @@ -197,7 +197,7 @@ def gsl_div_1d_1_sample( weight = weight + 2 / (nb_word_lengths * (nb_word_lengths + 1)) # correction - corr = ((len(m_xp) - 1) - (len(sim_xp) - 1)) / (2 * N) + corr = ((len(m_xp) - 1) - (len(sim_xp) - 1)) / (2 * ts_length) # add to measure gsl_divl = 2 * m_entr - sim_entr + corr diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 2356a124..92619555 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -24,9 +24,9 @@ from black_it.loss_functions.base import BaseLoss -def kernel(sq_dist: NDArray[np.float64], h: float, D: int) -> NDArray[np.float64]: +def kernel(sq_dist: NDArray[np.float64], h: float, d: int) -> NDArray[np.float64]: """Compute a kernel density estimation using a Gaussian kernel.""" - k = np.exp(-(sq_dist / (2 * h**2))) / (h**D * (2 * np.pi) ** (D / 2.0)) + k = np.exp(-(sq_dist / (2 * h**2))) / (h**d * (2 * np.pi) ** (d / 2.0)) return k @@ -57,15 +57,15 @@ def __init__( self.h = h @staticmethod - def _get_bandwidth_silverman(N: int, D: int) -> float: + def _get_bandwidth_silverman(n: int, d: int) -> float: """Return a reasonable bandwidth value computed using the Silverman's rule of thumb.""" - h = ((N * (D + 2)) / 4) ** (-1 / (D + 4)) + h = ((n * (d + 2)) / 4) ** (-1 / (d + 4)) return h @staticmethod - def _get_bandwidth_scott(N: int, D: int) -> float: + def _get_bandwidth_scott(n: int, d: int) -> float: """Return a reasonable bandwidth value computed using the Scott's rule of thumb.""" - h = N ** (-1 / (D + 4)) + h = n ** (-1 / (d + 4)) return h def compute_loss( @@ -81,9 +81,9 @@ def compute_loss( Returns: The loss value. """ - R = sim_data_ensemble.shape[0] # number of repetitions - S = sim_data_ensemble.shape[1] # simulation length - D = sim_data_ensemble.shape[2] # number of dimensions + r = sim_data_ensemble.shape[0] # number of repetitions + s = sim_data_ensemble.shape[1] # simulation length + d = sim_data_ensemble.shape[2] # number of dimensions if self.coordinate_weights is not None: warnings.warn( # noqa: B028 @@ -92,35 +92,35 @@ def compute_loss( RuntimeWarning, ) - filters = self._check_coordinate_filters(D) + filters = self._check_coordinate_filters(d) filtered_data = self._filter_data(filters, sim_data_ensemble) sim_data_ensemble = np.transpose(filtered_data, (1, 2, 0)) - h = self._check_bandwidth(S, D) + h = self._check_bandwidth(s, d) sq_dists_r_t_s = ( 1.0 - / D + / d * np.sum( (sim_data_ensemble[:, None, :, :] - real_data[None, :, None, :]) ** 2, axis=3, ) ) - kernel_r_t_s = kernel(sq_dists_r_t_s, h, D) - lik_real_series_r_t = np.sum(kernel_r_t_s, axis=2) / S + kernel_r_t_s = kernel(sq_dists_r_t_s, h, d) + lik_real_series_r_t = np.sum(kernel_r_t_s, axis=2) / s log_lik_real_series_r = np.sum(np.log(lik_real_series_r_t), axis=1) - log_lik_real_series = np.sum(log_lik_real_series_r, axis=0) / R + log_lik_real_series = np.sum(log_lik_real_series_r, axis=0) / r return -log_lik_real_series - def _check_bandwidth(self, S: int, D: int) -> float: + def _check_bandwidth(self, s: int, d: int) -> float: """Check the bandwidth self.h and return a usable one.""" h: float if isinstance(self.h, str): if self.h == "silverman": - h = self._get_bandwidth_silverman(S, D) + h = self._get_bandwidth_silverman(s, d) elif self.h == "scott": - h = self._get_bandwidth_scott(S, D) + h = self._get_bandwidth_scott(s, d) else: raise KeyError( "Select a valid rule of thumb (either 'silverman' or 'scott') " diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 73669e0a..48a08428 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -182,12 +182,12 @@ def compute_loss_1d( loss_1d = g.dot(g) return loss_1d if self._covariance_mat == _CovarianceMatrixType.INVERSE_VARIANCE.value: - W = np.diag( + W = np.diag( # noqa: N806 1.0 / np.mean((real_mom_1d[None, :] - ensemble_sim_mom_1d) ** 2, axis=0) ) else: self._covariance_mat = cast(NDArray[np.float64], self._covariance_mat) - W = self._covariance_mat + W = self._covariance_mat # noqa: N806 try: loss_1d = g.dot(W).dot(g) except ValueError as e: diff --git a/black_it/plot/plot_descriptive_statistics.py b/black_it/plot/plot_descriptive_statistics.py index e018ae38..b11c97f1 100644 --- a/black_it/plot/plot_descriptive_statistics.py +++ b/black_it/plot/plot_descriptive_statistics.py @@ -23,72 +23,72 @@ import statsmodels.api as sm -def ts_stats(ts: List[float]) -> None: +def ts_stats(time_series_raw: List[float]) -> None: """Show TS graphical descriptive statistics.""" color = "darkslateblue" alpha = 0.8 - Ts = np.array(ts, dtype="double") - SqDiff = np.append( - np.absolute(np.diff(Ts)), 0 + ts = np.array(time_series_raw, dtype="double") + sq_diff = np.append( + np.absolute(np.diff(ts)), 0 ) # not precise! shouldn't append '0'! - TsAcf = sm.tsa.acf(Ts, nlags=20) - SqDiffAcf = sm.tsa.acf(SqDiff, nlags=20) + ts_acf = sm.tsa.acf(ts, nlags=20) + sq_diff_acf = sm.tsa.acf(sq_diff, nlags=20) - TsPAcf = sm.tsa.pacf(Ts, nlags=20) - SqDiffPAcf = sm.tsa.pacf(SqDiff, nlags=20) + ts_p_acf = sm.tsa.pacf(ts, nlags=20) + sq_diff_p_acf = sm.tsa.pacf(sq_diff, nlags=20) - WINDOW_W = 20 + window_w = 20 - fig, _ax = plt.subplots(3, 4, figsize=(WINDOW_W, 15), sharex=False, sharey=False) + fig, _ax = plt.subplots(3, 4, figsize=(window_w, 15), sharex=False, sharey=False) sp1 = plt.subplot(3, 4, 1) - plt.hist(Ts, 50, facecolor=color, alpha=alpha) + plt.hist(ts, 50, facecolor=color, alpha=alpha) sp1.set_title("Ts hist") sp2 = plt.subplot(3, 4, 2) - plt.plot(TsAcf, marker="o", linestyle="None", c=color, alpha=alpha) + plt.plot(ts_acf, marker="o", linestyle="None", c=color, alpha=alpha) sp2.set_title("Ts acf") sp3 = plt.subplot(3, 4, 3) - plt.plot(TsPAcf, marker="o", linestyle="None", c=color, alpha=alpha) + plt.plot(ts_p_acf, marker="o", linestyle="None", c=color, alpha=alpha) sp3.set_title("Ts pacf") sp4 = plt.subplot(3, 4, 4) - plt.plot(Ts[0:1000], c=color, alpha=alpha) + plt.plot(ts[0:1000], c=color, alpha=alpha) sp4.set_title("Ts sample") sp5 = plt.subplot(3, 4, 5) - plt.hist(SqDiff, 50, facecolor=color, alpha=alpha) + plt.hist(sq_diff, 50, facecolor=color, alpha=alpha) sp5.set_title("Abs 1st diff hist") sp6 = plt.subplot(3, 4, 6) - plt.plot(SqDiffAcf, marker="o", linestyle="None", c=color, alpha=alpha) + plt.plot(sq_diff_acf, marker="o", linestyle="None", c=color, alpha=alpha) sp6.set_title("Abs 1st diff acf") sp7 = plt.subplot(3, 4, 7) - plt.plot(SqDiffPAcf, marker="o", linestyle="None", c=color, alpha=alpha) + plt.plot(sq_diff_p_acf, marker="o", linestyle="None", c=color, alpha=alpha) sp7.set_title("Abs 1st diff pacf") sp8 = plt.subplot(3, 4, 8) - plt.plot(SqDiff[0:1000], c=color, alpha=alpha) + plt.plot(sq_diff[0:1000], c=color, alpha=alpha) sp8.set_title("Abs 1st diff sample") sp9 = plt.subplot(3, 4, 9) - for i in range(len(Ts) - 3): - plt.plot(Ts[i + 1 : i + 3], Ts[i : i + 2], alpha=0.10, c=color) + for i in range(len(ts) - 3): + plt.plot(ts[i + 1 : i + 3], ts[i : i + 2], alpha=0.10, c=color) sp9.set_title("Ts X(t) vs X(t-1) traj") sp10 = plt.subplot(3, 4, 10) - for i in range(len(Ts) - 4): - plt.plot(SqDiff[i + 1 : i + 3], SqDiff[i : i + 2], alpha=0.10, c=color) + for i in range(len(ts) - 4): + plt.plot(sq_diff[i + 1 : i + 3], sq_diff[i : i + 2], alpha=0.10, c=color) sp10.set_title("Abs 1st diff X(t) vs X(t-1) traj") sp11 = plt.subplot(3, 4, 11) plt.plot( - Ts[1 : len(Ts)], - Ts[0 : len(Ts) - 1], + ts[1 : len(ts)], + ts[0 : len(ts) - 1], marker="o", linestyle="None", alpha=0.10, @@ -98,8 +98,8 @@ def ts_stats(ts: List[float]) -> None: sp12 = plt.subplot(3, 4, 12) plt.plot( - SqDiff[1 : len(Ts)], - SqDiff[0 : len(Ts) - 1], + sq_diff[1 : len(ts)], + sq_diff[0 : len(ts) - 1], marker="o", linestyle="None", alpha=0.10, diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index 929ab9ff..f893d622 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -46,13 +46,17 @@ def volume_d_dimensional_ball_radius_1(dims: int) -> float: ) -def cubetobox(X: NDArray[np.float64], space_bounds: NDArray) -> NDArray[np.float64]: +def cubetobox( + X: NDArray[np.float64], space_bounds: NDArray # noqa: N803 +) -> NDArray[np.float64]: """Go from normalized values (unit cube) to absolute values (box).""" box_points = space_bounds[0] + X * (space_bounds[1] - space_bounds[0]) return box_points -def boxtocube(X: NDArray[np.float64], space_bounds: NDArray) -> NDArray[np.float64]: +def boxtocube( + X: NDArray[np.float64], space_bounds: NDArray # noqa: N803 +) -> NDArray[np.float64]: """Go from absolute values (box) to normalized values (unit cube).""" cube_points = (X - space_bounds[0]) / (space_bounds[1] - space_bounds[0]) return cube_points @@ -79,18 +83,18 @@ def phi(r: float) -> float: """Compute phi.""" return r * r * r - Phi = [ + phis = [ [phi(np.linalg.norm(np.subtract(points[i], points[j]))) for j in range(n)] # type: ignore for i in range(n) ] - P = np.ones((n, d + 1)) + P = np.ones((n, d + 1)) # noqa: N806 P[:, 0:-1] = points - F = losses + F = losses # noqa: N806 - M = np.zeros((n + d + 1, n + d + 1)) - M[0:n, 0:n] = Phi + M = np.zeros((n + d + 1, n + d + 1)) # noqa: N806 + M[0:n, 0:n] = phis M[0:n, n : n + d + 1] = P M[n : n + d + 1, 0:n] = np.transpose(P) diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 027fdc4a..93700e53 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -101,7 +101,7 @@ def _validate_acquisition(acquisition: str) -> None: f"got {acquisition}" ) from e - def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: + def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 """Fit a gaussian process surrogate model.""" y = np.atleast_2d(y).T @@ -131,7 +131,7 @@ def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: m, _ = self._predict_mean_std(X) self._fmin = np.min(m) - def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: + def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 """Predict using a gaussian process surrogate model.""" # predict mean or expected improvement on the full sample set if self.acquisition == _AcquisitionTypes.EI.value: @@ -143,7 +143,7 @@ def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: return candidates_score def _predict_mean_std( - self, X: NDArray[np.float64] + self, X: NDArray[np.float64] # noqa: N803 ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: """ Predict mean and standard deviation of a fitted GP. @@ -155,12 +155,14 @@ def _predict_mean_std( The pair (mean, std). """ gpmodel = cast(GaussianProcessRegressor, self._gpmodel) - X = X[None, :] if X.ndim == 1 else X + X = X[None, :] if X.ndim == 1 else X # noqa: N806 m, s = gpmodel.predict(X, return_std=True, return_cov=False) s = np.clip(s, 1e-5, np.inf) return m, s - def _predict_EI(self, X: NDArray[np.float64], jitter: float) -> NDArray[np.float64]: + def _predict_EI( # noqa: N802 + self, X: NDArray[np.float64], jitter: float = 0.1 # noqa: N803 + ) -> NDArray[np.float64]: """ Compute the Expected Improvement per unit of cost. @@ -175,7 +177,7 @@ def _predict_EI(self, X: NDArray[np.float64], jitter: float) -> NDArray[np.float fmin = cast(float, self._fmin) - phi, Phi, u = self.get_quantiles(jitter, fmin, m, s) + phi, Phi, u = self.get_quantiles(jitter, fmin, m, s) # noqa: N806 f_acqu = s * (u * Phi + phi) @@ -205,6 +207,6 @@ def get_quantiles( u: NDArray[np.float64] = (fmin - m - acquisition_par) / s phi: NDArray[np.float64] = np.exp(-0.5 * u**2) / np.sqrt(2 * np.pi) - Phi: NDArray[np.float64] = 0.5 * erfc(-u / np.sqrt(2)) + Phi: NDArray[np.float64] = 0.5 * erfc(-u / np.sqrt(2)) # noqa: N806 return phi, Phi, u diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 89c9a9f6..4dce88e1 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -82,13 +82,15 @@ def n_classes(self) -> int: """Get the number of classes.""" return self._n_classes - def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: + def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 """Fit a random forest surrogate model.""" # Train surrogate - X, y_cat, _existing_points_quantiles = self.prepare_data_for_classifier( - X, y, self.n_classes - ) + ( + X, # noqa: N806 + y_cat, + _existing_points_quantiles, + ) = self.prepare_data_for_classifier(X, y, self.n_classes) self._classifier = RandomForestClassifier( n_estimators=self.n_estimators, @@ -98,7 +100,7 @@ def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: ) self._classifier.fit(X, y_cat) - def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: + def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 """Predict using a random forest surrogate model.""" # Predict quantiles self._classifier = cast(RandomForestClassifier, self._classifier) diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index 77c8af30..5935c869 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -79,11 +79,11 @@ def sample_candidates( return candidates @abstractmethod - def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: + def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 """Abstract method to fit the loss function of an ML surrogate.""" @abstractmethod - def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: + def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 """Abstract method for the predictions of an ML surrogate.""" def sample_batch( diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 00b357c4..9162ef19 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -121,7 +121,7 @@ def _clip_losses(y: NDArray[np.float64]) -> NDArray[np.float64]: return y - def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: + def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 """Fit a xgboost surrogate model.""" # prepare data y = self._clip_losses(y) @@ -140,7 +140,7 @@ def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: self._xg_regressor.fit(X, y) - def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: + def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 """Predict using a xgboost surrogate model.""" # predict over large pool of candidates _ = xgb.DMatrix(data=X) diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index b6b9e76e..630f67f0 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -99,8 +99,8 @@ def save_calibrator_state( parameters_precision: NDArray[np.float64], real_data: NDArray[np.float64], ensemble_size: int, - N: int, - D: int, + N: int, # noqa: N803 + D: int, # noqa: N803 convergence_precision: Optional[float], verbose: bool, saving_file: Optional[str], diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index 9ddac27a..6e97b1e0 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -148,7 +148,7 @@ """ -class gz_ndarray(NDArray): +class gz_ndarray(NDArray): # noqa: N801 """Convenience class to tell sqlite which ndarrays are to be compressed. If a user wants to compress/decompress the serialized version of an NDArray @@ -203,8 +203,8 @@ def load_calibrator_state( parameters_precision, real_data, ensemble_size, - N, - D, + N, # noqa: N806 + D, # noqa: N806 convergence_precision, verbose, saving_file, @@ -259,8 +259,8 @@ def save_calibrator_state( parameters_precision: NDArray[np.float64], real_data: NDArray[np.float64], ensemble_size: int, - N: int, - D: int, + N: int, # noqa: N803 + D: int, # noqa: N803 convergence_precision: Optional[float], verbose: bool, saving_file: Optional[str], diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index a9de3b67..b5730ca9 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -104,10 +104,10 @@ def hp_filter( Carnegie Mellon University discussion paper no. 451. """ nobs = len(time_series) - I = sps.eye(nobs, nobs) # noqa:E741 + I = sps.eye(nobs, nobs) # noqa: E741, N806 offsets = np.array([0, 1, 2]) data = np.repeat([[1.0], [-2.0], [1.0]], nobs, axis=1) - K = sps.dia_matrix((data, offsets), shape=(nobs - 2, nobs)) + K = sps.dia_matrix((data, offsets), shape=(nobs - 2, nobs)) # noqa: N806 trend = sps.linalg.spsolve(I + lamb * K.T.dot(K), time_series, use_umfpack=True) cycle = time_series - trend diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index f2ec7971..e1cd06cf 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -122,6 +122,6 @@ def compute_gini(model: BoltzmannWealthModel) -> float: """Compute the Gini index.""" agent_wealths = [cast(MoneyAgent, agent).wealth for agent in model.schedule.agents] x = sorted(agent_wealths) - N = model.num_agents - B = sum(xi * (N - i) for i, xi in enumerate(x)) / (N * sum(x)) - return 1 + (1 / N) - 2 * B + n = model.num_agents + b = sum(xi * (n - i) for i, xi in enumerate(x)) / (n * sum(x)) # noqa: N806 + return 1 + (1 / n) - 2 * b diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index 53f126a6..dce7d8c5 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -24,7 +24,7 @@ # **** -def BH2(theta: Sequence[float], N: int, seed: int): +def BH2(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 """ Model from Brock and Hommes 1998. 4.1.2. Fundamentalists versus trend chasers @@ -47,13 +47,13 @@ def BH2(theta: Sequence[float], N: int, seed: int): """ np.random.seed(seed=seed) - R = 1.10 + R = 1.10 # noqa: N806 beta = 3.6 sigma = 1.0 a = 1.0 - divEpsMin = 0 # -0.05 - divEpsMax = 0 # 0.05 - C = 1.0 + div_eps_min = 0 # -0.05 + div_eps_max = 0 # 0.05 + C = 1.0 # noqa: N806 x_lag2 = 0.10 x_lag1 = 0.10 @@ -74,7 +74,7 @@ def BH2(theta: Sequence[float], N: int, seed: int): for t in range(2, N + 1): x[t] = n[1] * g[1] * x[t - 1] / R - x[t] = x[t] + np.random.uniform(low=divEpsMin, high=divEpsMax, size=1) + x[t] = x[t] + np.random.uniform(low=div_eps_min, high=div_eps_max, size=1) n[0] = np.exp(bsa * (R * x[t - 1] * (R * x[t - 1] - x[t])) - beta * C) n[1] = np.exp(bsa * (x[t] - R * x[t - 1]) * (g[1] * x[t - 2] - R * x[t - 1])) @@ -84,7 +84,7 @@ def BH2(theta: Sequence[float], N: int, seed: int): # -def BH4(theta: Sequence[float], N: int, seed: int): +def BH4(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 """Model from Brock and Hommes 1998. 4.3 Four belief types: Fundamentalists versus trend versus bias @@ -109,7 +109,7 @@ def BH4(theta: Sequence[float], N: int, seed: int): """ np.random.seed(seed=seed) - R = 1.01 + R = 1.01 # noqa: N806 beta = 120 sigma = 0.04 @@ -130,7 +130,7 @@ def BH4(theta: Sequence[float], N: int, seed: int): b[i] = theta[i * 2 + 1] if len(theta) >= 9: - R = 1.0 + theta[8] + R = 1.0 + theta[8] # noqa: N806 if len(theta) >= 10: beta = theta[9] diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 066cb7b6..1f2abfaf 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -19,7 +19,7 @@ import numpy as np -def forest_fire(theta, N, rndSeed=0): +def forest_fire(theta, N, seed=0): # noqa: N803 """A simple model of a wildfire on a 2D grid. The model is taken from this example from Agent.jl @@ -33,7 +33,7 @@ def forest_fire(theta, N, rndSeed=0): Args: theta: the initial density of trees on the 2D grid N: the length of the simulation - rndSeed: the random seed of the simulation + seed: the random seed of the simulation Returns: An array containing the fraction of trees burned at each time step @@ -45,7 +45,6 @@ def forest_fire(theta, N, rndSeed=0): # the size of the grid is fixed xsize = 30 ysize = 30 - seed = rndSeed command = "julia forest_fire_julia.jl {} {} {} {} {}".format( density, n, xsize, ysize, seed diff --git a/examples/models/simple_models.py b/examples/models/simple_models.py index 6ada5ba5..006ff54d 100644 --- a/examples/models/simple_models.py +++ b/examples/models/simple_models.py @@ -35,14 +35,18 @@ from scipy.stats import alpha, bernoulli -def NormalM(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def NormalM( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Normal samples with adjustable mean.""" np.random.seed(seed=seed) y = np.random.normal(theta[0], 1, N) return np.atleast_2d(y).T -def NormalMV(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def NormalMV( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Normal samples with adjustable mean and variance.""" np.random.seed(seed=seed) @@ -50,7 +54,9 @@ def NormalMV(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def NormalBer_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def NormalBer_3P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" np.random.seed(seed=seed) @@ -65,7 +71,9 @@ def NormalBer_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float6 return np.atleast_2d(y).T -def NormalBer_5P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def NormalBer_5P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" np.random.seed(seed=seed) @@ -80,7 +88,9 @@ def NormalBer_5P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float6 return np.atleast_2d(y).T -def Alpha(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def Alpha( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Alpha iid samples. theta[0] shape param @@ -94,7 +104,9 @@ def Alpha(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def MarkovC_2P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def MarkovC_2P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -122,7 +134,9 @@ def MarkovC_2P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64] return np.atleast_2d(y).T -def MarkovC_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def MarkovC_3P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -144,7 +158,9 @@ def MarkovC_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64] return np.atleast_2d(y).T -def MarkovC_KP(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def MarkovC_KP( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -165,7 +181,9 @@ def MarkovC_KP(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64] return np.atleast_2d(y).T -def AR1(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def AR1( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """AR(1) model. Model 1 in Platt (2019) @@ -182,7 +200,9 @@ def AR1(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def AR1Ber_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def AR1Ber_3P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """AR(1) + Bernoulli.""" np.random.seed(seed=seed) @@ -203,7 +223,9 @@ def AR1Ber_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def AR1_2P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def AR1_2P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """AR(1) with 2 parameters.""" np.random.seed(seed=seed) @@ -216,7 +238,9 @@ def AR1_2P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def AR1_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def AR1_3P( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """AR(1) with 3 parameters.""" np.random.seed(seed=seed) @@ -230,7 +254,9 @@ def AR1_3P(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def ARMA2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def ARMA2( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ARMA(1, 1) model.""" np.random.seed(seed=seed) @@ -242,7 +268,9 @@ def ARMA2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y[1:]).T -def ARMAARCH2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def ARMAARCH2( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ARMA(2,2) ARCH(2) model. Model 2 of Platt (2019) - Param set 1 [a0,a1] @@ -268,7 +296,9 @@ def ARMAARCH2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y[2:]).T -def ARMAARCH4(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def ARMAARCH4( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ ARMA(2,2) ARCH(2) model. @@ -297,7 +327,9 @@ def ARMAARCH4(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y[2:]).T -def ARMAARCH6(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def ARMAARCH6( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ ARMA(2,2) ARCH(2). @@ -328,7 +360,9 @@ def ARMAARCH6(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y[2:]).T -def ARMAARCH4v2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def ARMAARCH4v2( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ ARMA(2,2) ARCH(2). @@ -355,7 +389,9 @@ def ARMAARCH4v2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64 return np.atleast_2d(y[2:]).T -def RWSB1(theta: Sequence[int], N: int, seed: int) -> NDArray[np.float64]: +def RWSB1( # noqa: N802 + theta: Sequence[int], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ RW with structural break. @@ -381,7 +417,9 @@ def RWSB1(theta: Sequence[int], N: int, seed: int) -> NDArray[np.float64]: return np.atleast_2d(y).T -def RWSB2(theta: Sequence[float], N: int, seed: int) -> NDArray[np.float64]: +def RWSB2( # noqa: N802 + theta: Sequence[float], N: int, seed: int # noqa: N803 +) -> NDArray[np.float64]: """ RW with structural break. diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index 5b92da7b..6caae6c8 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -21,7 +21,7 @@ from . import simlib -def SIR(theta, N, rndSeed): +def SIR(theta, N, seed): # noqa: N802, N803 """SIR_docker. C++ SIR model run in Docker container. @@ -42,7 +42,7 @@ def SIR(theta, N, rndSeed): return ret -def SIR_w_breaks(theta, N, rndSeed=None): +def SIR_w_breaks(theta, N, seed=None): # noqa: N802, N803 """SIR_docker_w_breaks.""" breaktime = int(theta[0]) diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index 505f4dae..c94160bc 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -17,12 +17,12 @@ """SIR models written in Python.""" import ndlib.models.epidemics as ep -import ndlib.models.ModelConfig as mc import networkx as nx import numpy as np +from ndlib.models import ModelConfig -def SIR(theta, N, seed): +def SIR(theta, N, seed): # noqa: N802, N803 """SIR model. 0 theta = [#LOC CONNECTIONS, @@ -42,12 +42,12 @@ def SIR(theta, N, seed): """ np.random.seed(seed=seed) - NumAgents = 100000 - g = nx.watts_strogatz_graph(NumAgents, int(theta[0]), theta[1], seed=theta[5]) + num_agents = 100000 + g = nx.watts_strogatz_graph(num_agents, int(theta[0]), theta[1], seed=theta[5]) model = ep.SIRModel(g) - cfg = mc.Configuration() + cfg = ModelConfig.Configuration() cfg.add_model_parameter("beta", theta[3]) # infection rate cfg.add_model_parameter("gamma", theta[4]) # recovery rate cfg.add_model_parameter("percentage_infected", theta[2]) @@ -55,20 +55,20 @@ def SIR(theta, N, seed): iterations = model.iteration_bunch(N, node_status=True) - outputNP = np.zeros((N, 3)) + output_np = np.zeros((N, 3)) for i in range(len(iterations)): entry = iterations[i]["node_count"] for j in range(3): - outputNP[i, j] = entry[j] + output_np[i, j] = entry[j] g.clear() - return outputNP + return output_np -def SIR_w_breaks(theta, N, seed): +def SIR_w_breaks(theta, N, seed): # noqa: N802, N803 """SIR model with structural breaks. 0 theta = [#LOC CONNECTIONS, @@ -94,12 +94,12 @@ def SIR_w_breaks(theta, N, seed): """ np.random.seed(seed=seed) - NumAgents = 100000 - g = nx.watts_strogatz_graph(NumAgents, int(theta[0]), theta[1], seed=theta[11]) + num_agents = 100000 + g = nx.watts_strogatz_graph(num_agents, int(theta[0]), theta[1], seed=theta[11]) model = ep.SIRModel(g) - cfg = mc.Configuration() + cfg = ModelConfig.Configuration() cfg.add_model_parameter("beta", theta[3]) # infection rate cfg.add_model_parameter("gamma", theta[4]) # recovery rate cfg.add_model_parameter("percentage_infected", theta[2]) @@ -116,29 +116,29 @@ def SIR_w_breaks(theta, N, seed): model.params["model"]["beta"] = theta[7] iterations3 = model.iteration_bunch(int(N - theta[10]), node_status=True) - outputNP = np.zeros((N, 3)) + output_np = np.zeros((N, 3)) for i in range(len(iterations0)): entry = iterations0[i]["node_count"] for j in range(3): - outputNP[i, j] = entry[j] + output_np[i, j] = entry[j] for i in range(len(iterations1)): entry = iterations1[i]["node_count"] for j in range(3): - outputNP[i + theta[8], j] = entry[j] + output_np[i + theta[8], j] = entry[j] for i in range(len(iterations2)): entry = iterations2[i]["node_count"] for j in range(3): - outputNP[i + theta[9], j] = entry[j] + output_np[i + theta[9], j] = entry[j] for i in range(len(iterations3)): entry = iterations3[i]["node_count"] for j in range(3): - outputNP[i + theta[10], j] = entry[j] + output_np[i + theta[10], j] = entry[j] - return outputNP + return output_np diff --git a/tests/test_losses/test_likelihood.py b/tests/test_losses/test_likelihood.py index 865ee75e..ca4d122d 100644 --- a/tests/test_losses/test_likelihood.py +++ b/tests/test_losses/test_likelihood.py @@ -58,17 +58,17 @@ def test_likelihood_2d_wsigma() -> None: """Test the computation of the Likelihood in the Likelihood loss in 2d.""" # sample from a Gaussian distribution. np.random.seed(11) - sigma, D = 3.0, 2 - real_data = np.random.normal(0, sigma, size=(10, D)) + sigma, d = 3.0, 2 + real_data = np.random.normal(0, sigma, size=(10, d)) expected_neg_log_likelihood = -np.sum( -0.5 / sigma**2 * np.sum(real_data**2, axis=1) - - D / 2.0 * np.log(2.0 * np.pi * sigma**2), + - d / 2.0 * np.log(2.0 * np.pi * sigma**2), axis=0, ) expected_likelihood = np.exp(-expected_neg_log_likelihood) - sim_data_ensemble = np.random.normal(0, sigma, size=(1, 1000000, D)) + sim_data_ensemble = np.random.normal(0, sigma, size=(1, 1000000, d)) loss = LikelihoodLoss(h=sigma) neg_log_lik = loss.compute_loss(sim_data_ensemble, real_data) lik = np.exp(-neg_log_lik) diff --git a/tests/test_utils/test_base.py b/tests/test_utils/test_base.py index 2af0d3f8..08637cab 100644 --- a/tests/test_utils/test_base.py +++ b/tests/test_utils/test_base.py @@ -38,11 +38,11 @@ def test_assert_custom_exception() -> None: with pytest.raises(ValueError, match=message): _assert(False, message, exception_class=ValueError) - class CustomException(Exception): + class CustomError(Exception): """Custom exception.""" - with pytest.raises(CustomException, match=message): - _assert(False, message, exception_class=CustomException) + with pytest.raises(CustomError, match=message): + _assert(False, message, exception_class=CustomError) def test_digitize_data() -> None: diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index cd4b4ff5..c78cdc97 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -31,8 +31,8 @@ def test_save_and_load_calibrator_state() -> None: parameters_precision = np.array([0.01, 0.01]) real_data = np.random.randn(100, 5) ensemble_size = 5 - N = 30 - D = 2 + N = 30 # noqa: N806 + D = 2 # noqa: N806 convergence_precision = 0.1 verbose = True saving_folder = "saving_folder" diff --git a/tests/test_utils/test_sqlite3_checkpointing.py b/tests/test_utils/test_sqlite3_checkpointing.py index e36eda25..16859478 100644 --- a/tests/test_utils/test_sqlite3_checkpointing.py +++ b/tests/test_utils/test_sqlite3_checkpointing.py @@ -35,8 +35,8 @@ def test_sqlite3_checkpointing() -> None: parameters_precision = np.array([0.1, 0.1, 0.1]) real_data = np.linspace(0, 30) ensemble_size = 5 - N = 30 - D = 3 + N = 30 # noqa: N806 + D = 3 # noqa: N806 convergence_precision = 0.1 verbose = True saving_file = "test" From de169ea695989507276a180991b0be618bdf4030 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 13:32:43 +0200 Subject: [PATCH 15/56] lint: fix pydocstyle errors To reproduce the errors: ``` ruff check --select "D" black_it tests examples scripts ``` Most of the errors have been fixed using the '--fix' flag. --- .ruff.toml | 2 +- black_it/calibrator.py | 21 ++++++---------- black_it/loss_functions/base.py | 9 +++---- black_it/loss_functions/gsl_div.py | 24 +++++++------------ black_it/loss_functions/likelihood.py | 6 ++--- black_it/loss_functions/minkowski.py | 6 ++--- black_it/loss_functions/msm.py | 12 ++++------ black_it/plot/plot_results.py | 3 +-- black_it/samplers/base.py | 12 ++++------ black_it/samplers/best_batch.py | 6 ++--- black_it/samplers/cors.py | 13 ++++------ black_it/samplers/gaussian_process.py | 20 ++++++---------- black_it/samplers/halton.py | 24 +++++++------------ black_it/samplers/particle_swarm.py | 9 +++---- black_it/samplers/r_sequence.py | 15 ++++-------- black_it/samplers/random_forest.py | 6 ++--- black_it/samplers/random_uniform.py | 3 +-- black_it/samplers/surrogate.py | 9 +++---- black_it/samplers/xgboost.py | 3 +-- black_it/schedulers/base.py | 15 ++++-------- black_it/schedulers/rl/agents/base.py | 6 ++--- .../schedulers/rl/agents/epsilon_greedy.py | 3 +-- black_it/schedulers/rl/rl_scheduler.py | 6 ++--- black_it/schedulers/round_robin.py | 3 +-- black_it/search_space.py | 6 ++--- black_it/utils/base.py | 12 ++++------ black_it/utils/json_pandas_checkpointing.py | 6 ++--- black_it/utils/seedable.py | 9 +++---- black_it/utils/sqlite3_checkpointing.py | 18 +++++--------- black_it/utils/time_series.py | 15 ++++-------- examples/models/economics/boltzmann_wealth.py | 3 +-- examples/models/economics/brock_hommes.py | 10 +++----- examples/models/forest_fire/forest_fire.py | 3 ++- examples/models/simple_models.py | 18 +++++--------- examples/models/sir/simlib.py | 3 +-- scripts/__init__.py | 17 +++++++++++++ scripts/check_copyright.py | 3 +-- tests/test_losses/test_gsl.py | 3 +-- tests/test_plot/test_plot_results.py | 6 ++--- tests/test_samplers/test_base.py | 3 +-- tests/utils/base.py | 17 +++++-------- tests/utils/docs.py | 6 ++--- tests/utils/strategies.py | 9 +++---- 43 files changed, 148 insertions(+), 255 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index dd20c879..a12092d8 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N"] +select = ["F","W","E","C90","I","N","D"] ignore = ["E203"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index b245b66e..30ca0653 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -63,8 +63,7 @@ def __init__( random_state: Optional[int] = None, n_jobs: Optional[int] = None, ): - """ - Initialize the Calibrator object. + """Initialize the Calibrator object. It must be initialized with details on the parameters to explore, on the model to calibrate, on the samplers and on the loss function to use. @@ -175,8 +174,7 @@ def _set_samplers_seeds(self) -> None: @staticmethod def _construct_samplers_id_table(samplers: List[BaseSampler]) -> Dict[str, int]: - """ - Construct the samplers-by-id table. + """Construct the samplers-by-id table. Given the list (built-in or user-defined) of samplers a calibration session is going to use, return a map from the sampler human-readable @@ -239,8 +237,7 @@ def update_samplers_id_table(self, samplers: Sequence[BaseSampler]) -> None: def restore_from_checkpoint( cls, checkpoint_path: str, model: Callable ) -> "Calibrator": - """ - Return an instantiated class from a database file and a model simulator. + """Return an instantiated class from a database file and a model simulator. Args: checkpoint_path: the name of the database file to read from @@ -312,8 +309,7 @@ def restore_from_checkpoint( return calibrator def simulate_model(self, params: NDArray) -> NDArray: - """ - Simulate the model. + """Simulate the model. This method calls the model simulator in parallel on a given set of parameter values, a number of repeated evaluations are performed for each parameter to average out random fluctuations. @@ -341,8 +337,7 @@ def simulate_model(self, params: NDArray) -> NDArray: return simulated_data def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: - """ - Run calibration for n batches. + """Run calibration for n batches. Args: n_batches (int): number of 'batches' to be executed. Each batch runs over all methods @@ -464,8 +459,7 @@ def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: def check_convergence( losses_samp: NDArray, n_sampled_params: int, convergence_precision: int ) -> bool: - """ - Check convergence of the calibration. + """Check convergence of the calibration. Args: losses_samp: the sampled losses @@ -482,8 +476,7 @@ def check_convergence( return converged def create_checkpoint(self, file_name: Union[str, os.PathLike]) -> None: - """ - Save the current state of the object. + """Save the current state of the object. Args: file_name: the name of the folder where the data will be saved diff --git a/black_it/loss_functions/base.py b/black_it/loss_functions/base.py index 3bfa737b..c239b25c 100644 --- a/black_it/loss_functions/base.py +++ b/black_it/loss_functions/base.py @@ -36,8 +36,7 @@ def __init__( coordinate_weights: Optional[NDArray] = None, coordinate_filters: Optional[List[Optional[Callable]]] = None, ): - """ - Initialize the loss function. + """Initialize the loss function. Args: coordinate_weights: the weights of the loss coordinates. @@ -50,8 +49,7 @@ def __init__( def compute_loss( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Compute the loss between simulated and real data. + """Compute the loss between simulated and real data. Args: sim_data_ensemble: an ensemble of simulated data, of shape (ensemble_size, N, D) @@ -140,8 +138,7 @@ def _check_coordinate_filters(self, num_coords: int) -> List[Optional[Callable]] def compute_loss_1d( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Return the loss between a specific coordinate of two time series. + """Return the loss between a specific coordinate of two time series. Concrete classes have to override this method in order to implement new loss functions. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 7ba94ae8..82fd8af8 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -This module contains the GSL-div implementation. +"""This module contains the GSL-div implementation. - Lamperti, F. (2018). An information theoretic criterion for empirical validation of simulation models. @@ -64,8 +63,7 @@ class GslDivLoss(BaseLoss): - """ - Class for the Gsl-div loss. + """Class for the Gsl-div loss. Example: >>> expected_loss = 0.39737637181336855 @@ -84,8 +82,7 @@ def __init__( coordinate_weights: Optional[NDArray] = None, coordinate_filters: Optional[List[Optional[Callable]]] = None, ) -> None: - """ - Initialize the GSL-div loss object. + """Initialize the GSL-div loss object. Args: nb_values: number of values the digitised series can take @@ -101,8 +98,7 @@ def __init__( def compute_loss_1d( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Return the GSL-div measure. + """Return the GSL-div measure. From (Lamperti, 2017): @@ -213,8 +209,7 @@ def discretize( start_index: Union[np.float64, float], stop_index: Union[np.float64, float], ) -> NDArray[np.int64]: - """ - Discretize the TS in 'nb_values' finite states. + """Discretize the TS in 'nb_values' finite states. >>> GslDivLoss.discretize( ... [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], @@ -242,8 +237,7 @@ def discretize( @staticmethod def get_words(time_series: NDArray[np.float64], length: int) -> NDArray: - """ - Return an overlapping array of words (int32) of 'length' given a discretised vector. + """Return an overlapping array of words (int32) of 'length' given a discretised vector. >>> GslDivLoss.get_words(np.asarray([1, 2, 2, 2]), 2) array([12, 22, 22]) @@ -273,8 +267,7 @@ def get_words(time_series: NDArray[np.float64], length: int) -> NDArray: @staticmethod def get_words_est_prob(time_series: NDArray[np.float64]) -> NDArray[np.float64]: - """ - Return an array of estimated probabilities given an array of words (int32). + """Return an array of estimated probabilities given an array of words (int32). Args: time_series: any univariate array of words @@ -288,8 +281,7 @@ def get_words_est_prob(time_series: NDArray[np.float64]) -> NDArray[np.float64]: @staticmethod def get_sh_entr(probs: NDArray[np.float64], log_base: float) -> float: - """ - Return the Shannon entropy given an array of probabilities. + """Return the Shannon entropy given an array of probabilities. Args: probs: an array of probabilities describing the discrete probability distribution diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 92619555..7d42f44c 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -44,8 +44,7 @@ def __init__( coordinate_filters: Optional[List[Optional[Callable]]] = None, h: Union[str, float] = "silverman", ): - """ - Initialize the loss function. + """Initialize the loss function. Args: coordinate_weights: the weights of the loss coordinates. @@ -71,8 +70,7 @@ def _get_bandwidth_scott(n: int, d: int) -> float: def compute_loss( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Compute the loss between simulated and real data. + """Compute the loss between simulated and real data. Args: sim_data_ensemble: an ensemble of simulated data, of shape (ensemble_size, N, D) diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index f8702e84..f26f68c3 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -33,8 +33,7 @@ def __init__( coordinate_weights: Optional[NDArray] = None, coordinate_filters: Optional[List[Optional[Callable]]] = None, ) -> None: - """ - Loss computed using a Minkowski distance. + """Loss computed using a Minkowski distance. The [Minkowski distance](https://en.wikipedia.org/wiki/Minkowski_distance) is a generalization of both the Manhattan distance (p=1) and the Euclidean distance (p=2). @@ -55,8 +54,7 @@ def __init__( def compute_loss_1d( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Call scipy.spatial.distance.minkowski() on its arguments. + """Call scipy.spatial.distance.minkowski() on its arguments. Args: sim_data_ensemble: the first operand diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 48a08428..80ded962 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -Method-of-moments loss function. +"""Method-of-moments loss function. This module contains the implementation of the loss function based on the 'method of moments'. @@ -59,8 +58,7 @@ def __init__( coordinate_filters: Optional[List[Optional[Callable]]] = None, standardise_moments: bool = False, ): - """ - Initialize the loss function based on the 'method of moments'. + """Initialize the loss function based on the 'method of moments'. Returns the MSM objective function, i.e. the square difference between the moments of the two time series. @@ -101,8 +99,7 @@ def _validate_covariance_and_calculator( moment_calculator: MomentCalculator, covariance_mat: Union[NDArray[np.float64], str], ) -> None: - """ - Validate the covariance matrix. + """Validate the covariance matrix. Args: moment_calculator: the moment calculator @@ -148,8 +145,7 @@ def _validate_covariance_and_calculator( def compute_loss_1d( self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] ) -> float: - """ - Compute the loss based on the 'method of moments'. + """Compute the loss based on the 'method of moments'. Returns the MSM objective function, i.e. the square difference between the moments of the two time series. diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 0819083c..0fc37c61 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -272,8 +272,7 @@ def plot_sampling_batch_nums( def plot_sampling_interact(saving_folder: Union[str, os.PathLike]) -> None: - """ - Plot the parameter sampled colored according to the sampling method used to sample them. + """Plot the parameter sampled colored according to the sampling method used to sample them. The method allows to interactively choose the batch numbers included in the plot. diff --git a/black_it/samplers/base.py b/black_it/samplers/base.py index 9dbfadc1..05a8866e 100644 --- a/black_it/samplers/base.py +++ b/black_it/samplers/base.py @@ -27,8 +27,7 @@ class BaseSampler(BaseSeedable, ABC): - """ - BaseSampler interface. + """BaseSampler interface. This is the base class for all samplers. """ @@ -39,8 +38,7 @@ def __init__( random_state: Optional[int] = None, max_deduplication_passes: int = 5, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -59,8 +57,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample a number of new parameters fixed by the 'batch_size' attribute. + """Sample a number of new parameters fixed by the 'batch_size' attribute. Args: batch_size: number of samples to collect @@ -78,8 +75,7 @@ def sample( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample from the search space. + """Sample from the search space. Args: search_space: an object containing the details of the parameter search space diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index 52dd0970..f03f02de 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -47,8 +47,7 @@ def __init__( b: float = 1.0, perturbation_range: int = 6, ): - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -84,8 +83,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample from the search space using a genetic algorithm. + """Sample from the search space using a genetic algorithm. Args: batch_size: the number of points to sample diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index f893d622..2044dfe7 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -14,11 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -Implementation the CORS sampler. +"""Implementation the CORS sampler. - Regis, Rommel G., and Christine A. Shoemaker. "Constrained global optimization of expensive black box functions - using radial basis functions." Journal of Global optimization 31.1 (2005): 153-171. +Regis, Rommel G., and Christine A. Shoemaker. "Constrained global optimization of expensive black box functions +using radial basis functions." Journal of Global optimization 31.1 (2005): 153-171. """ from math import factorial @@ -63,8 +62,7 @@ def boxtocube( def rbf(points: NDArray[np.float64], losses: NDArray[np.float64]) -> Callable: - """ - Build RBF-fit for given points (see Holmstrom, 2008 for details). + """Build RBF-fit for given points (see Holmstrom, 2008 for details). Implementation mostly taken from: https://github.com/paulknysh/blackbox/blob/cd8baa0cc344f36b3fddf910ae8037f62009619c/black_box/blackbox.py#L148-L195 @@ -137,8 +135,7 @@ def __init__( random_state: Optional[int] = None, verbose: bool = False, ) -> None: - """ - Initialize the CORS sampler. + """Initialize the CORS sampler. Args: batch_size: the batch size diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 93700e53..207784dc 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -40,8 +40,7 @@ def __str__(self) -> str: class GaussianProcessSampler(MLSurrogateSampler): - """ - This class implements the Gaussian process-based sampler. + """This class implements the Gaussian process-based sampler. In particular, the sampling is based on a Gaussian Process interpolation of the loss function. @@ -58,8 +57,7 @@ def __init__( acquisition: str = "expected_improvement", jitter: float = 0.1, ): - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -83,13 +81,12 @@ def __init__( @staticmethod def _validate_acquisition(acquisition: str) -> None: - """ - Check that the required acquisition is among the supported ones. + """Check that the required acquisition is among the supported ones. Args: acquisition: the acquisition provided as input of the constructor. - Raises + Raises: ValueError: if the provided acquisition type is not among the allowed ones. """ try: @@ -145,8 +142,7 @@ def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 def _predict_mean_std( self, X: NDArray[np.float64] # noqa: N803 ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: - """ - Predict mean and standard deviation of a fitted GP. + """Predict mean and standard deviation of a fitted GP. Args: X: the points on which the predictions should be performed @@ -163,8 +159,7 @@ def _predict_mean_std( def _predict_EI( # noqa: N802 self, X: NDArray[np.float64], jitter: float = 0.1 # noqa: N803 ) -> NDArray[np.float64]: - """ - Compute the Expected Improvement per unit of cost. + """Compute the Expected Improvement per unit of cost. Args: X: the points on which the predictions should be performed @@ -190,8 +185,7 @@ def get_quantiles( m: NDArray[np.float64], s: NDArray[np.float64], ) -> Tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]: - """ - Quantiles of the Gaussian distribution useful to determine the acquisition function values. + """Quantiles of the Gaussian distribution useful to determine the acquisition function values. Args: acquisition_par: parameter of the acquisition function diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index 14d835f0..e811e446 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -30,8 +30,7 @@ class HaltonSampler(BaseSampler): - """ - Halton low discrepancy sequence. + """Halton low discrepancy sequence. This snippet implements the Halton sequence following the generalization of a sequence of *Van der Corput* in n-dimensions. @@ -43,8 +42,7 @@ def __init__( random_state: Optional[int] = None, max_deduplication_passes: int = 5, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -58,8 +56,7 @@ def __init__( self._reset_sequence_index() def _set_random_state(self, random_state: Optional[int]) -> None: - """ - Set the random state (private use). + """Set the random state (private use). For the Halton sampler, it also resets the sequence index. @@ -82,8 +79,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample points using Halton sequence. + """Sample points using Halton sequence. Args: batch_size: the number of samples @@ -102,8 +98,7 @@ def sample_batch( return digitize_data(sampled_points, search_space.param_grid) def _halton(self, nb_samples: int, dims: int) -> NDArray[np.float64]: - """ - Get a Halton sequence. + """Get a Halton sequence. It uses a simple prime number generator, which takes the first `dims` primes. @@ -126,8 +121,7 @@ def _halton(self, nb_samples: int, dims: int) -> NDArray[np.float64]: class _PrimesIterator: - """ - This class implements an iterator that iterates over all primes via unbounded Sieve of Erathosthenes. + """This class implements an iterator that iterates over all primes via unbounded Sieve of Erathosthenes. Adapted from: @@ -170,8 +164,7 @@ def __init__(self) -> None: self._cached_primes: List[int] = [2] def get_n_primes(self, n: int) -> NDArray[np.int64]: - """ - Get the first n primes. + """Get the first n primes. Args: n: the number of primes. @@ -192,8 +185,7 @@ def get_n_primes(self, n: int) -> NDArray[np.int64]: def halton( sample_size: int, bases: NDArray[np.int64], n_start: int ) -> NDArray[np.float64]: - """ - Van der Corput sequence, generalized as to accept a starting point in the sequence. + """Van der Corput sequence, generalized as to accept a starting point in the sequence. Args: sample_size: number of element of the sequence diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index c348f313..5c7cb7e6 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -26,8 +26,7 @@ class ParticleSwarmSampler(BaseSampler): - """ - Implementation of a particle swarm sampler. + """Implementation of a particle swarm sampler. This sampler implements the particle swarm sampling method commonly used in particle swarm optimization (PSO), introduced in: @@ -66,8 +65,7 @@ def __init__( c2: float = 0.1, global_minimum_across_samplers: bool = False, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -148,8 +146,7 @@ def _set_up(self, dims: int) -> None: self._global_best_particle_id = 0 def _get_best_position(self) -> NDArray[np.float64]: - """ - Get the position corresponding to the global optimum the particles should converge to. + """Get the position corresponding to the global optimum the particles should converge to. If _global_minimum_across_samplers is False, then this method returns the current position of the particle that in its history has sampled, so far, the best set of parameters. diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index 9e749dc5..75ca976b 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -37,8 +37,7 @@ def __init__( random_state: Optional[int] = None, max_deduplication_passes: int = 5, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -54,8 +53,7 @@ def __init__( @classmethod def compute_phi(cls, nb_dims: int) -> float: - """ - Get an approximation of phi^nb_dims. + """Get an approximation of phi^nb_dims. Args: nb_dims: the number of dimensions. @@ -72,8 +70,7 @@ def compute_phi(cls, nb_dims: int) -> float: return phi def _set_random_state(self, random_state: Optional[int]) -> None: - """ - Set the random state (private use). + """Set the random state (private use). For the RSequence sampler, it also resets the sequence index and the sequence start. @@ -97,8 +94,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample points using the R-sequence. + """Sample points using the R-sequence. Args: batch_size: the number of samples @@ -117,8 +113,7 @@ def sample_batch( return digitize_data(sampled_points, search_space.param_grid) def _r_sequence(self, nb_samples: int, dims: int) -> NDArray[np.float64]: - """ - Compute the R-sequence (http://extremelearning.com.au/unreasonable-effectiveness-of-quasirandom-sequences/). + """Compute the R-sequence (http://extremelearning.com.au/unreasonable-effectiveness-of-quasirandom-sequences/). Args: nb_samples: number of points to sample diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 4dce88e1..510f8fcb 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -38,8 +38,7 @@ def __init__( criterion: str = "gini", n_classes: int = 10, ) -> None: - """ - Random forest sampling. + """Random forest sampling. Note: this class makes use of sklearn.ensemble.RandomForestClassifier. @@ -114,8 +113,7 @@ def prepare_data_for_classifier( existing_losses: NDArray[np.float64], num_bins: int, ) -> Tuple[NDArray[np.float64], NDArray[np.int64], NDArray[np.float64]]: - """ - Prepare data for the classifier. + """Prepare data for the classifier. Args: existing_points: the parameters already sampled diff --git a/black_it/samplers/random_uniform.py b/black_it/samplers/random_uniform.py index 20722cb8..d050b5f6 100644 --- a/black_it/samplers/random_uniform.py +++ b/black_it/samplers/random_uniform.py @@ -33,8 +33,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample uniformly from the search space. + """Sample uniformly from the search space. Args: batch_size: the number of points to sample diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index 5935c869..2f0d7090 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -29,8 +29,7 @@ class MLSurrogateSampler(BaseSampler): - """ - MLSurrogateSampler interface. + """MLSurrogateSampler interface. This is the base class for all machine learning surrogate samplers. """ @@ -42,8 +41,7 @@ def __init__( max_deduplication_passes: int = 5, candidate_pool_size: Optional[int] = None, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: batch_size: the number of points sampled every time the sampler is called @@ -93,8 +91,7 @@ def sample_batch( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: - """ - Sample a number of new parameters fixed by the 'batch_size' attribute. + """Sample a number of new parameters fixed by the 'batch_size' attribute. Args: batch_size: number of samples to collect diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 9162ef19..6af88e26 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -44,8 +44,7 @@ def __init__( alpha: float = 1.0, n_estimators: int = 10, ) -> None: - """ - Sampler based on a xgboost surrogate model of the loss function. + """Sampler based on a xgboost surrogate model of the loss function. Note: this class makes use of the xgboost library, for more information on the XGBoost parameters visit https://xgboost.readthedocs.io/en/stable/parameter.html. diff --git a/black_it/schedulers/base.py b/black_it/schedulers/base.py index ffd3de76..504c88d6 100644 --- a/black_it/schedulers/base.py +++ b/black_it/schedulers/base.py @@ -27,8 +27,7 @@ class BaseScheduler(BaseSeedable, ABC): - """ - BaseScheduler interface. + """BaseScheduler interface. This is the base class for all schedulers. """ @@ -38,8 +37,7 @@ def __init__( samplers: Sequence[BaseSampler], random_state: Optional[int] = None, ) -> None: - """ - Initialize the scheduler. + """Initialize the scheduler. Args: samplers: the list of samplers to be scheduled @@ -61,8 +59,7 @@ def _set_random_state(self, random_state: Optional[int]) -> None: sampler.random_state = self._get_random_seed() def start_session(self) -> None: - """ - Set up the scheduler for a new session. + """Set up the scheduler for a new session. The default is a no-op. """ @@ -79,8 +76,7 @@ def update( new_losses: NDArray[np.float64], new_simulated_data: NDArray[np.float64], ) -> None: - """ - Update the state of the scheduler after each batch. + """Update the state of the scheduler after each batch. Args: batch_id: the batch id of the . Must be an integer equal or greater than 0. @@ -90,8 +86,7 @@ def update( """ def end_session(self) -> None: - """ - Tear down the scheduler at the end of the session. + """Tear down the scheduler at the end of the session. The default is a no-op. """ diff --git a/black_it/schedulers/rl/agents/base.py b/black_it/schedulers/rl/agents/base.py index c0c646f3..41e377f4 100644 --- a/black_it/schedulers/rl/agents/base.py +++ b/black_it/schedulers/rl/agents/base.py @@ -35,8 +35,7 @@ def learn( reward: SupportsFloat, next_state: ObsType, ) -> None: - """ - Learn from an agent-environment interaction timestep. + """Learn from an agent-environment interaction timestep. Args: state: the observation from where the action was taken @@ -48,8 +47,7 @@ def learn( @abstractmethod def policy(self, state: ObsType) -> ActType: - """ - Get the next action. + """Get the next action. Args: state: the state from where to take the action diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index ad395b20..0a245c40 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -34,8 +34,7 @@ def __init__( initial_values: float = 0.0, random_state: Optional[int] = None, ) -> None: - """ - Initialize the agent object. + """Initialize the agent object. Args: n_actions: the number of actions diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index f5ffc0e1..cc9b980b 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -30,8 +30,7 @@ class RLScheduler(BaseScheduler): - """ - This class implement a RL-based scheduler. + """This class implement a RL-based scheduler. It is agnostic wrt the RL algorithm being used. """ @@ -75,8 +74,7 @@ def _set_random_state(self, random_state: Optional[int]) -> None: def _add_or_get_bootstrap_sampler( cls, samplers: Sequence[BaseSampler] ) -> Tuple[Sequence[BaseSampler], int]: - """ - Add or retrieve a sampler for bootstrapping. + """Add or retrieve a sampler for bootstrapping. Many samplers do require some "bootstrapping" of the calibration process, i.e. a set of parameters whose loss has been already evaluated, e.g. samplers based on ML surrogates or on evolutionary approaches. diff --git a/black_it/schedulers/round_robin.py b/black_it/schedulers/round_robin.py index c686221e..e2767078 100644 --- a/black_it/schedulers/round_robin.py +++ b/black_it/schedulers/round_robin.py @@ -25,8 +25,7 @@ class RoundRobinScheduler(BaseScheduler): - """ - This class implement a simple round-robin sampler scheduler. + """This class implement a simple round-robin sampler scheduler. The round-robin scheduler takes in input a list of samplers [S_0, S_1, ..., S_{n-1}], and, at batch i, it proposes the (i % n)-th sampler. diff --git a/black_it/search_space.py b/black_it/search_space.py index 0e2694e5..9e97c60f 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -31,8 +31,7 @@ def __init__( parameters_precision: Union[NDArray[np.float64], List[float]], verbose: bool, ): - """ - Initialize the SearchSpace object. + """Initialize the SearchSpace object. The values of parameters_bounds and parameters_precision parameters have to satisfy the following constraints, otherwise an exception (subclass @@ -90,8 +89,7 @@ def _check_bounds( parameters_bounds: Union[NDArray[np.float64], List[List[float]]], parameters_precision: Union[NDArray[np.float64], List[float]], ) -> None: - """ - Ensure parameter_bounds and parameter_precision have acceptable values. + """Ensure parameter_bounds and parameter_precision have acceptable values. This is an helper function for SearchSpace.__init__(). diff --git a/black_it/utils/base.py b/black_it/utils/base.py index 7e85c607..fdc3dcf7 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -36,8 +36,7 @@ def _assert( def check_arg(condition: bool, message: str) -> None: - """ - Check a condition over an argument (i.e. raises ValueError in case it is false). + """Check a condition over an argument (i.e. raises ValueError in case it is false). Args: condition: the condition to check @@ -61,8 +60,7 @@ def positive_float(arg: float) -> float: def digitize_data( data: NDArray[np.float64], param_grid: List[NDArray[np.float64]] ) -> NDArray[np.float64]: - """ - Return a discretized version of the input sorted_array. + """Return a discretized version of the input sorted_array. Args: data: the input array to be discretized @@ -105,8 +103,7 @@ def get_closest(sorted_array: NDArray, values: NDArray) -> NDArray: def is_symmetric(a: NDArray[np.float64]) -> bool: - """ - Check if a matrix is symmetric. + """Check if a matrix is symmetric. Args: a: the matrix @@ -118,8 +115,7 @@ def is_symmetric(a: NDArray[np.float64]) -> bool: class NumpyArrayEncoder(JSONEncoder): - """ - Custom JSONEncoder for Numpy arrays. + """Custom JSONEncoder for Numpy arrays. Solution from https://pynative.com/python-serialize-numpy-ndarray-into-json/. """ diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index 630f67f0..fa80a38d 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -31,8 +31,7 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) -> Tuple: - """ - Load calibrator data from a given folder. + """Load calibrator data from a given folder. Args: checkpoint_path: the folder where the data are stored @@ -118,8 +117,7 @@ def save_calibrator_state( batch_num_samp: NDArray[np.int64], method_samp: NDArray[np.int64], ) -> None: - """ - Store the state of the calibrator in a given folder. + """Store the state of the calibrator in a given folder. Args: checkpoint_path: path to the checkpoint diff --git a/black_it/utils/seedable.py b/black_it/utils/seedable.py index 004b7b89..275c84fc 100644 --- a/black_it/utils/seedable.py +++ b/black_it/utils/seedable.py @@ -23,8 +23,7 @@ class BaseSeedable: - """ - BaseSeedable base class. + """BaseSeedable base class. This is the base class for all objects that need to keep a random state. @@ -39,8 +38,7 @@ def __init__( self, random_state: Optional[int] = None, ) -> None: - """ - Initialize the sampler. + """Initialize the sampler. Args: random_state: the internal state of the sampler, fixing this numbers the object (e.g. a calibrator, @@ -78,8 +76,7 @@ def _get_random_seed(self) -> int: def get_random_seed(random_generator: np.random.Generator) -> int: - """ - Get a random seed from a random generator. + """Get a random seed from a random generator. Sample an integer in the range [0, 2^32 - 1]. diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index 6e97b1e0..34f85201 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -168,8 +168,7 @@ class gz_ndarray(NDArray): # noqa: N801 def load_calibrator_state( checkpoint_path: PathLike, ) -> Tuple: - """ - Load the calibration state. + """Load the calibration state. Checks that the schema version stored in checkpoint_path has the same value of SCHEMA_VERSION. Currently there are no conversion mechanisms in place, so @@ -276,8 +275,7 @@ def save_calibrator_state( batch_num_samp: NDArray[np.int64], method_samp: NDArray[np.int64], ) -> None: - """ - Save a calibration state. + """Save a calibration state. Args: checkpoint_path: path to the checkpoint @@ -356,8 +354,7 @@ def save_calibrator_state( def npndarray_to_sqlite_binary(numpy_array: NDArray) -> sqlite3.Binary: - """ - Serialize an NumPy NDArray to SQLite binary. + """Serialize an NumPy NDArray to SQLite binary. Taken from: http://stackoverflow.com/a/31312102/190597 (SoulNibbler) @@ -374,8 +371,7 @@ def npndarray_to_sqlite_binary(numpy_array: NDArray) -> sqlite3.Binary: def sqlite_binary_to_npndarray(sqlite_binary: bytes) -> NDArray: - """ - Deserialize a NumPy NDArray from SQLite binary. + """Deserialize a NumPy NDArray from SQLite binary. Taken from: http://stackoverflow.com/a/31312102/190597 (SoulNibbler) @@ -392,8 +388,7 @@ def sqlite_binary_to_npndarray(sqlite_binary: bytes) -> NDArray: def gz_ndarray_to_gzipped_sqlite_binary(numpy_array: gz_ndarray) -> sqlite3.Binary: - """ - Serialize a NumPy NDArray to a gzipped SQLite binary. + """Serialize a NumPy NDArray to a gzipped SQLite binary. The custom gz_ndarray type is a convenience type used to trigger the sqlite3 adapter mechanism. If a user coerces a NDArray to gz_ndarray, it @@ -414,8 +409,7 @@ def gz_ndarray_to_gzipped_sqlite_binary(numpy_array: gz_ndarray) -> sqlite3.Bina def gzipped_sqlite_binary_to_npndarray(sqlite_binary: bytes) -> NDArray: - """ - Deserialize a NumPy NDArray from a gzipped SQLite binary. + """Deserialize a NumPy NDArray from a gzipped SQLite binary. Args: sqlite_binary: the gzipped SQLite binary version of the array. diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index b5730ca9..8f94e016 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -26,8 +26,7 @@ def get_mom_ts(time_series: NDArray[np.float64]) -> NDArray[np.float64]: - """ - Compute specific moments from a time series. + """Compute specific moments from a time series. Args: time_series: the time series @@ -89,8 +88,7 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: def hp_filter( time_series: NDArray[np.float64], lamb: float = 1600 ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: - """ - Apply the HP filter to a time series. + """Apply the HP filter to a time series. Args: time_series: the one dimensional series to be filtered @@ -116,8 +114,7 @@ def hp_filter( def hp_cycle_lamb1600_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: - """ - Get the cycle part of the HP filter to a time series using lamb = 1600. + """Get the cycle part of the HP filter to a time series using lamb = 1600. Args: time_series: the one dimensional series to be filtered @@ -129,8 +126,7 @@ def hp_cycle_lamb1600_filter(time_series: NDArray[np.float64]) -> NDArray[np.flo def log_and_hp_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: - """ - Take the log of the values and remove the HP trend from them. + """Take the log of the values and remove the HP trend from them. Args: time_series: the one dimensional series to be filtered @@ -145,8 +141,7 @@ def log_and_hp_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: def diff_log_demean_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: - """ - Take the difference of log of the values and remove the mean from them. + """Take the difference of log of the values and remove the mean from them. Args: time_series: the one dimensional series to be filtered diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index e1cd06cf..ea6b8397 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -Mesa-based Boltzmann Wealth Model. +"""Mesa-based Boltzmann Wealth Model. Taken from the 'Mesa' documentation: diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index dce7d8c5..3a6bbd42 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -14,20 +14,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +"""Implementation of the model in (Brock and Hommes, 1998).""" + from typing import Sequence import numpy as np from scipy.special import softmax -# **** -# Brock and Hommes 1998 -# **** - def BH2(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 - """ - Model from Brock and Hommes 1998. - 4.1.2. Fundamentalists versus trend chasers + """Model from Brock and Hommes 1998. 4.1.2. Fundamentalists versus trend chasers. theta structure: [g1, b1, g2, b2] diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 1f2abfaf..cb8bf6db 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +"""A simple model of a wildfire on a 2D grid.""" + import subprocess import numpy as np @@ -38,7 +40,6 @@ def forest_fire(theta, N, seed=0): # noqa: N803 Returns: An array containing the fraction of trees burned at each time step """ - density = theta[0] n = N - 1 diff --git a/examples/models/simple_models.py b/examples/models/simple_models.py index 006ff54d..9454ac8b 100644 --- a/examples/models/simple_models.py +++ b/examples/models/simple_models.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -A module with a series of simple non-ABM models. +"""A module with a series of simple non-ABM models. List of models: - Normal0 @@ -299,8 +298,7 @@ def ARMAARCH2( # noqa: N802 def ARMAARCH4( # noqa: N802 theta: Sequence[float], N: int, seed: int # noqa: N803 ) -> NDArray[np.float64]: - """ - ARMA(2,2) ARCH(2) model. + """ARMA(2,2) ARCH(2) model. DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] @@ -330,8 +328,7 @@ def ARMAARCH4( # noqa: N802 def ARMAARCH6( # noqa: N802 theta: Sequence[float], N: int, seed: int # noqa: N803 ) -> NDArray[np.float64]: - """ - ARMA(2,2) ARCH(2). + """ARMA(2,2) ARCH(2). DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] @@ -363,8 +360,7 @@ def ARMAARCH6( # noqa: N802 def ARMAARCH4v2( # noqa: N802 theta: Sequence[float], N: int, seed: int # noqa: N803 ) -> NDArray[np.float64]: - """ - ARMA(2,2) ARCH(2). + """ARMA(2,2) ARCH(2). DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] @@ -392,8 +388,7 @@ def ARMAARCH4v2( # noqa: N802 def RWSB1( # noqa: N802 theta: Sequence[int], N: int, seed: int # noqa: N803 ) -> NDArray[np.float64]: - """ - RW with structural break. + """RW with structural break. DONOVAN MODEL 3 - PARAM SET 1 [tau] [*tau,sigma1,sigma2,drift1,drift2] [*700,0.1,0.2,1,2] @@ -420,8 +415,7 @@ def RWSB1( # noqa: N802 def RWSB2( # noqa: N802 theta: Sequence[float], N: int, seed: int # noqa: N803 ) -> NDArray[np.float64]: - """ - RW with structural break. + """RW with structural break. DONOVAN MODEL 3 - PARAM SET 2 [sigma1,sigma2] [tau,*sigma1,*sigma2,drift1,drift2] [700,*0.1,*0.2,1,2] diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 064ff06a..0671de4c 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -Executes the C++ simulator with chosen command line parameters, and returns the simulation output as a Python list. +"""Executes the C++ simulator with chosen command line parameters, and returns the simulation output as a Python list. The library can also be invoked directly from command line on a custom simulator command. diff --git a/scripts/__init__.py b/scripts/__init__.py index e69de29b..5c7e4874 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Package for scripts.""" diff --git a/scripts/check_copyright.py b/scripts/check_copyright.py index 2c8e228c..84be3d72 100755 --- a/scripts/check_copyright.py +++ b/scripts/check_copyright.py @@ -15,8 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" -This script checks that all the Python files of the repository have the copyright notice. +"""This script checks that all the Python files of the repository have the copyright notice. In particular: - (optional) the Python shebang diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index 0be32b40..fdd0ae67 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -26,8 +26,7 @@ def is_sorted(array: np.ndarray, reverse: bool = False) -> bool: - """ - Test that a unidimensional array is sorted. + """Test that a unidimensional array is sorted. Args: array: the array diff --git a/tests/test_plot/test_plot_results.py b/tests/test_plot/test_plot_results.py index 80612073..1fca17b0 100644 --- a/tests/test_plot/test_plot_results.py +++ b/tests/test_plot/test_plot_results.py @@ -135,8 +135,7 @@ def test_plot_losses_method_num_raises_error_if_method_num_not_known( @patch("ipywidgets.widgets.interaction.interact") def test_plot_losses_interact(*_mocks: Any) -> None: - """ - Test 'plot_losses_interact' function. + """Test 'plot_losses_interact' function. Note that this function does not test the interaction with the plots, as the 'ipywidgets.widgets.interaction.interact' function is mocked. @@ -149,8 +148,7 @@ def test_plot_losses_interact(*_mocks: Any) -> None: @patch("ipywidgets.widgets.interaction.interact") def test_plot_sampling_interact(*_mocks: Any) -> None: # noqa - """ - Test 'plot_sampling_interact' function. + """Test 'plot_sampling_interact' function. Note that this function does not test the interaction with the plots, as the 'ipywidgets.widgets.interaction.interact' function is mocked. diff --git a/tests/test_samplers/test_base.py b/tests/test_samplers/test_base.py index d8df1104..543551f3 100644 --- a/tests/test_samplers/test_base.py +++ b/tests/test_samplers/test_base.py @@ -40,8 +40,7 @@ class TestSetRandomState: """Test 'BaseSampler.random_state' setter.""" class MyCustomSampler(BaseSampler): - """ - Custom sampler for testing purposes. + """Custom sampler for testing purposes. Test that two consecutive samples with the same give the same result. """ diff --git a/tests/utils/base.py b/tests/utils/base.py index 4dcf954f..43cece6c 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -40,8 +40,7 @@ class PopenResult: def run_process( command: List[str], timeout: float = DEFAULT_SUBPROCESS_TIMEOUT ) -> PopenResult: - """ - Run a process, and wait for it to stop. + """Run a process, and wait for it to stop. Args: command: the command to run. @@ -78,8 +77,7 @@ def run_process( def requires_docker(pytest_func_or_cls: Callable) -> Callable: - """ - Wrap a Pytest function that requires docker. + """Wrap a Pytest function that requires docker. Args: pytest_func_or_cls: the pytest function/class to wrap. @@ -91,8 +89,7 @@ def requires_docker(pytest_func_or_cls: Callable) -> Callable: def requires_binary(binary_name: str) -> Callable: - """ - Decorate a pytest class or method to skip test if a binary is not installed locally. + """Decorate a pytest class or method to skip test if a binary is not installed locally. Args: binary_name: the binary name @@ -110,8 +107,7 @@ def action() -> None: def pytest_decorator_factory(action: Callable) -> Callable: - """ - Wrap a Pytest function/method/class to include the running of an action. + """Wrap a Pytest function/method/class to include the running of an action. For methods/functions 'f', the wrapper function looks like: @@ -127,7 +123,7 @@ def wrapper(*args, **kwargs): If 'c' does not have setup_class, setup_class = action. Else, new_setup_class is equal to wrapper above (with f = setup_class) - Args: + Args: action: the preamble function to call before the test or the setup_class method Returns: @@ -135,8 +131,7 @@ def wrapper(*args, **kwargs): """ def decorator(pytest_func_or_cls: Union[Callable, Type]) -> Callable: - """ - Implement the decorator. + """Implement the decorator. Args: pytest_func_or_cls: a Pytest function or class. diff --git a/tests/utils/docs.py b/tests/utils/docs.py index 33c958da..2d8b6a48 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -27,8 +27,7 @@ def code_block_filter(block_dict: Dict, language: Optional[str] = None) -> bool: - """ - Check Mistletoe block is a code block. + """Check Mistletoe block is a code block. Args: block_dict: the block dictionary describing a Mistletoe document node @@ -86,8 +85,7 @@ class BasePythonMarkdownDocs(BaseTestMarkdownDocs): @classmethod def setup_class(cls) -> None: - """ - Set up class. + """Set up class. It sets the initial value of locals and globals. """ diff --git a/tests/utils/strategies.py b/tests/utils/strategies.py index 0d2cb480..f383df11 100644 --- a/tests/utils/strategies.py +++ b/tests/utils/strategies.py @@ -29,8 +29,7 @@ def time_series(max_length: int = MAX_TIME_SERIES_LENGTH) -> st.SearchStrategy: - """ - Return a strategy to generate time_series. + """Return a strategy to generate time_series. Args: max_length: the max length of the time series. @@ -50,8 +49,7 @@ def discretize_args( max_length: int = MAX_TIME_SERIES_LENGTH, max_nb_values: int = MAX_NB_VALUES, ) -> Tuple: - """ - Return the strategies for the arguments of the 'discretize' function. + """Return the strategies for the arguments of the 'discretize' function. Args: draw: the Hypothesis draw function @@ -74,8 +72,7 @@ def get_words_args( max_length: int = MAX_TIME_SERIES_LENGTH, max_word_length: int = MAX_WORD_LENGTH, ) -> Tuple: - """ - Return the strategies for the arguments of the 'discretize' function. + """Return the strategies for the arguments of the 'discretize' function. Args: draw: the Hypothesis draw function From 65ed299bcef7110a7e955edde7b7e158eab1f46a Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 13:46:22 +0200 Subject: [PATCH 16/56] lint: fix pyupgrade errors To reproduce the errors: ``` ruff check --select "UP" black_it tests examples scripts ``` All the errors have been fixed using the '--fix' flag. --- .ruff.toml | 2 +- black_it/utils/json_pandas_checkpointing.py | 2 +- examples/models/forest_fire/forest_fire.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index a12092d8..69cbcadd 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D"] +select = ["F","W","E","C90","I","N","D","UP"] ignore = ["E203"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index fa80a38d..bdff3ec3 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -41,7 +41,7 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - all the data needed to reconstruct the calibrator state """ checkpoint_path = Path(checkpoint_path) - with open(checkpoint_path / "calibration_params.json", "r") as f: + with open(checkpoint_path / "calibration_params.json") as f: cp = json.load(f) cr = pd.read_csv(checkpoint_path / "calibration_results.csv") diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index cb8bf6db..24ea0e47 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -47,9 +47,7 @@ def forest_fire(theta, N, seed=0): # noqa: N803 xsize = 30 ysize = 30 - command = "julia forest_fire_julia.jl {} {} {} {} {}".format( - density, n, xsize, ysize, seed - ) + command = f"julia forest_fire_julia.jl {density} {n} {xsize} {ysize} {seed}" res = subprocess.run( command.split(), From 128dcfaee4c418d6e8ea47be90c573069c255552 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 15:45:07 +0200 Subject: [PATCH 17/56] lint: fix flake8-annotations errors To reproduce the errors: ``` ruff check --select "ANN" --ignore "ANN101,ANN102" black_it tests examples scripts ``` Some errors were fixed with --fix, while others have been fixed by hand. Errors ANN101 (https://beta.ruff.rs/docs/rules/missing-type-self/) and ANN102 (https://beta.ruff.rs/docs/rules/missing-type-cls/) have been ignored since we considered not strictly necessary. --- .ruff.toml | 4 ++-- black_it/calibrator.py | 2 +- black_it/loss_functions/base.py | 2 +- black_it/loss_functions/likelihood.py | 2 +- black_it/loss_functions/msm.py | 2 +- black_it/samplers/best_batch.py | 2 +- black_it/samplers/gaussian_process.py | 2 +- black_it/schedulers/base.py | 4 ++-- black_it/schedulers/round_robin.py | 5 +++-- black_it/search_space.py | 2 +- black_it/utils/base.py | 4 ++-- examples/docker-sir.py | 2 +- examples/models/economics/brock_hommes.py | 11 ++++++++--- examples/models/forest_fire/forest_fire.py | 6 +++++- examples/models/sir/simlib.py | 2 +- examples/models/sir/sir_docker.py | 8 ++++++-- examples/models/sir/sir_python.py | 8 ++++++-- setup.cfg | 3 +++ tests/test_calibrator.py | 6 ++++-- tests/test_plot/test_plot_results.py | 9 ++++----- tests/test_utils/test_sqlite3_checkpointing.py | 3 +-- tests/utils/base.py | 4 +++- tests/utils/docs.py | 7 ++++--- 23 files changed, 62 insertions(+), 38 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 69cbcadd..5e2e15ad 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,5 +1,5 @@ -select = ["F","W","E","C90","I","N","D","UP"] -ignore = ["E203"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN"] +ignore = ["ANN101", "ANN102", "E203"] # Allow autofix for all enabled rules (when `--fix`) is provided. #fixable = ["ALL"] diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 30ca0653..898c95b6 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -62,7 +62,7 @@ def __init__( saving_folder: Optional[str] = None, random_state: Optional[int] = None, n_jobs: Optional[int] = None, - ): + ) -> None: """Initialize the Calibrator object. It must be initialized with details on the parameters to explore, diff --git a/black_it/loss_functions/base.py b/black_it/loss_functions/base.py index c239b25c..891a4696 100644 --- a/black_it/loss_functions/base.py +++ b/black_it/loss_functions/base.py @@ -35,7 +35,7 @@ def __init__( self, coordinate_weights: Optional[NDArray] = None, coordinate_filters: Optional[List[Optional[Callable]]] = None, - ): + ) -> None: """Initialize the loss function. Args: diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 7d42f44c..976a6ca8 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -43,7 +43,7 @@ def __init__( coordinate_weights: Optional[NDArray] = None, coordinate_filters: Optional[List[Optional[Callable]]] = None, h: Union[str, float] = "silverman", - ): + ) -> None: """Initialize the loss function. Args: diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 80ded962..934ea5f5 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -57,7 +57,7 @@ def __init__( moment_calculator: MomentCalculator = get_mom_ts_1d, coordinate_filters: Optional[List[Optional[Callable]]] = None, standardise_moments: bool = False, - ): + ) -> None: """Initialize the loss function based on the 'method of moments'. Returns the MSM objective function, i.e. the square difference between diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index f03f02de..5d1568dc 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -46,7 +46,7 @@ def __init__( a: float = 3.0, b: float = 1.0, perturbation_range: int = 6, - ): + ) -> None: """Initialize the sampler. Args: diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 207784dc..f21b91d3 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -56,7 +56,7 @@ def __init__( optimize_restarts: int = 5, acquisition: str = "expected_improvement", jitter: float = 0.1, - ): + ) -> None: """Initialize the sampler. Args: diff --git a/black_it/schedulers/base.py b/black_it/schedulers/base.py index 504c88d6..0fb7cbb3 100644 --- a/black_it/schedulers/base.py +++ b/black_it/schedulers/base.py @@ -17,7 +17,7 @@ """This module defines the 'BaseScheduler' base class.""" import contextlib from abc import ABC, abstractmethod -from typing import Any, Optional, Sequence, Tuple +from typing import Generator, Optional, Sequence, Tuple import numpy as np from numpy.typing import NDArray @@ -92,7 +92,7 @@ def end_session(self) -> None: """ @contextlib.contextmanager - def session(self) -> Any: + def session(self) -> Generator[None, None, None]: """Start the session of the scheduler with a context manager.""" self.start_session() yield diff --git a/black_it/schedulers/round_robin.py b/black_it/schedulers/round_robin.py index e2767078..e463e45a 100644 --- a/black_it/schedulers/round_robin.py +++ b/black_it/schedulers/round_robin.py @@ -15,7 +15,6 @@ # along with this program. If not, see . """This module implements the 'RoundRobinScheduler' scheduler.""" -from typing import Any import numpy as np from numpy._typing import NDArray @@ -31,7 +30,9 @@ class RoundRobinScheduler(BaseScheduler): and, at batch i, it proposes the (i % n)-th sampler. """ - def __init__(self, *args: Any, **kwargs: Any) -> None: + def __init__( # type: ignore[no-untyped-def] + self, *args, **kwargs # noqa: ANN002,ANN003 + ) -> None: """Initialize the round-robin scheduler.""" super().__init__(*args, **kwargs) diff --git a/black_it/search_space.py b/black_it/search_space.py index 9e97c60f..cc10adfc 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -30,7 +30,7 @@ def __init__( parameters_bounds: Union[NDArray[np.float64], List[List[float]]], parameters_precision: Union[NDArray[np.float64], List[float]], verbose: bool, - ): + ) -> None: """Initialize the SearchSpace object. The values of parameters_bounds and parameters_precision parameters have diff --git a/black_it/utils/base.py b/black_it/utils/base.py index fdc3dcf7..14c8f917 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -45,7 +45,7 @@ def check_arg(condition: bool, message: str) -> None: _assert(condition, message, exception_class=ValueError) -def ensure_float(arg: Any) -> None: +def ensure_float(arg: Any) -> None: # noqa: ANN401 """Check that the argument is a float.""" check_arg(isinstance(arg, (float, np.float64)), f"expected a float, got {arg}") @@ -120,7 +120,7 @@ class NumpyArrayEncoder(JSONEncoder): Solution from https://pynative.com/python-serialize-numpy-ndarray-into-json/. """ - def default(self, o: Any) -> Any: + def default(self, o: Any) -> Any: # noqa: ANN401 """Implement custom JSON serialization for NumPy NDArrays.""" if isinstance(o, np.ndarray): return o.tolist() diff --git a/examples/docker-sir.py b/examples/docker-sir.py index 4a3620d9..9124b4ef 100755 --- a/examples/docker-sir.py +++ b/examples/docker-sir.py @@ -25,7 +25,7 @@ # generate a synthetic dataset to test the calibrator N = 10 seed = 10000 - real_data = SIR(true_params, N, seed) + real_data = SIR(true_params, N, seed) # type: ignore[arg-type] # initialize a loss function loss = MinkowskiLoss() diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index 3a6bbd42..c0f3d077 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -16,13 +16,16 @@ """Implementation of the model in (Brock and Hommes, 1998).""" -from typing import Sequence +from typing import Optional, Sequence import numpy as np +from numpy.typing import NDArray from scipy.special import softmax -def BH2(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 +def BH2( # noqa: N802 + theta: Sequence[float], N: int, seed: Optional[int] # noqa: N803 +) -> NDArray: """Model from Brock and Hommes 1998. 4.1.2. Fundamentalists versus trend chasers. theta structure: @@ -80,7 +83,9 @@ def BH2(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 # -def BH4(theta: Sequence[float], N: int, seed: int): # noqa: N802, N803 +def BH4( # noqa: N802 + theta: Sequence[float], N: int, seed: Optional[int] # noqa: N803 +) -> NDArray: """Model from Brock and Hommes 1998. 4.3 Four belief types: Fundamentalists versus trend versus bias diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 24ea0e47..cfeab9de 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -17,11 +17,15 @@ """A simple model of a wildfire on a 2D grid.""" import subprocess +from typing import Optional import numpy as np +from numpy.typing import NDArray -def forest_fire(theta, N, seed=0): # noqa: N803 +def forest_fire( + theta: NDArray, N: int, seed: Optional[int] = 0 # noqa: N803 +) -> NDArray: """A simple model of a wildfire on a 2D grid. The model is taken from this example from Agent.jl diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 0671de4c..1ab7a137 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -159,7 +159,7 @@ def _execute_simulator_subprocess(args: List[str]) -> List[Dict[str, int]]: return simulation_output -def run_single_simulation(): +def run_single_simulation() -> None: """Test function that runs the simulator with a fixed set of parameters.""" # example of formally correct values for simulation parameters sim_params = { diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index 6caae6c8..a2854fa3 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -15,13 +15,15 @@ # along with this program. If not, see . """SIR models written in C and run in Docker containers.""" +from typing import Optional import numpy as np +from numpy.typing import NDArray from . import simlib -def SIR(theta, N, seed): # noqa: N802, N803 +def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, N803 """SIR_docker. C++ SIR model run in Docker container. @@ -42,7 +44,9 @@ def SIR(theta, N, seed): # noqa: N802, N803 return ret -def SIR_w_breaks(theta, N, seed=None): # noqa: N802, N803 +def SIR_w_breaks( # noqa: N802 + theta: NDArray, N: int, seed: Optional[int] = None # noqa: N803 +) -> NDArray: """SIR_docker_w_breaks.""" breaktime = int(theta[0]) diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index c94160bc..2631a9a6 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -15,14 +15,16 @@ # along with this program. If not, see . """SIR models written in Python.""" +from typing import Optional import ndlib.models.epidemics as ep import networkx as nx import numpy as np from ndlib.models import ModelConfig +from numpy.typing import NDArray -def SIR(theta, N, seed): # noqa: N802, N803 +def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, N803 """SIR model. 0 theta = [#LOC CONNECTIONS, @@ -68,7 +70,9 @@ def SIR(theta, N, seed): # noqa: N802, N803 return output_np -def SIR_w_breaks(theta, N, seed): # noqa: N802, N803 +def SIR_w_breaks( # noqa: N802 + theta: NDArray, N: int, seed: Optional[int] # noqa: N803 +) -> NDArray: """SIR model with structural breaks. 0 theta = [#LOC CONNECTIONS, diff --git a/setup.cfg b/setup.cfg index 23454144..5323a4f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,9 @@ ignore_missing_imports = True [mypy-pytest] ignore_missing_imports = True +[mypy-_pytest.*] +ignore_missing_imports = True + # Per-module options for examples dir: [mypy-mesa.*] diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index 65585398..d9081b2b 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -17,11 +17,11 @@ """This module contains tests for the Calibrator.calibrate method.""" import sys from pathlib import Path -from typing import Any from unittest.mock import MagicMock, patch import numpy as np import pytest +from _pytest.capture import CaptureFixture from numpy.typing import NDArray from black_it.calibrator import Calibrator @@ -182,7 +182,9 @@ def test_calibrator_calibrate(self, n_jobs: int) -> None: assert np.allclose(params, self.expected_params) assert np.allclose(losses, self.expected_losses) - def test_calibrator_with_check_convergence(self, capsys: Any) -> None: + def test_calibrator_with_check_convergence( + self, capsys: CaptureFixture[str] + ) -> None: """Test the Calibrator.calibrate method with convergence check.""" cal = Calibrator( samplers=[ diff --git a/tests/test_plot/test_plot_results.py b/tests/test_plot/test_plot_results.py index 1fca17b0..1fe6d300 100644 --- a/tests/test_plot/test_plot_results.py +++ b/tests/test_plot/test_plot_results.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This test module contains tests for the plot_results module.""" -from typing import Any -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest @@ -123,7 +122,7 @@ def teardown(self) -> None: @patch("pandas.read_csv", return_value={"method_samp": set()}) def test_plot_losses_method_num_raises_error_if_method_num_not_known( - *_mocks: Any, + *_mocks: MagicMock, ) -> None: """Test that 'plot_losses_method_num' raises error if the method num is not known.""" method_num = 1 @@ -134,7 +133,7 @@ def test_plot_losses_method_num_raises_error_if_method_num_not_known( @patch("ipywidgets.widgets.interaction.interact") -def test_plot_losses_interact(*_mocks: Any) -> None: +def test_plot_losses_interact(*_mocks: MagicMock) -> None: """Test 'plot_losses_interact' function. Note that this function does not test the interaction with the plots, @@ -147,7 +146,7 @@ def test_plot_losses_interact(*_mocks: Any) -> None: @patch("ipywidgets.widgets.interaction.interact") -def test_plot_sampling_interact(*_mocks: Any) -> None: # noqa +def test_plot_sampling_interact(*_mocks: MagicMock) -> None: """Test 'plot_sampling_interact' function. Note that this function does not test the interaction with the plots, diff --git a/tests/test_utils/test_sqlite3_checkpointing.py b/tests/test_utils/test_sqlite3_checkpointing.py index 16859478..85ed3d86 100644 --- a/tests/test_utils/test_sqlite3_checkpointing.py +++ b/tests/test_utils/test_sqlite3_checkpointing.py @@ -16,7 +16,6 @@ """This module contains tests for the SQLite3-based checkpointing.""" from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any from unittest.mock import MagicMock, patch import numpy as np @@ -115,7 +114,7 @@ def test_sqlite3_checkpointing_loading_when_code_state_version_different( @patch.object(black_it.utils.sqlite3_checkpointing, "Path") @patch("sqlite3.connect") def test_sqlite3_checkpointing_saving_when_error_occurs( - connect_mock: MagicMock, *_mocks: Any + connect_mock: MagicMock, *_mocks: MagicMock ) -> None: """Test saving function when an error occurs.""" error_message = "error" diff --git a/tests/utils/base.py b/tests/utils/base.py index 43cece6c..faaa0b41 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -141,7 +141,9 @@ def decorator(pytest_func_or_cls: Union[Callable, Type]) -> Callable: """ @wraps(pytest_func_or_cls) - def wrapper(*args, **kwargs): # type: ignore + def wrapper( # type: ignore[no-untyped-def] # noqa: ANN202 + *args, **kwargs # noqa: ANN002,ANN003 + ): action() return pytest_func_or_cls(*args, **kwargs) diff --git a/tests/utils/docs.py b/tests/utils/docs.py index 2d8b6a48..8866fab2 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -18,7 +18,8 @@ import json from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional +from unittest.mock import MagicMock import mistletoe from mistletoe.ast_renderer import ASTRenderer @@ -93,10 +94,10 @@ def setup_class(cls) -> None: cls.locals = {} cls.globals = {} - def _assert(self, locals_: Dict, *mocks: Any) -> None: + def _assert(self, locals_: Dict, *mocks: MagicMock) -> None: """Do assertions after Python code execution.""" - def test_python_blocks(self, *mocks: Any) -> None: + def test_python_blocks(self, *mocks: MagicMock) -> None: """Run Python code block in sequence.""" python_blocks = self.python_blocks From 412dc9b99c85d68bc0525cc5ffe01cc7aeef09f7 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 16:02:23 +0200 Subject: [PATCH 18/56] lint: ignore 'S' checks in Ruff config This is because bandit's rules support in Ruff is still not complete at the time of writing. https://github.com/astral-sh/ruff/issues/1646 --- .ruff.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 5e2e15ad..bfae5700 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,5 +1,5 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN"] -ignore = ["ANN101", "ANN102", "E203"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC"] +ignore = ["ANN101", "ANN102", "E203", "S"] # Allow autofix for all enabled rules (when `--fix`) is provided. #fixable = ["ALL"] From 4036e7dbb65b98166e09c1ee329ab69f429e39b1 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 25 Aug 2023 16:08:40 +0200 Subject: [PATCH 19/56] lint: fix flake8-blind-except errors To reproduce the errors: ``` ruff check --select "BLE" black_it tests examples scripts ``` --- .ruff.toml | 2 +- tests/utils/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index bfae5700..bd84ec4f 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE"] ignore = ["ANN101", "ANN102", "E203", "S"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/tests/utils/base.py b/tests/utils/base.py index faaa0b41..aba75cbf 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -59,7 +59,7 @@ def run_process( ) try: stdout_bytes, stderr_bytes = process.communicate(timeout=timeout) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # clean up process # first, try to stop it gracefully process.send_signal(signal.SIGTERM) From 3b9f9643e4abcf6ccc9143323535c79f46d8219d Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Mon, 28 Aug 2023 10:23:11 +0200 Subject: [PATCH 20/56] lint: ignore flake8-boolean-trap errors To reproduce the errors: ``` ruff check --select "FBT" black_it tests examples scripts\n ``` In fact, we added "FBT" in the ignore list. The reason is that fixing them would have required changing the signature of many functions, which could have broken user code in many ways. We decided that such errors do not have the highest priority for the improvmement of the quality of the software. We fixed a couple of errors that do not have an impact on the user code since they deal with private/test functions: black_it/utils/time_series.py:83:32: FBT003 Boolean positional value in function call tests/test_losses/test_gsl.py:28:34: FBT001 Boolean-typed positional argument in function definition tests/test_losses/test_gsl.py:28:34: FBT002 Boolean default positional argument in function definition --- .ruff.toml | 2 +- black_it/utils/time_series.py | 2 +- tests/test_losses/test_gsl.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index bd84ec4f..a6e47702 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,5 +1,5 @@ select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE"] -ignore = ["ANN101", "ANN102", "E203", "S"] +ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. #fixable = ["ALL"] diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index 8f94e016..280e24bd 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -80,7 +80,7 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: avg_vec_mom[16] = ts_diff_acf[4] avg_vec_mom[17] = ts_diff_acf[5] - np.nan_to_num(avg_vec_mom, False) + np.nan_to_num(avg_vec_mom, copy=False) return avg_vec_mom diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index fdd0ae67..3d1ee9ed 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -25,7 +25,7 @@ from tests.utils.strategies import discretize_args, get_words_args -def is_sorted(array: np.ndarray, reverse: bool = False) -> bool: +def is_sorted(array: np.ndarray, *, reverse: bool = False) -> bool: """Test that a unidimensional array is sorted. Args: From 62cc667f9187231d0e8f0cdc5a792021525cc06a Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Mon, 28 Aug 2023 17:30:17 +0200 Subject: [PATCH 21/56] lint: fix flake8-builtins errors To reproduce the errors: ``` ruff check --select "A" black_it tests examples scripts ``` tests/utils/docs.py:84:5: A003 Class attribute `locals` is shadowing a Python builtin tests/utils/docs.py:85:5: A003 Class attribute `globals` is shadowing a Python builtin The simple solution was to rename in a proper way the variables 'locals' and 'globals'. --- .ruff.toml | 2 +- tests/utils/docs.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index a6e47702..0186a672 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/tests/utils/docs.py b/tests/utils/docs.py index 8866fab2..d4493a87 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -81,8 +81,8 @@ def setup_class(cls) -> None: class BasePythonMarkdownDocs(BaseTestMarkdownDocs): """Test Markdown documentation by running Python snippets in sequence.""" - locals: Dict - globals: Dict + locals_dict: Dict + globals_dict: Dict @classmethod def setup_class(cls) -> None: @@ -91,8 +91,8 @@ def setup_class(cls) -> None: It sets the initial value of locals and globals. """ super().setup_class() - cls.locals = {} - cls.globals = {} + cls.locals_dict = {} + cls.globals_dict = {} def _assert(self, locals_: Dict, *mocks: MagicMock) -> None: """Do assertions after Python code execution.""" @@ -101,7 +101,7 @@ def test_python_blocks(self, *mocks: MagicMock) -> None: """Run Python code block in sequence.""" python_blocks = self.python_blocks - globals_, locals_ = self.globals, self.locals + globals_, locals_ = self.globals_dict, self.locals_dict for python_code_block in python_blocks: exec(python_code_block, globals_, locals_) # nosec self._assert(locals_, *mocks) From 33d968c587f53624a9347fe6546642286270ad69 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 09:13:37 +0200 Subject: [PATCH 22/56] lint: fix flake8-commas errors To reproduce the errors: ``` ruff check --select "COM" black_it tests examples scripts ``` The error were fixed using the '--fix' flag. --- .ruff.toml | 2 +- black_it/calibrator.py | 29 ++++--- black_it/loss_functions/base.py | 10 ++- black_it/loss_functions/fourier.py | 4 +- black_it/loss_functions/gsl_div.py | 15 +++- black_it/loss_functions/likelihood.py | 12 ++- black_it/loss_functions/minkowski.py | 4 +- black_it/loss_functions/msm.py | 22 +++-- black_it/plot/plot_descriptive_statistics.py | 3 +- black_it/plot/plot_results.py | 15 +++- black_it/samplers/base.py | 15 +++- black_it/samplers/best_batch.py | 15 ++-- black_it/samplers/cors.py | 14 ++-- black_it/samplers/gaussian_process.py | 14 +++- black_it/samplers/halton.py | 14 +++- black_it/samplers/particle_swarm.py | 14 ++-- black_it/samplers/r_sequence.py | 8 +- black_it/samplers/random_forest.py | 5 +- black_it/samplers/surrogate.py | 13 ++- black_it/samplers/xgboost.py | 5 +- black_it/schedulers/rl/envs/base.py | 3 +- black_it/schedulers/rl/rl_scheduler.py | 7 +- black_it/schedulers/round_robin.py | 4 +- black_it/search_space.py | 38 ++++++--- black_it/utils/base.py | 3 +- black_it/utils/json_pandas_checkpointing.py | 5 +- black_it/utils/sqlite3_checkpointing.py | 10 ++- black_it/utils/time_series.py | 3 +- examples/models/economics/boltzmann_wealth.py | 7 +- examples/models/economics/brock_hommes.py | 8 +- examples/models/forest_fire/forest_fire.py | 4 +- examples/models/simple_models.py | 82 ++++++++++++++----- examples/models/sir/simlib.py | 13 +-- examples/models/sir/sir_docker.py | 4 +- examples/models/sir/sir_python.py | 4 +- tests/test_calibrator.py | 17 ++-- tests/test_docs/test_readme.py | 2 +- tests/test_examples/base.py | 3 +- tests/test_examples/test_brock_hommes.py | 2 +- tests/test_losses/test_base.py | 12 ++- tests/test_losses/test_gsl.py | 10 ++- tests/test_losses/test_likelihood.py | 6 +- tests/test_losses/test_msm.py | 2 +- tests/test_plot/base.py | 4 +- tests/test_plot/test_plot_results.py | 7 +- tests/test_samplers/test_base.py | 21 +++-- tests/test_samplers/test_best_batch.py | 4 +- tests/test_samplers/test_gaussian_process.py | 3 +- tests/test_samplers/test_halton.py | 2 +- tests/test_samplers/test_particle_swarm.py | 2 +- tests/test_samplers/test_random_forest.py | 3 +- tests/test_samplers/test_random_uniform.py | 4 +- tests/test_samplers/test_rseq.py | 2 +- tests/test_samplers/test_xgboost.py | 4 +- tests/test_search_space.py | 4 +- tests/test_utils/test_base.py | 5 +- .../test_pandas_json_checkpointing.py | 2 +- .../test_utils/test_sqlite3_checkpointing.py | 5 +- tests/test_utils/test_time_series.py | 2 +- tests/utils/base.py | 9 +- tests/utils/docs.py | 4 +- 61 files changed, 386 insertions(+), 183 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 0186a672..df8abe49 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 898c95b6..c5fa98a6 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -134,15 +134,16 @@ def __init__( self.n_jobs = n_jobs if n_jobs is not None else multiprocessing.cpu_count() print( - f"Selecting {self.n_jobs} processes for the parallel evaluation of the model" + f"Selecting {self.n_jobs} processes for the parallel evaluation of the model", ) self.scheduler = self.__validate_samplers_and_scheduler_constructor_args( - samplers, scheduler + samplers, + scheduler, ) self.samplers_id_table = self._construct_samplers_id_table( - list(self.scheduler.samplers) + list(self.scheduler.samplers), ) @classmethod @@ -156,7 +157,7 @@ def __validate_samplers_and_scheduler_constructor_args( both_not_none = samplers is not None and scheduler is not None if both_none and both_not_none: raise ValueError( - "only one between 'samplers' and 'scheduler' must be provided" + "only one between 'samplers' and 'scheduler' must be provided", ) if samplers is not None: @@ -235,7 +236,9 @@ def update_samplers_id_table(self, samplers: Sequence[BaseSampler]) -> None: @classmethod def restore_from_checkpoint( - cls, checkpoint_path: str, model: Callable + cls, + checkpoint_path: str, + model: Callable, ) -> "Calibrator": """Return an instantiated class from a database file and a model simulator. @@ -331,7 +334,8 @@ def simulate_model(self, params: NDArray) -> NDArray: simulated_data = np.array(simulated_data_list) simulated_data = np.reshape( - simulated_data, (params.shape[0], self.ensemble_size, self.N, self.D) + simulated_data, + (params.shape[0], self.ensemble_size, self.N, self.D), ) return simulated_data @@ -379,7 +383,8 @@ def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: for sim_data_ensemble in new_simulated_data: new_loss = self.loss_function.compute_loss( - sim_data_ensemble, self.real_data + sim_data_ensemble, + self.real_data, ) new_losses.append(new_loss) @@ -391,14 +396,14 @@ def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: ( self.batch_num_samp, [self.current_batch_index] * method.batch_size, - ) + ), ) self.method_samp = np.hstack( ( self.method_samp, [self.samplers_id_table[type(method).__name__]] * method.batch_size, - ) + ), ) # logging @@ -419,7 +424,7 @@ def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: ----> avg loss exist params: {avg_dist_existing_points} ----> curr min loss: {np.min(self.losses_samp)} ====> total elapsed time: {elapsed_tot}s - """ + """, ), end="", ) @@ -457,7 +462,9 @@ def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: @staticmethod def check_convergence( - losses_samp: NDArray, n_sampled_params: int, convergence_precision: int + losses_samp: NDArray, + n_sampled_params: int, + convergence_precision: int, ) -> bool: """Check convergence of the calibration. diff --git a/black_it/loss_functions/base.py b/black_it/loss_functions/base.py index 891a4696..73f3487f 100644 --- a/black_it/loss_functions/base.py +++ b/black_it/loss_functions/base.py @@ -47,7 +47,9 @@ def __init__( self.coordinate_filters = coordinate_filters def compute_loss( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute the loss between simulated and real data. @@ -87,7 +89,7 @@ def _filter_data( [ filter_(sim_data_ensemble[j, :, i]) for j in range(sim_data_ensemble.shape[0]) - ] + ], ) filtered_data.append(filtered_data_1d) @@ -136,7 +138,9 @@ def _check_coordinate_filters(self, num_coords: int) -> List[Optional[Callable]] @abstractmethod def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Return the loss between a specific coordinate of two time series. diff --git a/black_it/loss_functions/fourier.py b/black_it/loss_functions/fourier.py index 5924508a..d23f0cf8 100644 --- a/black_it/loss_functions/fourier.py +++ b/black_it/loss_functions/fourier.py @@ -120,7 +120,9 @@ def __init__( super().__init__(coordinate_weights, coordinate_filters) def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute Euclidean distance between the Fourier transform of the two time series. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 82fd8af8..2d0a9c04 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -96,7 +96,9 @@ def __init__( self.nb_word_lengths = nb_word_lengths def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Return the GSL-div measure. @@ -141,11 +143,18 @@ def compute_loss_1d( for sim_data in sim_data_ensemble: # discretize simulated series sim_xd = self.discretize( - sim_data, nb_values, np.min(sim_data), np.max(sim_data) + sim_data, + nb_values, + np.min(sim_data), + np.max(sim_data), ) loss = self.gsl_div_1d_1_sample( - sim_xd, obs_xd, nb_word_lengths, nb_values, ts_length + sim_xd, + obs_xd, + nb_word_lengths, + nb_values, + ts_length, ) gsl_loss += loss diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 976a6ca8..2cb9d9c1 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -68,7 +68,9 @@ def _get_bandwidth_scott(n: int, d: int) -> float: return h def compute_loss( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute the loss between simulated and real data. @@ -122,7 +124,7 @@ def _check_bandwidth(self, s: int, d: int) -> float: else: raise KeyError( "Select a valid rule of thumb (either 'silverman' or 'scott') " - "or directly a numerical value for the bandwidth" + "or directly a numerical value for the bandwidth", ) else: h = self.h @@ -130,9 +132,11 @@ def _check_bandwidth(self, s: int, d: int) -> float: return h def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute likelihood loss on a single dimension, not available.""" raise NotImplementedError( - "The likelihood cannot be currently computed on a single dimension." + "The likelihood cannot be currently computed on a single dimension.", ) diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index f26f68c3..33fd88c1 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -52,7 +52,9 @@ def __init__( super().__init__(coordinate_weights) def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Call scipy.spatial.distance.minkowski() on its arguments. diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 934ea5f5..665de43a 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -86,7 +86,8 @@ def __init__( default is False. """ MethodOfMomentsLoss._validate_covariance_and_calculator( - moment_calculator, covariance_mat + moment_calculator, + covariance_mat, ) super().__init__(coordinate_weights, coordinate_filters) @@ -121,7 +122,7 @@ def _validate_covariance_and_calculator( _CovarianceMatrixType(covariance_mat) except ValueError as exc: raise ValueError( - f"expected one of {list(map(lambda x: x.value, _CovarianceMatrixType))}, got {covariance_mat})" + f"expected one of {list(map(lambda x: x.value, _CovarianceMatrixType))}, got {covariance_mat})", ) from exc return @@ -129,21 +130,23 @@ def _validate_covariance_and_calculator( # a non null covariance_mat was given if not is_symmetric(covariance_mat): raise ValueError( - "the provided covariance matrix is not valid as it is not a symmetric matrix" + "the provided covariance matrix is not valid as it is not a symmetric matrix", ) if (moment_calculator is get_mom_ts_1d) and (covariance_mat.shape[0] != 18): raise ValueError( "the provided covariance matrix is not valid as it has a wrong shape: " - f"expected 18, got {covariance_mat.shape[0]}" + f"expected 18, got {covariance_mat.shape[0]}", ) return raise ValueError( - "please specify a valid covariance matrix, either as a string or directly as a numpy array" + "please specify a valid covariance matrix, either as a string or directly as a numpy array", ) def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute the loss based on the 'method of moments'. @@ -158,7 +161,7 @@ def compute_loss_1d( """ # compute the moments for the simulated ensemble ensemble_sim_mom_1d = np.array( - [self._moment_calculator(s) for s in sim_data_ensemble] + [self._moment_calculator(s) for s in sim_data_ensemble], ) # compute moments of the real time series @@ -179,7 +182,8 @@ def compute_loss_1d( return loss_1d if self._covariance_mat == _CovarianceMatrixType.INVERSE_VARIANCE.value: W = np.diag( # noqa: N806 - 1.0 / np.mean((real_mom_1d[None, :] - ensemble_sim_mom_1d) ** 2, axis=0) + 1.0 + / np.mean((real_mom_1d[None, :] - ensemble_sim_mom_1d) ** 2, axis=0), ) else: self._covariance_mat = cast(NDArray[np.float64], self._covariance_mat) @@ -198,7 +202,7 @@ def compute_loss_1d( raise ValueError( f"The size of the covariance matrix ({covariance_size}) " - f"and the number of moments ({moments_size}) should be identical" + f"and the number of moments ({moments_size}) should be identical", ) from e return loss_1d diff --git a/black_it/plot/plot_descriptive_statistics.py b/black_it/plot/plot_descriptive_statistics.py index b11c97f1..9b11d97d 100644 --- a/black_it/plot/plot_descriptive_statistics.py +++ b/black_it/plot/plot_descriptive_statistics.py @@ -30,7 +30,8 @@ def ts_stats(time_series_raw: List[float]) -> None: ts = np.array(time_series_raw, dtype="double") sq_diff = np.append( - np.absolute(np.diff(ts)), 0 + np.absolute(np.diff(ts)), + 0, ) # not precise! shouldn't append '0'! ts_acf = sm.tsa.acf(ts, nlags=20) diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 0fc37c61..5a977495 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -46,7 +46,8 @@ def _get_samplers_id_table(saving_folder: Union[str, os.PathLike]) -> Dict[str, def _get_samplers_names( - saving_folder: Union[str, os.PathLike], ids: List[int] + saving_folder: Union[str, os.PathLike], + ids: List[int], ) -> List[str]: """Get the names of the samplers from their ids and from the checkpoint of the calibration. @@ -81,7 +82,11 @@ def plot_convergence(saving_folder: Union[str, os.PathLike]) -> None: plt.figure() g = sns.lineplot( - data=df, x="batch_num_samp", y="losses_samp", hue="method_samp", palette="tab10" + data=df, + x="batch_num_samp", + y="losses_samp", + hue="method_samp", + palette="tab10", ) g = sns.lineplot( @@ -165,7 +170,8 @@ def plot_sampling(saving_folder: Union[str, os.PathLike]) -> None: def plot_losses_method_num( - saving_folder: Union[str, os.PathLike], method_num: int + saving_folder: Union[str, os.PathLike], + method_num: int, ) -> None: """Plot the parameter sampled by a specific sampling method, and color them according to their loss value. @@ -228,7 +234,8 @@ def plot_losses_interact(saving_folder: Union[str, os.PathLike]) -> None: def plot_sampling_batch_nums( - saving_folder: Union[str, os.PathLike], batch_nums: Collection[int] + saving_folder: Union[str, os.PathLike], + batch_nums: Collection[int], ) -> None: """Plot the parameter sampled in specific batches colored according to the sampling method used to sample them. diff --git a/black_it/samplers/base.py b/black_it/samplers/base.py index 05a8866e..2c777417 100644 --- a/black_it/samplers/base.py +++ b/black_it/samplers/base.py @@ -86,7 +86,10 @@ def sample( the sampled parameters """ samples = self.sample_batch( - self.batch_size, search_space, existing_points, existing_losses + self.batch_size, + search_space, + existing_points, + existing_losses, ) for n in range(self.max_deduplication_passes): @@ -98,21 +101,25 @@ def sample( break new_samples = self.sample_batch( - num_duplicates, search_space, existing_points, existing_losses + num_duplicates, + search_space, + existing_points, + existing_losses, ) samples[duplicates] = new_samples if n == self.max_deduplication_passes - 1: print( f"Warning: Repeated samples still found after {self.max_deduplication_passes} duplication passes." - " This is probably due to a small search space." + " This is probably due to a small search space.", ) return samples @staticmethod def find_and_get_duplicates( - new_points: NDArray[np.float64], existing_points: NDArray[np.float64] + new_points: NDArray[np.float64], + existing_points: NDArray[np.float64], ) -> List: """Find the points in 'new_points' that are already present in 'existing_points'. diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index 5d1568dc..ea698559 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -98,7 +98,7 @@ def sample_batch( raise ValueError( "best-batch sampler requires a number of existing points " f"which is at least the batch size {batch_size}, " - f"got {len(existing_points)}" + f"got {len(existing_points)}", ) # sort existing params @@ -107,10 +107,12 @@ def sample_batch( ][:batch_size, :] candidate_point_indexes: NDArray[np.int64] = self.random_generator.integers( - 0, batch_size, size=batch_size + 0, + batch_size, + size=batch_size, ) sampled_points: NDArray[np.float64] = np.copy( - candidate_points[candidate_point_indexes] + candidate_points[candidate_point_indexes], ) beta_binom_rv = betabinom(n=search_space.dims - 1, a=self.a, b=self.b) @@ -119,11 +121,14 @@ def sample_batch( for sampled_point in sampled_points: num_shocks: NDArray[np.int64] = beta_binom_rv.rvs(size=1) + 1 params_shocked: NDArray[np.int64] = self.random_generator.choice( - search_space.dims, tuple(num_shocks), replace=False + search_space.dims, + tuple(num_shocks), + replace=False, ) for index in params_shocked: shock_size: int = self.random_generator.integers( - 1, self.perturbation_range + 1, + self.perturbation_range, ) shock_sign: int = (self.random_generator.integers(0, 2) * 2) - 1 diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index 2044dfe7..1a0c5f93 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -46,7 +46,8 @@ def volume_d_dimensional_ball_radius_1(dims: int) -> float: def cubetobox( - X: NDArray[np.float64], space_bounds: NDArray # noqa: N803 + X: NDArray[np.float64], # noqa: N803 + space_bounds: NDArray, ) -> NDArray[np.float64]: """Go from normalized values (unit cube) to absolute values (box).""" box_points = space_bounds[0] + X * (space_bounds[1] - space_bounds[0]) @@ -54,7 +55,8 @@ def cubetobox( def boxtocube( - X: NDArray[np.float64], space_bounds: NDArray # noqa: N803 + X: NDArray[np.float64], # noqa: N803 + space_bounds: NDArray, ) -> NDArray[np.float64]: """Go from absolute values (box) to normalized values (unit cube).""" cube_points = (X - space_bounds[0]) / (space_bounds[1] - space_bounds[0]) @@ -104,7 +106,7 @@ def phi(r: float) -> float: except LinAlgError: # might help with singular matrices print( - "Singular matrix occurred during RBF-fit construction. RBF-fit might be inaccurate!" + "Singular matrix occurred during RBF-fit construction. RBF-fit might be inaccurate!", ) sol = np.linalg.lstsq(M, v)[0] @@ -146,7 +148,9 @@ def __init__( verbose: activate verbose mode """ super().__init__( - batch_size, random_state=random_state, max_deduplication_passes=0 + batch_size, + random_state=random_state, + max_deduplication_passes=0, ) self._max_samples = max_samples self._rho0 = positive_float(rho0) @@ -210,7 +214,7 @@ def sample_batch( { "type": "ineq", "fun": lambda x, localk=k: np.linalg.norm( - np.subtract(x, current_points[localk]) + np.subtract(x, current_points[localk]), ) - r, # noqa: B023 } diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index f21b91d3..5e5f5b67 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -71,7 +71,10 @@ def __init__( self._validate_acquisition(acquisition) super().__init__( - batch_size, random_state, max_deduplication_passes, candidate_pool_size + batch_size, + random_state, + max_deduplication_passes, + candidate_pool_size, ) self.optimize_restarts = optimize_restarts self.acquisition = acquisition @@ -95,7 +98,7 @@ def _validate_acquisition(acquisition: str) -> None: raise ValueError( "expected one of the following acquisition types: " f"[{' '.join(map(str, _AcquisitionTypes))}], " - f"got {acquisition}" + f"got {acquisition}", ) from e def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 @@ -140,7 +143,8 @@ def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 return candidates_score def _predict_mean_std( - self, X: NDArray[np.float64] # noqa: N803 + self, + X: NDArray[np.float64], # noqa: N803 ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: """Predict mean and standard deviation of a fitted GP. @@ -157,7 +161,9 @@ def _predict_mean_std( return m, s def _predict_EI( # noqa: N802 - self, X: NDArray[np.float64], jitter: float = 0.1 # noqa: N803 + self, + X: NDArray[np.float64], # noqa: N803 + jitter: float = 0.1, ) -> NDArray[np.float64]: """Compute the Expected Improvement per unit of cost. diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index e811e446..f5b4ed6b 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -69,7 +69,8 @@ def _set_random_state(self, random_state: Optional[int]) -> None: def _reset_sequence_index(self) -> None: """Reset the sequence index pointer.""" self._sequence_index = self.random_generator.integers( - _MIN_SEQUENCE_START_INDEX, _MAX_SEQUENCE_START_INDEX + _MIN_SEQUENCE_START_INDEX, + _MAX_SEQUENCE_START_INDEX, ) def sample_batch( @@ -91,7 +92,8 @@ def sample_batch( the parameter sampled """ unit_cube_points: NDArray[np.float64] = self._halton( - batch_size, search_space.dims + batch_size, + search_space.dims, ) p_bounds: NDArray[np.float64] = search_space.parameters_bounds sampled_points = p_bounds[0] + unit_cube_points * (p_bounds[1] - p_bounds[0]) @@ -112,7 +114,9 @@ def _halton(self, nb_samples: int, dims: int) -> NDArray[np.float64]: bases: NDArray[np.int64] = self._prime_number_generator.get_n_primes(dims) # Generate a sample using a Halton sequence. sample: NDArray[np.float64] = halton( - sample_size=nb_samples, bases=bases, n_start=self._sequence_index + sample_size=nb_samples, + bases=bases, + n_start=self._sequence_index, ) # increment sequence start index for the next sampling @@ -183,7 +187,9 @@ def get_n_primes(self, n: int) -> NDArray[np.int64]: def halton( - sample_size: int, bases: NDArray[np.int64], n_start: int + sample_size: int, + bases: NDArray[np.int64], + n_start: int, ) -> NDArray[np.float64]: """Van der Corput sequence, generalized as to accept a starting point in the sequence. diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index 5c7cb7e6..a21a6469 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -78,7 +78,9 @@ def __init__( """ # max_duplication_passes must be zero because the sampler is stateful super().__init__( - batch_size, random_state=random_state, max_deduplication_passes=0 + batch_size, + random_state=random_state, + max_deduplication_passes=0, ) # The batch size is the number of sampled parameters per iteration. In a Black-it sampler, each call to @@ -131,11 +133,11 @@ def c2(self) -> float: def _set_up(self, dims: int) -> None: """Set up the sampler.""" self._curr_particle_positions = self.random_generator.random( - size=(self.batch_size, dims) + size=(self.batch_size, dims), ) self._curr_particle_velocities = ( self.random_generator.random( - size=cast(NDArray, self._curr_particle_positions).shape + size=cast(NDArray, self._curr_particle_positions).shape, ) - 0.5 ) @@ -205,7 +207,9 @@ def sample_batch( return digitize_data(sampled_points, search_space.param_grid) def _update_best( - self, existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64] + self, + existing_points: NDArray[np.float64], + existing_losses: NDArray[np.float64], ) -> None: """Update the best local and global positions.""" _assert( @@ -224,7 +228,7 @@ def _update_best( previous_points = existing_points[batch_index_start:batch_index_stop] previous_losses = existing_losses[batch_index_start:batch_index_stop] for particle_id, (point, loss) in enumerate( - zip(previous_points, previous_losses) + zip(previous_points, previous_losses), ): best_particle_positions = cast(NDArray, self._best_particle_positions) best_position_losses = cast(NDArray, self._best_position_losses) diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index 75ca976b..c4415acf 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -83,7 +83,8 @@ def _set_random_state(self, random_state: Optional[int]) -> None: def _reset(self) -> None: """Reset the index of the sequence.""" self._sequence_index = self.random_generator.integers( - _MIN_SEQUENCE_START_INDEX, _MAX_SEQUENCE_START_INDEX + _MIN_SEQUENCE_START_INDEX, + _MAX_SEQUENCE_START_INDEX, ) self._sequence_start = self.random_generator.random() @@ -106,7 +107,8 @@ def sample_batch( the parameter sampled """ unit_cube_points: NDArray[np.float64] = self._r_sequence( - batch_size, search_space.dims + batch_size, + search_space.dims, ) p_bounds: NDArray[np.float64] = search_space.parameters_bounds sampled_points = p_bounds[0] + unit_cube_points * (p_bounds[1] - p_bounds[0]) @@ -124,7 +126,7 @@ def _r_sequence(self, nb_samples: int, dims: int) -> NDArray[np.float64]: """ phi = self.compute_phi(dims) alpha: NDArray[np.float64] = np.power(1 / phi, np.arange(1, dims + 1)).reshape( - (1, -1) + (1, -1), ) end_index = self._sequence_index + nb_samples indexes = np.arange(self._sequence_index, end_index).reshape((-1, 1)) diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 510f8fcb..2816be40 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -58,7 +58,10 @@ def __init__( ) super().__init__( - batch_size, random_state, max_deduplication_passes, candidate_pool_size + batch_size, + random_state, + max_deduplication_passes, + candidate_pool_size, ) self._n_estimators = n_estimators diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index 2f0d7090..70bbc532 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -70,9 +70,13 @@ def sample_candidates( ) -> NDArray[np.float64]: """Get a large pool of candidate parameters.""" candidates = RandomUniformSampler( - candidate_pool_size, random_state=self._get_random_seed() + candidate_pool_size, + random_state=self._get_random_seed(), ).sample_batch( - candidate_pool_size, search_space, existing_points, existing_losses + candidate_pool_size, + search_space, + existing_points, + existing_losses, ) return candidates @@ -104,7 +108,10 @@ def sample_batch( """ # Get a large pool of potential candidates candidates = self.sample_candidates( - self.candidate_pool_size, search_space, existing_points, existing_losses + self.candidate_pool_size, + search_space, + existing_points, + existing_losses, ) # Train surrogate model diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 6af88e26..3e954af5 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -64,7 +64,10 @@ def __init__( Lamperti, Roventini, and Sani, "Agent-based model calibration using machine learning surrogates" """ super().__init__( - batch_size, random_state, max_deduplication_passes, candidate_pool_size + batch_size, + random_state, + max_deduplication_passes, + candidate_pool_size, ) self._colsample_bytree = colsample_bytree diff --git a/black_it/schedulers/rl/envs/base.py b/black_it/schedulers/rl/envs/base.py index 3b24aa75..bbde83f4 100644 --- a/black_it/schedulers/rl/envs/base.py +++ b/black_it/schedulers/rl/envs/base.py @@ -63,7 +63,8 @@ def reset( return self.reset_state(), {} def step( - self, action: np.int64 + self, + action: np.int64, ) -> Tuple[ObsType, SupportsFloat, bool, bool, Dict[str, Any]]: """Do a step.""" _assert(self.action_space.contains(action)) diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index cc9b980b..b76505a4 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -45,7 +45,7 @@ def __init__( """Initialize the scheduler.""" self._original_samplers = samplers new_samplers, self._halton_sampler_id = self._add_or_get_bootstrap_sampler( - samplers + samplers, ) self._agent = agent @@ -72,7 +72,8 @@ def _set_random_state(self, random_state: Optional[int]) -> None: @classmethod def _add_or_get_bootstrap_sampler( - cls, samplers: Sequence[BaseSampler] + cls, + samplers: Sequence[BaseSampler], ) -> Tuple[Sequence[BaseSampler], int]: """Add or retrieve a sampler for bootstrapping. @@ -97,7 +98,7 @@ def _add_or_get_bootstrap_sampler( new_sampler = HaltonSampler(batch_size=1) return tuple(list(samplers) + cast(List[BaseSampler], [new_sampler])), len( - samplers + samplers, ) def _train(self) -> None: diff --git a/black_it/schedulers/round_robin.py b/black_it/schedulers/round_robin.py index e463e45a..75bc25d6 100644 --- a/black_it/schedulers/round_robin.py +++ b/black_it/schedulers/round_robin.py @@ -31,7 +31,9 @@ class RoundRobinScheduler(BaseScheduler): """ def __init__( # type: ignore[no-untyped-def] - self, *args, **kwargs # noqa: ANN002,ANN003 + self, + *args, # noqa: ANN002 + **kwargs, # noqa: ANN003 ) -> None: """Initialize the round-robin scheduler.""" super().__init__(*args, **kwargs) diff --git a/black_it/search_space.py b/black_it/search_space.py index cc10adfc..740dbe63 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -118,7 +118,7 @@ def _check_bounds( ) for i, (lower_bound, upper_bound, precision) in enumerate( - zip(parameters_bounds[0], parameters_bounds[1], parameters_precision) + zip(parameters_bounds[0], parameters_bounds[1], parameters_precision), ): # ensure lower bounds and upper bounds do not have the same value if lower_bound == upper_bound: @@ -136,7 +136,10 @@ def _check_bounds( # allowed parameter span if precision > (upper_bound - lower_bound): raise PrecisionGreaterThanBoundsRangeError( - i, lower_bound, upper_bound, precision + i, + lower_bound, + upper_bound, + precision, ) @property @@ -186,7 +189,7 @@ class BoundsNotOfSizeTwoError(SearchSpaceError): def __init__(self, count_bounds_subarrays: int) -> None: # noqa: D107 super().__init__( f"parameters_bounds must be a two dimensional array. This one has " - f"size {count_bounds_subarrays}." + f"size {count_bounds_subarrays}.", ) self.count_bounds_subarrays = count_bounds_subarrays @@ -202,12 +205,14 @@ class BoundsOfDifferentLengthError(SearchSpaceError): """ def __init__( # noqa: D107 - self, lower_bounds_length: int, upper_bounds_length: int + self, + lower_bounds_length: int, + upper_bounds_length: int, ) -> None: super().__init__( f"parameters_bounds subarrays must be of the same length. Lower " f"bounds length: {lower_bounds_length}, upper bounds length: " - f"{upper_bounds_length}." + f"{upper_bounds_length}.", ) self.lower_bounds_length = lower_bounds_length self.upper_bounds_length = upper_bounds_length @@ -222,11 +227,13 @@ class BadPrecisionLengthError(SearchSpaceError): """ def __init__( # noqa: D107 - self, precisions_length: int, bounds_length: int + self, + precisions_length: int, + bounds_length: int, ) -> None: super().__init__( f"parameters_precision array has {precisions_length} elements. Its " - f"length, instead, has to be {bounds_length}, the same as the bounds'." + f"length, instead, has to be {bounds_length}, the same as the bounds'.", ) self.precisions_length = precisions_length self.bounds_length = bounds_length @@ -245,7 +252,7 @@ def __init__(self, param_index: int, bound_value: float) -> None: # noqa: D107 f"Parameter {param_index}'s lower and upper bounds have the same " f"value ({bound_value}). This calibrator cannot handle that. " f"Please redefine externally your model in order to hardcode " - f"parameter {param_index} to {bound_value}." + f"parameter {param_index} to {bound_value}.", ) self.param_index = param_index self.bound_value = bound_value @@ -261,11 +268,14 @@ class LowerBoundGreaterThanUpperBoundError(SearchSpaceError): """ def __init__( # noqa: D107 - self, param_index: int, lower_bound: float, upper_bound: float + self, + param_index: int, + lower_bound: float, + upper_bound: float, ) -> None: super().__init__( f"Parameter {param_index}'s lower bound ({lower_bound}) must be " - f"lower than its upper bound ({upper_bound})." + f"lower than its upper bound ({upper_bound}).", ) self.param_index = param_index self.lower_bound = lower_bound @@ -296,13 +306,17 @@ class PrecisionGreaterThanBoundsRangeError(SearchSpaceError): """ def __init__( # noqa: D107 - self, param_index: int, lower_bound: float, upper_bound: float, precision: float + self, + param_index: int, + lower_bound: float, + upper_bound: float, + precision: float, ) -> None: super().__init__( f"Parameter {param_index}'s allowed range is [{lower_bound}, " f"{upper_bound}]. The requested precision is {precision}, but this " f"number cannot be greater than the bounds range " - f"({upper_bound}-{lower_bound}={upper_bound-lower_bound})." + f"({upper_bound}-{lower_bound}={upper_bound-lower_bound}).", ) self.param_index = param_index self.lower_bound = lower_bound diff --git a/black_it/utils/base.py b/black_it/utils/base.py index 14c8f917..70601da9 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -58,7 +58,8 @@ def positive_float(arg: float) -> float: def digitize_data( - data: NDArray[np.float64], param_grid: List[NDArray[np.float64]] + data: NDArray[np.float64], + param_grid: List[NDArray[np.float64]], ) -> NDArray[np.float64]: """Return a discretized version of the input sorted_array. diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index bdff3ec3..b466b98d 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -205,7 +205,10 @@ def save_calibrator_state( series_file = tables.open_file(str(series_filepath), mode="w") atom = tables.Float64Atom() array_c = series_file.create_earray( - series_file.root, "data", atom, (0, *series_samp.shape[1:]) + series_file.root, + "data", + atom, + (0, *series_samp.shape[1:]), ) array_c.append(series_samp) series_file.close() diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index 34f85201..e93f238b 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -182,19 +182,20 @@ def load_calibrator_state( """ checkpoint_path = Path(checkpoint_path) connection = sqlite3.connect( - checkpoint_path / "checkpoint.sqlite", detect_types=sqlite3.PARSE_DECLTYPES + checkpoint_path / "checkpoint.sqlite", + detect_types=sqlite3.PARSE_DECLTYPES, ) try: cursor = connection.cursor() checkpoint_schema_version: int = cursor.execute( - SQL_LOAD_USER_VERSION + SQL_LOAD_USER_VERSION, ).fetchone()[0] if checkpoint_schema_version != SCHEMA_VERSION: raise Exception( f"The checkpoint you want to load has been generated with another version of the code:\n" f"\tCheckpoint schema version: {checkpoint_schema_version}" - f"\tSchema version of the current code: {SCHEMA_VERSION}" + f"\tSchema version of the current code: {SCHEMA_VERSION}", ) ( @@ -310,7 +311,8 @@ def save_calibrator_state( loss_function_pickled = pickle.dumps(loss_function) connection = sqlite3.connect( - checkpoint_path / "checkpoint.sqlite", detect_types=sqlite3.PARSE_DECLTYPES + checkpoint_path / "checkpoint.sqlite", + detect_types=sqlite3.PARSE_DECLTYPES, ) try: diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index 280e24bd..e9947811 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -86,7 +86,8 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: def hp_filter( - time_series: NDArray[np.float64], lamb: float = 1600 + time_series: NDArray[np.float64], + lamb: float = 1600, ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: """Apply the HP filter to a time series. diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index ea6b8397..9c5f5411 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -55,7 +55,8 @@ def __init__( self.mean_init_wealth = mean_init_wealth self.schedule = RandomActivation(self) self.datacollector = DataCollector( - model_reporters={"Gini": compute_gini}, agent_reporters={"Wealth": "wealth"} + model_reporters={"Gini": compute_gini}, + agent_reporters={"Wealth": "wealth"}, ) # Create agents for i in range(self.num_agents): @@ -94,7 +95,9 @@ def __init__(self, unique_id: int, model: BoltzmannWealthModel) -> None: def move(self) -> None: """Move the agent.""" possible_steps = self.model.grid.get_neighborhood( - self.pos, moore=True, include_center=False + self.pos, + moore=True, + include_center=False, ) new_position = self.random.choice(possible_steps) self.model.grid.move_agent(self, new_position) diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index c0f3d077..f472bdfd 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -24,7 +24,9 @@ def BH2( # noqa: N802 - theta: Sequence[float], N: int, seed: Optional[int] # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: Optional[int], ) -> NDArray: """Model from Brock and Hommes 1998. 4.1.2. Fundamentalists versus trend chasers. @@ -84,7 +86,9 @@ def BH2( # noqa: N802 # def BH4( # noqa: N802 - theta: Sequence[float], N: int, seed: Optional[int] # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: Optional[int], ) -> NDArray: """Model from Brock and Hommes 1998. diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index cfeab9de..d9a9ec15 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -24,7 +24,9 @@ def forest_fire( - theta: NDArray, N: int, seed: Optional[int] = 0 # noqa: N803 + theta: NDArray, + N: int, # noqa: N803 + seed: Optional[int] = 0, ) -> NDArray: """A simple model of a wildfire on a 2D grid. diff --git a/examples/models/simple_models.py b/examples/models/simple_models.py index 9454ac8b..d1e014f7 100644 --- a/examples/models/simple_models.py +++ b/examples/models/simple_models.py @@ -35,7 +35,9 @@ def NormalM( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Normal samples with adjustable mean.""" np.random.seed(seed=seed) @@ -44,7 +46,9 @@ def NormalM( # noqa: N802 def NormalMV( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Normal samples with adjustable mean and variance.""" np.random.seed(seed=seed) @@ -54,7 +58,9 @@ def NormalMV( # noqa: N802 def NormalBer_3P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" np.random.seed(seed=seed) @@ -71,7 +77,9 @@ def NormalBer_3P( # noqa: N802 def NormalBer_5P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" np.random.seed(seed=seed) @@ -88,7 +96,9 @@ def NormalBer_5P( # noqa: N802 def Alpha( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Alpha iid samples. @@ -104,7 +114,9 @@ def Alpha( # noqa: N802 def MarkovC_2P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -134,7 +146,9 @@ def MarkovC_2P( # noqa: N802 def MarkovC_3P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -158,7 +172,9 @@ def MarkovC_3P( # noqa: N802 def MarkovC_KP( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" np.random.seed(seed=seed) @@ -181,7 +197,9 @@ def MarkovC_KP( # noqa: N802 def AR1( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """AR(1) model. @@ -200,7 +218,9 @@ def AR1( # noqa: N802 def AR1Ber_3P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """AR(1) + Bernoulli.""" np.random.seed(seed=seed) @@ -223,7 +243,9 @@ def AR1Ber_3P( # noqa: N802 def AR1_2P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """AR(1) with 2 parameters.""" np.random.seed(seed=seed) @@ -238,7 +260,9 @@ def AR1_2P( # noqa: N802 def AR1_3P( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """AR(1) with 3 parameters.""" np.random.seed(seed=seed) @@ -254,7 +278,9 @@ def AR1_3P( # noqa: N802 def ARMA2( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """ARMA(1, 1) model.""" np.random.seed(seed=seed) @@ -268,7 +294,9 @@ def ARMA2( # noqa: N802 def ARMAARCH2( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """ARMA(2,2) ARCH(2) model. @@ -296,7 +324,9 @@ def ARMAARCH2( # noqa: N802 def ARMAARCH4( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """ARMA(2,2) ARCH(2) model. @@ -310,7 +340,7 @@ def ARMAARCH4( # noqa: N802 y = np.zeros(N + 2) for i in range(2, N + 1): s[i] = np.sqrt( - theta[2] + theta[3] * (e[i - 1] ** 2) + theta[4] * (e[i - 2] ** 2) + theta[2] + theta[3] * (e[i - 1] ** 2) + theta[4] * (e[i - 2] ** 2), ) for i in range(2, N + 1): y[i] = ( @@ -326,7 +356,9 @@ def ARMAARCH4( # noqa: N802 def ARMAARCH6( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """ARMA(2,2) ARCH(2). @@ -342,7 +374,7 @@ def ARMAARCH6( # noqa: N802 s[i] = np.sqrt( theta[2] + theta[3] * ((s[i - 1] * e[i - 1]) ** 2) - + theta[4] * ((s[i - 2] * e[i - 2]) ** 2) + + theta[4] * ((s[i - 2] * e[i - 2]) ** 2), ) for i in range(2, N + 1): y[i] = ( @@ -358,7 +390,9 @@ def ARMAARCH6( # noqa: N802 def ARMAARCH4v2( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """ARMA(2,2) ARCH(2). @@ -372,7 +406,7 @@ def ARMAARCH4v2( # noqa: N802 y = np.zeros(N + 2) for i in range(2, N + 1): s[i] = np.sqrt( - theta[2] + theta[3] * ((y[i - 1]) ** 2) + theta[4] * ((y[i - 2]) ** 2) + theta[2] + theta[3] * ((y[i - 1]) ** 2) + theta[4] * ((y[i - 2]) ** 2), ) y[i] = ( 0 @@ -386,7 +420,9 @@ def ARMAARCH4v2( # noqa: N802 def RWSB1( # noqa: N802 - theta: Sequence[int], N: int, seed: int # noqa: N803 + theta: Sequence[int], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """RW with structural break. @@ -413,7 +449,9 @@ def RWSB1( # noqa: N802 def RWSB2( # noqa: N802 - theta: Sequence[float], N: int, seed: int # noqa: N803 + theta: Sequence[float], + N: int, # noqa: N803 + seed: int, ) -> NDArray[np.float64]: """RW with structural break. diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 1ab7a137..886e77f7 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -51,7 +51,8 @@ def parse_simulator_output(stdout: str) -> List[Dict[str, int]]: regex = re.compile(r"^EPOCH.*(initial status|ended with)") noise_filtered_out = filter( - lambda line: regex.match(line) is not None, stdout.splitlines() + lambda line: regex.match(line) is not None, + stdout.splitlines(), ) results_as_text = (line.split(":", 1)[1] for line in noise_filtered_out) results = (json.loads(text) for text in results_as_text) @@ -60,7 +61,8 @@ def parse_simulator_output(stdout: str) -> List[Dict[str, int]]: def _build_simulator_cmdline( - docker_image_name: str, sim_params: Dict[str, str] + docker_image_name: str, + sim_params: Dict[str, str], ) -> List[str]: """Convert a configuration object in a list of command line parameters. @@ -94,15 +96,16 @@ def _build_simulator_cmdline( ( (f"--{argname}", str(argvalue)) for argname, argvalue in sim_params.items() - ) - ) + ), + ), ) return args def execute_simulator( - path_to_simulator: str, sim_params: Dict[str, str] + path_to_simulator: str, + sim_params: Dict[str, str], ) -> List[Dict[str, int]]: """Execute the simulator with the given parameters, and return a structured output. diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index a2854fa3..dc51e0ad 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -45,7 +45,9 @@ def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, def SIR_w_breaks( # noqa: N802 - theta: NDArray, N: int, seed: Optional[int] = None # noqa: N803 + theta: NDArray, + N: int, # noqa: N803 + seed: Optional[int] = None, ) -> NDArray: """SIR_docker_w_breaks.""" breaktime = int(theta[0]) diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index 2631a9a6..46e38e87 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -71,7 +71,9 @@ def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, def SIR_w_breaks( # noqa: N802 - theta: NDArray, N: int, seed: Optional[int] # noqa: N803 + theta: NDArray, + N: int, # noqa: N803 + seed: Optional[int], ) -> NDArray: """SIR model with structural breaks. diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index d9081b2b..b2f46d19 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -59,7 +59,7 @@ class TestCalibrate: [0.8, 0.06], [0.99, 1.0], [0.06, 0.98], - ] + ], ) expected_losses = [ @@ -95,7 +95,7 @@ class TestCalibrate: [0.03, 0.36], [0.8, 0.06], [0.06, 0.98], - ] + ], ) darwin_expected_losses = [ @@ -183,7 +183,8 @@ def test_calibrator_calibrate(self, n_jobs: int) -> None: assert np.allclose(losses, self.expected_losses) def test_calibrator_with_check_convergence( - self, capsys: CaptureFixture[str] + self, + capsys: CaptureFixture[str], ) -> None: """Test the Calibrator.calibrate method with convergence check.""" cal = Calibrator( @@ -245,7 +246,8 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N _, _ = cal.calibrate(2) cal_restored = Calibrator.restore_from_checkpoint( - saving_folder_path_str, model=model + saving_folder_path_str, + model=model, ) # loop over all attributes of the classes @@ -254,7 +256,8 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N # if the attribute is an object just check the equality of their names if key == "samplers": for method1, method2 in zip( - vars_cal["samplers"], cal_restored.scheduler.samplers + vars_cal["samplers"], + cal_restored.scheduler.samplers, ): assert type(method1).__name__ == type(method2).__name__ # noqa if key == "scheduler": @@ -283,7 +286,7 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N # test the setting of a new sampler to the calibrator object best_batch_sampler = BestBatchSampler(batch_size=1) cal.set_samplers( - [random_sampler, best_batch_sampler] + [random_sampler, best_batch_sampler], ) # note: only the second sampler is new assert len(cal.scheduler.samplers) == 2 assert type(cal.scheduler.samplers[1]).__name__ == "BestBatchSampler" @@ -293,7 +296,7 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N # test the setting of a new scheduler to the calibrator object rsequence = RSequenceSampler(batch_size=1) cal.set_scheduler( - RoundRobinScheduler([rsequence]) + RoundRobinScheduler([rsequence]), ) # note: only the second sampler is new assert len(cal.scheduler.samplers) == 1 assert type(cal.scheduler.samplers[0]).__name__ == "RSequenceSampler" diff --git a/tests/test_docs/test_readme.py b/tests/test_docs/test_readme.py index 512b2b0f..c642a9f3 100644 --- a/tests/test_docs/test_readme.py +++ b/tests/test_docs/test_readme.py @@ -44,7 +44,7 @@ def test_quickstart_same_code_of_example(self) -> None: # test quickstart body is contained in example script quickstart_body = "\n".join( - quickstart_code_snippet.splitlines()[nb_import_lines:] + quickstart_code_snippet.splitlines()[nb_import_lines:], ) assert indent(quickstart_body, " " * 4) in example_code diff --git a/tests/test_examples/base.py b/tests/test_examples/base.py index 900a0f18..e7c3a3f2 100644 --- a/tests/test_examples/base.py +++ b/tests/test_examples/base.py @@ -38,7 +38,8 @@ def test_run(self) -> None: try: process_output: PopenResult = run_process( - [sys.executable, str(self.script_path)], timeout=self.timeout + [sys.executable, str(self.script_path)], + timeout=self.timeout, ) except RuntimeError as exc: pytest.fail(str(exc)) diff --git a/tests/test_examples/test_brock_hommes.py b/tests/test_examples/test_brock_hommes.py index 9cd4e372..dacd5682 100644 --- a/tests/test_examples/test_brock_hommes.py +++ b/tests/test_examples/test_brock_hommes.py @@ -242,7 +242,7 @@ def test_bh4_model() -> None: [-0.0171874], [-0.01127672], [0.0], - ] + ], ) result = BH4(theta, n, seed=seed) assert result.shape == (n, 1) diff --git a/tests/test_losses/test_base.py b/tests/test_losses/test_base.py index ef63a547..4f365b10 100644 --- a/tests/test_losses/test_base.py +++ b/tests/test_losses/test_base.py @@ -41,7 +41,9 @@ def __init__( self.loss_constant = loss_constant def compute_loss_1d( - self, sim_data_ensemble: NDArray[np.float64], real_data: NDArray[np.float64] + self, + sim_data_ensemble: NDArray[np.float64], + real_data: NDArray[np.float64], ) -> float: """Compute the loss (constant).""" return float(np.sum(sim_data_ensemble) * self.loss_constant) @@ -64,10 +66,12 @@ def compute_loss_1d( def setup(self) -> None: """Set up the tests.""" self.loss = TestComputeLoss.MyCustomLoss( - self.loss_constant, self.coordinate_weights, self.coordinate_filters + self.loss_constant, + self.coordinate_weights, + self.coordinate_filters, ) self.sim_data = np.ones( - (self.nb_sim_ensemble, self.nb_sim_rows, self.nb_sim_coords) + (self.nb_sim_ensemble, self.nb_sim_rows, self.nb_sim_coords), ) self.real_data = np.ones((self.nb_real_rows, self.nb_real_coords)) @@ -97,7 +101,7 @@ def test_run(self) -> None: result = self.loss.compute_loss(self.sim_data, self.real_data) assert result == self.coordinate_weights[0] * np.sum( - self.sim_data[:, :, 0] + self.sim_data[:, :, 0], ) + self.coordinate_weights[1] * np.sum(self.sim_data[:, :, 1]) diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index 3d1ee9ed..f87aab0f 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -70,13 +70,19 @@ def test_ordering_is_preserved(self, args: Tuple) -> None: time_series, nb_values, start_index, stop_index = args increasing_time_series = np.sort(time_series) increasing_actual = GslDivLoss.discretize( - increasing_time_series, nb_values, start_index, stop_index + increasing_time_series, + nb_values, + start_index, + stop_index, ) assert is_sorted(increasing_actual, reverse=False) decreasing_time_series = increasing_time_series[::-1] decreasing_actual = GslDivLoss.discretize( - decreasing_time_series, nb_values, start_index, stop_index + decreasing_time_series, + nb_values, + start_index, + stop_index, ) assert is_sorted(decreasing_actual, reverse=True) diff --git a/tests/test_losses/test_likelihood.py b/tests/test_losses/test_likelihood.py index ca4d122d..ef732007 100644 --- a/tests/test_losses/test_likelihood.py +++ b/tests/test_losses/test_likelihood.py @@ -26,7 +26,8 @@ def test_likelihood_1d() -> None: real_data = np.random.normal(0, 1, size=(7, 1)) expected_neg_log_likelihood = -np.sum( - -0.5 * np.sum(real_data**2, axis=1) - 0.5 * np.log(2.0 * np.pi), axis=0 + -0.5 * np.sum(real_data**2, axis=1) - 0.5 * np.log(2.0 * np.pi), + axis=0, ) expected_likelihood = np.exp(-expected_neg_log_likelihood) @@ -44,7 +45,8 @@ def test_likelihood_2d() -> None: real_data = np.random.normal(0, 1, size=(10, 2)) expected_neg_log_likelihood = -np.sum( - -0.5 * np.sum(real_data**2, axis=1) - 2.0 / 2.0 * np.log(2.0 * np.pi), axis=0 + -0.5 * np.sum(real_data**2, axis=1) - 2.0 / 2.0 * np.log(2.0 * np.pi), + axis=0, ) expected_likelihood = np.exp(-expected_neg_log_likelihood) sim_data_ensemble = np.random.normal(0, 1, size=(1, 1000000, 2)) diff --git a/tests/test_losses/test_msm.py b/tests/test_losses/test_msm.py index f8215f47..5744d0c5 100644 --- a/tests/test_losses/test_msm.py +++ b/tests/test_losses/test_msm.py @@ -110,7 +110,7 @@ def custom_moment_calculator(time_series: NDArray) -> NDArray: with pytest.raises( ValueError, match=re.escape( - "The size of the covariance matrix (3) and the number of moments (1) should be identical" + "The size of the covariance matrix (3) and the number of moments (1) should be identical", ), ): loss_func = MethodOfMomentsLoss( diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index 4f0de032..86b2d1f2 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -50,7 +50,9 @@ def run(self) -> None: return comparison_result = compare_images( - str(self.expected_image), str(actual_figure_path), self.tolerance + str(self.expected_image), + str(actual_figure_path), + self.tolerance, ) if comparison_result is not None: logging.warning("%s", comparison_result) diff --git a/tests/test_plot/test_plot_results.py b/tests/test_plot/test_plot_results.py index 1fe6d300..a4a7b7b0 100644 --- a/tests/test_plot/test_plot_results.py +++ b/tests/test_plot/test_plot_results.py @@ -69,7 +69,7 @@ class TestPlotLossesMethodNum(BasePlotResultsTest): saving_folder = EXAMPLE_SAVING_FOLDER @pytest.mark.skip( - "for this test case, we use the parametrized method 'test_run_by_method_num'" + "for this test case, we use the parametrized method 'test_run_by_method_num'", ) def test_run(self) -> None: """Run the test.""" @@ -98,7 +98,7 @@ class TestPlotBatchNums(BasePlotResultsTest): saving_folder = EXAMPLE_SAVING_FOLDER @pytest.mark.skip( - "for this test case, we use the parametrized method 'test_run_by_batch_num'" + "for this test case, we use the parametrized method 'test_run_by_batch_num'", ) def test_run(self) -> None: """Run the test.""" @@ -127,7 +127,8 @@ def test_plot_losses_method_num_raises_error_if_method_num_not_known( """Test that 'plot_losses_method_num' raises error if the method num is not known.""" method_num = 1 with pytest.raises( - ValueError, match=f"Samplers with method_num = {method_num} was never used" + ValueError, + match=f"Samplers with method_num = {method_num} was never used", ): plot_losses_method_num("dummy_folder", method_num) diff --git a/tests/test_samplers/test_base.py b/tests/test_samplers/test_base.py index 543551f3..d6cc0fc0 100644 --- a/tests/test_samplers/test_base.py +++ b/tests/test_samplers/test_base.py @@ -25,7 +25,7 @@ def test_find_and_get_duplicates() -> None: """Test the find_and_get_duplicates method.""" existing_points = np.array([[0, 1, 2], [0, 1, 2], [3, 4, 5]]) new_points = np.array( - [[0, 1, 2], [0, 1, 2], [3, 4, 5], [9, 10, 11], [12, 13, 14], [12, 13, 14]] + [[0, 1, 2], [0, 1, 2], [3, 4, 5], [9, 10, 11], [12, 13, 14], [12, 13, 14]], ) BaseSampler.__abstractmethods__ = frozenset() @@ -62,7 +62,8 @@ def setup(self) -> None: self.default_seed = 42 self.batch_size = 32 self.sampler = TestSetRandomState.MyCustomSampler( - self.batch_size, random_state=self.default_seed + self.batch_size, + random_state=self.default_seed, ) self.search_space = SearchSpace(self.bounds, self.bounds_step, False) # if SearchSpace has been successfully constructed we are assured that @@ -75,17 +76,25 @@ def test_set_random_state_gives_same_result(self) -> None: seed = 11 self.sampler.random_state = seed expected_result_1 = self.sampler.sample( - self.search_space, self.existing_points, self.existing_losses + self.search_space, + self.existing_points, + self.existing_losses, ) expected_result_2 = self.sampler.sample( - self.search_space, self.existing_points, self.existing_losses + self.search_space, + self.existing_points, + self.existing_losses, ) self.sampler.random_state = seed actual_result_1 = self.sampler.sample( - self.search_space, self.existing_points, self.existing_losses + self.search_space, + self.existing_points, + self.existing_losses, ) actual_result_2 = self.sampler.sample( - self.search_space, self.existing_points, self.existing_losses + self.search_space, + self.existing_points, + self.existing_losses, ) assert (expected_result_1 == actual_result_1).all() assert (expected_result_2 == actual_result_2).all() diff --git a/tests/test_samplers/test_best_batch.py b/tests/test_samplers/test_best_batch.py index 85f79ea8..1d5191fe 100644 --- a/tests/test_samplers/test_best_batch.py +++ b/tests/test_samplers/test_best_batch.py @@ -30,7 +30,7 @@ [0.05, 0.05], [0.0, 0.03], [0.03, 0.04], - ] + ], ) @@ -67,7 +67,7 @@ def test_best_batch_clipping() -> None: sampler = BestBatchSampler(batch_size=8, random_state=0) param_grid = SearchSpace( parameters_bounds=np.array( - [[lower_bound, upper_bound], [lower_bound, upper_bound]] + [[lower_bound, upper_bound], [lower_bound, upper_bound]], ).T, parameters_precision=np.array([0.001, 0.001]), verbose=False, diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 625d0429..9e2a5e26 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -33,7 +33,8 @@ def setup(self) -> None: @classmethod def _construct_fake_grid( - cls, n: int = 3 + cls, + n: int = 3, ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: """Construct a fake grid of evaluated losses.""" xs = np.linspace(0, 1, n) diff --git a/tests/test_samplers/test_halton.py b/tests/test_samplers/test_halton.py index 5b189492..7198e983 100644 --- a/tests/test_samplers/test_halton.py +++ b/tests/test_samplers/test_halton.py @@ -29,7 +29,7 @@ [0.72, 0.28], [0.47, 0.61], [0.97, 0.95], - ] + ], ) diff --git a/tests/test_samplers/test_particle_swarm.py b/tests/test_samplers/test_particle_swarm.py index c1e8e588..38c3de37 100644 --- a/tests/test_samplers/test_particle_swarm.py +++ b/tests/test_samplers/test_particle_swarm.py @@ -63,7 +63,7 @@ [0.0, 0.0], [0.85, 0.0], [0.08, 0.0], - ] + ], ) diff --git a/tests/test_samplers/test_random_forest.py b/tests/test_samplers/test_random_forest.py index f3f604e1..f6ee95a6 100644 --- a/tests/test_samplers/test_random_forest.py +++ b/tests/test_samplers/test_random_forest.py @@ -59,6 +59,7 @@ def test_random_forest_candidate_pool_size() -> None: expected_candidate_pool_size = 32 sampler2 = RandomForestSampler( - batch_size=batch_size, candidate_pool_size=expected_candidate_pool_size + batch_size=batch_size, + candidate_pool_size=expected_candidate_pool_size, ) assert sampler2.candidate_pool_size == expected_candidate_pool_size diff --git a/tests/test_samplers/test_random_uniform.py b/tests/test_samplers/test_random_uniform.py index cab93b2d..7f7e7091 100644 --- a/tests/test_samplers/test_random_uniform.py +++ b/tests/test_samplers/test_random_uniform.py @@ -29,7 +29,7 @@ [0.04, 0.61], [0.07, 0.98], [0.01, 0.73], - ] + ], ) @@ -57,7 +57,7 @@ def test_random_uniform_uniqueness() -> None: # then, sampled_params_after_deduplication will be the next element to be sampled. element_after_deduplication = np.array([[0.63, 0.54]]) expected_params_unique = np.vstack( - (element_after_deduplication, expected_params[1:]) + (element_after_deduplication, expected_params[1:]), ) sampler = RandomUniformSampler(batch_size=8, random_state=0) diff --git a/tests/test_samplers/test_rseq.py b/tests/test_samplers/test_rseq.py index 2e037106..d7927222 100644 --- a/tests/test_samplers/test_rseq.py +++ b/tests/test_samplers/test_rseq.py @@ -29,7 +29,7 @@ [0.71, 0.29], [0.47, 0.86], [0.22, 0.43], - ] + ], ) diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index 54a96644..0628c3dd 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -78,7 +78,7 @@ def test_clip_losses() -> None: -0.2, # b3 1.01, # g4 0.01, - ] + ], ) # b4 parameter_bounds = [ @@ -120,7 +120,7 @@ def test_clip_losses() -> None: 3.40282347e38, 3.40282347e38, 2.94273501e41, - ] + ], ), ) diff --git a/tests/test_search_space.py b/tests/test_search_space.py index 04be9171..791a935f 100644 --- a/tests/test_search_space.py +++ b/tests/test_search_space.py @@ -73,7 +73,9 @@ def test_search_space_fails_bounds_of_different_length() -> None: with pytest.raises(BoundsOfDifferentLengthError) as exc_info: _ = SearchSpace( - np.array([lower, upper], dtype=object), np.array(precision), True + np.array([lower, upper], dtype=object), + np.array(precision), + True, ) assert exc_info.value.lower_bounds_length == len(lower) assert exc_info.value.upper_bounds_length == len(upper) diff --git a/tests/test_utils/test_base.py b/tests/test_utils/test_base.py index 08637cab..5b599457 100644 --- a/tests/test_utils/test_base.py +++ b/tests/test_utils/test_base.py @@ -76,8 +76,9 @@ def test_is_symmetric() -> None: @given( hypothesis.extra.numpy.arrays( - np.float64, hypothesis.extra.numpy.array_shapes(max_dims=3, max_side=3) - ) + np.float64, + hypothesis.extra.numpy.array_shapes(max_dims=3, max_side=3), + ), ) def test_numpy_array_json_encoder_for_any_numpy_array(array: NDArray) -> None: """Test JSONEncoder 'NumpyArrayEncoder' for any NumPy array.""" diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index c78cdc97..bff8dcb0 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -38,7 +38,7 @@ def test_save_and_load_calibrator_state() -> None: saving_folder = "saving_folder" initial_random_seed = 0 random_generator_state = np.random.default_rng( - initial_random_seed + initial_random_seed, ).bit_generator.state model_name = "model" samplers = ["method_a", "method_b"] # list of objects diff --git a/tests/test_utils/test_sqlite3_checkpointing.py b/tests/test_utils/test_sqlite3_checkpointing.py index 85ed3d86..bf016954 100644 --- a/tests/test_utils/test_sqlite3_checkpointing.py +++ b/tests/test_utils/test_sqlite3_checkpointing.py @@ -41,7 +41,7 @@ def test_sqlite3_checkpointing() -> None: saving_file = "test" initial_random_seed = 0 random_generator_state = np.random.default_rng( - initial_random_seed + initial_random_seed, ).bit_generator.state model_name = "model" samplers = ["method_a", "method_b"] # list of objects @@ -114,7 +114,8 @@ def test_sqlite3_checkpointing_loading_when_code_state_version_different( @patch.object(black_it.utils.sqlite3_checkpointing, "Path") @patch("sqlite3.connect") def test_sqlite3_checkpointing_saving_when_error_occurs( - connect_mock: MagicMock, *_mocks: MagicMock + connect_mock: MagicMock, + *_mocks: MagicMock, ) -> None: """Test saving function when an error occurs.""" error_message = "error" diff --git a/tests/test_utils/test_time_series.py b/tests/test_utils/test_time_series.py index e78667d9..09f31a43 100644 --- a/tests/test_utils/test_time_series.py +++ b/tests/test_utils/test_time_series.py @@ -42,7 +42,7 @@ def test_get_mom_ts() -> None: [-0.11193418, 0.11907523, -0.04560956, 0.03323841, -0.11316762], [-0.02379168, -0.00181507, -0.00476993, 0.13203911, -0.06712482], [0.00665585, -0.06428739, -0.11528277, 0.02027778, -0.07112011], - ] + ], ) np.random.seed(42) time_series = np.random.rand(100, 5) diff --git a/tests/utils/base.py b/tests/utils/base.py index aba75cbf..d7cebd27 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -38,7 +38,8 @@ class PopenResult: def run_process( - command: List[str], timeout: float = DEFAULT_SUBPROCESS_TIMEOUT + command: List[str], + timeout: float = DEFAULT_SUBPROCESS_TIMEOUT, ) -> PopenResult: """Run a process, and wait for it to stop. @@ -142,7 +143,8 @@ def decorator(pytest_func_or_cls: Union[Callable, Type]) -> Callable: @wraps(pytest_func_or_cls) def wrapper( # type: ignore[no-untyped-def] # noqa: ANN202 - *args, **kwargs # noqa: ANN002,ANN003 + *args, # noqa: ANN002 + **kwargs, # noqa: ANN003 ): action() return pytest_func_or_cls(*args, **kwargs) @@ -171,5 +173,6 @@ def wrapper( # type: ignore[no-untyped-def] # noqa: ANN202 skip_on_windows = pytest.mark.skipif( - sys.platform == "win32", reason="This test cannot be ran on Windows" + sys.platform == "win32", + reason="This test cannot be ran on Windows", ) diff --git a/tests/utils/docs.py b/tests/utils/docs.py index d4493a87..b5f03e8c 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -71,10 +71,10 @@ def setup_class(cls) -> None: markdown_json = json.loads(markdown_parsed) cls.blocks = markdown_json["children"] cls.code_blocks = list( - map(code_block_extractor, filter(code_block_filter, cls.blocks)) + map(code_block_extractor, filter(code_block_filter, cls.blocks)), ) cls.python_blocks = list( - map(code_block_extractor, filter(python_code_block_filter, cls.blocks)) + map(code_block_extractor, filter(python_code_block_filter, cls.blocks)), ) From c37ff05d5914c6115e074d73a07a5356afc1f6f7 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 20 Sep 2023 12:04:30 +0200 Subject: [PATCH 23/56] lint: fix flake8-copyright errors To reproduce the errors: ``` ruff check --select "CPY" black_it tests examples scripts ``` The TestReadme is updated accordingly so to remove the copyright and license notice before comparing the example/main.py with the code snippet in the readme. --- .ruff.toml | 2 +- examples/docker-sir.py | 16 ++++++++++++++++ examples/main.py | 16 ++++++++++++++++ tests/test_docs/base.py | 3 +++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.ruff.toml b/.ruff.toml index df8abe49..5d2012ed 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/examples/docker-sir.py b/examples/docker-sir.py index 9124b4ef..dbfdc39f 100755 --- a/examples/docker-sir.py +++ b/examples/docker-sir.py @@ -1,4 +1,20 @@ #!/usr/bin/env python3 +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """SIR model calibration example using a Docker-based simulator.""" from models.sir.sir_docker import SIR # type: ignore diff --git a/examples/main.py b/examples/main.py index 7273bbd1..6dd47bdd 100755 --- a/examples/main.py +++ b/examples/main.py @@ -1,4 +1,20 @@ #!/usr/bin/env python3 +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """This is a simple example showing the main features of the library.""" import models.simple_models as md # type: ignore diff --git a/tests/test_docs/base.py b/tests/test_docs/base.py index 90d71551..9f8e754a 100644 --- a/tests/test_docs/base.py +++ b/tests/test_docs/base.py @@ -35,6 +35,9 @@ def extract_example_code(self) -> str: # remove shebang example_code = example_code.replace("#!/usr/bin/env python3\n", "") + # remove copyright notice and license + example_code = re.sub("^#.*\n(\n?)", "", example_code, flags=re.MULTILINE) + # remove docstring example_code = re.sub('""".*"""\n', "", example_code) From 443eea7c3c52148068f0bc320ef71001e823dc3b Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 09:17:23 +0200 Subject: [PATCH 24/56] lint: fix flake8-comprehensions errors To reproduce the errors: ``` ruff check --select "C4" black_it tests examples scripts ``` The error were fixed using the '--fix' flag. --- .ruff.toml | 2 +- black_it/loss_functions/msm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 5d2012ed..7035d48c 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 665de43a..2395fb07 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -122,7 +122,7 @@ def _validate_covariance_and_calculator( _CovarianceMatrixType(covariance_mat) except ValueError as exc: raise ValueError( - f"expected one of {list(map(lambda x: x.value, _CovarianceMatrixType))}, got {covariance_mat})", + f"expected one of {[x.value for x in _CovarianceMatrixType]}, got {covariance_mat})", ) from exc return From 9c5da9dbf57ead62ab9274f9e41d71b7de4a748c Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 09:21:17 +0200 Subject: [PATCH 25/56] lint: fix flake8-errmsg errors To reproduce the errors: ``` ruff check --select "EM" black_it tests examples scripts ``` The error were fixed using the '--fix' flag. --- .ruff.toml | 2 +- black_it/calibrator.py | 3 ++- black_it/loss_functions/likelihood.py | 10 +++++++--- black_it/loss_functions/msm.py | 23 ++++++++++++++++------- black_it/plot/plot_results.py | 3 ++- black_it/samplers/best_batch.py | 8 +++++--- black_it/samplers/gaussian_process.py | 8 +++++--- black_it/schedulers/rl/envs/mab.py | 3 ++- black_it/schedulers/rl/rl_scheduler.py | 6 ++++-- black_it/utils/sqlite3_checkpointing.py | 10 +++++++--- tests/utils/base.py | 3 ++- 11 files changed, 53 insertions(+), 26 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 7035d48c..7773934d 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index c5fa98a6..2eeea9ef 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -156,8 +156,9 @@ def __validate_samplers_and_scheduler_constructor_args( both_none = samplers is None and scheduler is None both_not_none = samplers is not None and scheduler is not None if both_none and both_not_none: + msg = "only one between 'samplers' and 'scheduler' must be provided" raise ValueError( - "only one between 'samplers' and 'scheduler' must be provided", + msg, ) if samplers is not None: diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 2cb9d9c1..3c8f3912 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -122,9 +122,12 @@ def _check_bandwidth(self, s: int, d: int) -> float: elif self.h == "scott": h = self._get_bandwidth_scott(s, d) else: - raise KeyError( + msg = ( "Select a valid rule of thumb (either 'silverman' or 'scott') " - "or directly a numerical value for the bandwidth", + "or directly a numerical value for the bandwidth" + ) + raise KeyError( + msg, ) else: h = self.h @@ -137,6 +140,7 @@ def compute_loss_1d( real_data: NDArray[np.float64], ) -> float: """Compute likelihood loss on a single dimension, not available.""" + msg = "The likelihood cannot be currently computed on a single dimension." raise NotImplementedError( - "The likelihood cannot be currently computed on a single dimension.", + msg, ) diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 2395fb07..49e278f4 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -121,26 +121,32 @@ def _validate_covariance_and_calculator( try: _CovarianceMatrixType(covariance_mat) except ValueError as exc: + msg = f"expected one of {[x.value for x in _CovarianceMatrixType]}, got {covariance_mat})" raise ValueError( - f"expected one of {[x.value for x in _CovarianceMatrixType]}, got {covariance_mat})", + msg, ) from exc return if isinstance(covariance_mat, np.ndarray): # a non null covariance_mat was given if not is_symmetric(covariance_mat): + msg = "the provided covariance matrix is not valid as it is not a symmetric matrix" raise ValueError( - "the provided covariance matrix is not valid as it is not a symmetric matrix", + msg, ) if (moment_calculator is get_mom_ts_1d) and (covariance_mat.shape[0] != 18): - raise ValueError( + msg = ( "the provided covariance matrix is not valid as it has a wrong shape: " - f"expected 18, got {covariance_mat.shape[0]}", + f"expected 18, got {covariance_mat.shape[0]}" + ) + raise ValueError( + msg, ) return + msg = "please specify a valid covariance matrix, either as a string or directly as a numpy array" raise ValueError( - "please specify a valid covariance matrix, either as a string or directly as a numpy array", + msg, ) def compute_loss_1d( @@ -200,9 +206,12 @@ def compute_loss_1d( # original error. raise - raise ValueError( + msg = ( f"The size of the covariance matrix ({covariance_size}) " - f"and the number of moments ({moments_size}) should be identical", + f"and the number of moments ({moments_size}) should be identical" + ) + raise ValueError( + msg, ) from e return loss_1d diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 5a977495..9bb81cac 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -182,7 +182,8 @@ def plot_losses_method_num( df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) if method_num not in set(df["method_samp"]): - raise ValueError(f"Samplers with method_num = {method_num} was never used") + msg = f"Samplers with method_num = {method_num} was never used" + raise ValueError(msg) df = df.loc[df["method_samp"] == method_num] diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index ea698559..0c2e84fb 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -95,10 +95,12 @@ def sample_batch( the sampled parameters (an array of shape `(self.batch_size, search_space.dims)`) """ if len(existing_points) < batch_size: - raise ValueError( + msg = ( "best-batch sampler requires a number of existing points " - f"which is at least the batch size {batch_size}, " - f"got {len(existing_points)}", + f"which is at least the batch size {batch_size}, got {len(existing_points)}" + ) + raise ValueError( + msg, ) # sort existing params diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 5e5f5b67..f61d9db4 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -95,10 +95,12 @@ def _validate_acquisition(acquisition: str) -> None: try: _AcquisitionTypes(acquisition) except ValueError as e: - raise ValueError( + msg = ( "expected one of the following acquisition types: " - f"[{' '.join(map(str, _AcquisitionTypes))}], " - f"got {acquisition}", + f"[{' '.join(map(str, _AcquisitionTypes))}], got {acquisition}" + ) + raise ValueError( + msg, ) from e def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 diff --git a/black_it/schedulers/rl/envs/mab.py b/black_it/schedulers/rl/envs/mab.py index f728e012..1c85d7f2 100644 --- a/black_it/schedulers/rl/envs/mab.py +++ b/black_it/schedulers/rl/envs/mab.py @@ -35,7 +35,8 @@ def reset_state(self) -> int: def get_reward(self, best_param: NDArray, best_loss: float) -> float: """Get the reward.""" if self._curr_best_loss is None: - raise ValueError("cannot get reward, curr_best_loss should be already set") + msg = "cannot get reward, curr_best_loss should be already set" + raise ValueError(msg) reward = 0.0 if best_loss < self._curr_best_loss: reward = (self._curr_best_loss - best_loss) / self._curr_best_loss diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index b76505a4..0093ae31 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -116,7 +116,8 @@ def _train(self) -> None: def start_session(self) -> None: """Set up the scheduler for a new session.""" if not self._stopped: - raise ValueError("cannot start session: the session has already started") + msg = "cannot start session: the session has already started" + raise ValueError(msg) self._stopped = False self._agent_thread = threading.Thread(target=self._train) self._agent_thread.start() @@ -153,7 +154,8 @@ def update( def end_session(self) -> None: """Tear down the scheduler at the end of the session.""" if self._stopped: - raise ValueError("cannot start session: the session has not started yet") + msg = "cannot start session: the session has not started yet" + raise ValueError(msg) self._stopped = True self._out_queue.put(None) cast(threading.Thread, self._agent_thread).join() diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index e93f238b..a0e71e96 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -192,10 +192,14 @@ def load_calibrator_state( SQL_LOAD_USER_VERSION, ).fetchone()[0] if checkpoint_schema_version != SCHEMA_VERSION: + msg = ( + "The checkpoint you want to load has been generated with another version of the code" + ":\n\tCheckpoint schema version:" + f" {checkpoint_schema_version}" + f"\tSchema version of the current code: {SCHEMA_VERSION}" + ) raise Exception( - f"The checkpoint you want to load has been generated with another version of the code:\n" - f"\tCheckpoint schema version: {checkpoint_schema_version}" - f"\tSchema version of the current code: {SCHEMA_VERSION}", + msg, ) ( diff --git a/tests/utils/base.py b/tests/utils/base.py index d7cebd27..ec6b65e3 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -69,7 +69,8 @@ def run_process( if poll is None: # if graceful stop failed, kill the process process.terminate() - raise RuntimeError(f"command {command} failed with error: {exc}") from exc + msg = f"command {command} failed with error: {exc}" + raise RuntimeError(msg) from exc returncode = process.returncode stdout = stdout_bytes.decode() if stdout_bytes else "" From dfeb9d296b7c53a8b31b25dcdf02f897228154ab Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 09:23:33 +0200 Subject: [PATCH 26/56] lint: fix flake8-executable errors To reproduce the errors: ``` ruff check --select "EXE" black_it tests examples scripts ``` The (unique) error was fixed manually. The black_it/calibrator.py module contained a shebang, but it was not executable. It used to be executable, so probably when the executable part was removed (i.e. __name__ == "__main__"), the shebang was not updated accordingly. --- .ruff.toml | 2 +- black_it/calibrator.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 7773934d..0c799d74 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 2eeea9ef..cd043229 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Black-box ABM Calibration Kit (Black-it) # Copyright (C) 2021-2023 Banca d'Italia # From 692dbd11aff6fe9078f47637093092690d523298 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 10:19:50 +0200 Subject: [PATCH 27/56] lint: fix flake8-future-annotations errors To reproduce the errors: ``` ruff check --select "FA" black_it tests examples scripts ``` The error were fixed manually by adding 'from __future__ import annotations' in the modules where the errors occurred. The change caused several errors from pyupgrade, which were fixed using --fix. --- .ruff.toml | 2 +- black_it/calibrator.py | 35 ++++++++++--------- black_it/loss_functions/base.py | 13 +++---- black_it/loss_functions/fourier.py | 8 +++-- black_it/loss_functions/gsl_div.py | 16 +++++---- black_it/loss_functions/likelihood.py | 10 +++--- black_it/loss_functions/minkowski.py | 8 +++-- black_it/loss_functions/msm.py | 12 ++++--- black_it/plot/plot_descriptive_statistics.py | 4 +-- black_it/plot/plot_results.py | 26 +++++++------- black_it/samplers/base.py | 6 ++-- black_it/samplers/best_batch.py | 5 ++- black_it/samplers/cors.py | 6 ++-- black_it/samplers/gaussian_process.py | 16 +++++---- black_it/samplers/halton.py | 10 +++--- black_it/samplers/particle_swarm.py | 20 ++++++----- black_it/samplers/r_sequence.py | 6 ++-- black_it/samplers/random_forest.py | 12 ++++--- black_it/samplers/surrogate.py | 6 ++-- black_it/samplers/xgboost.py | 10 +++--- black_it/schedulers/base.py | 10 +++--- .../schedulers/rl/agents/epsilon_greedy.py | 5 +-- black_it/schedulers/rl/envs/base.py | 16 +++++---- black_it/schedulers/rl/rl_scheduler.py | 16 +++++---- black_it/search_space.py | 15 ++++---- black_it/utils/base.py | 10 +++--- black_it/utils/json_pandas_checkpointing.py | 12 ++++--- black_it/utils/seedable.py | 13 ++++--- black_it/utils/sqlite3_checkpointing.py | 12 ++++--- black_it/utils/time_series.py | 5 ++- examples/models/economics/brock_hommes.py | 7 ++-- examples/models/forest_fire/forest_fire.py | 4 +-- examples/models/sir/simlib.py | 14 ++++---- examples/models/sir/sir_docker.py | 6 ++-- examples/models/sir/sir_python.py | 6 ++-- tests/test_losses/test_base.py | 12 ++++--- tests/test_losses/test_gsl.py | 11 +++--- tests/test_plot/base.py | 8 +++-- .../test_plot_descriptive_statistics.py | 6 ++-- tests/test_samplers/test_gaussian_process.py | 8 +++-- tests/test_search_space.py | 9 +++-- tests/utils/base.py | 8 +++-- tests/utils/docs.py | 20 +++++------ tests/utils/strategies.py | 8 +++-- 44 files changed, 259 insertions(+), 213 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 0c799d74..ceb164a3 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index cd043229..2e3247dc 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -15,13 +15,14 @@ # along with this program. If not, see . """This module contains the main class Calibrator.""" +from __future__ import annotations import multiprocessing import os import textwrap import time import warnings -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union, cast +from typing import Callable, Sequence, cast import numpy as np from joblib import Parallel, delayed @@ -50,17 +51,17 @@ def __init__( loss_function: BaseLoss, real_data: NDArray[np.float64], model: Callable, - parameters_bounds: Union[NDArray[np.float64], List[List[float]]], - parameters_precision: Union[NDArray[np.float64], List[float]], + parameters_bounds: NDArray[np.float64] | list[list[float]], + parameters_precision: NDArray[np.float64] | list[float], ensemble_size: int, - samplers: Optional[Sequence[BaseSampler]] = None, - scheduler: Optional[BaseScheduler] = None, - sim_length: Optional[int] = None, - convergence_precision: Optional[int] = None, + samplers: Sequence[BaseSampler] | None = None, + scheduler: BaseScheduler | None = None, + sim_length: int | None = None, + convergence_precision: int | None = None, verbose: bool = True, - saving_folder: Optional[str] = None, - random_state: Optional[int] = None, - n_jobs: Optional[int] = None, + saving_folder: str | None = None, + random_state: int | None = None, + n_jobs: int | None = None, ) -> None: """Initialize the Calibrator object. @@ -148,8 +149,8 @@ def __init__( @classmethod def __validate_samplers_and_scheduler_constructor_args( cls, - samplers: Optional[Sequence[BaseSampler]], - scheduler: Optional[BaseScheduler], + samplers: Sequence[BaseSampler] | None, + scheduler: BaseScheduler | None, ) -> BaseScheduler: """Validate the 'samplers' and the 'scheduler' arguments provided to the constructor.""" both_none = samplers is None and scheduler is None @@ -174,7 +175,7 @@ def _set_samplers_seeds(self) -> None: self._get_random_seed() @staticmethod - def _construct_samplers_id_table(samplers: List[BaseSampler]) -> Dict[str, int]: + def _construct_samplers_id_table(samplers: list[BaseSampler]) -> dict[str, int]: """Construct the samplers-by-id table. Given the list (built-in or user-defined) of samplers a calibration @@ -203,7 +204,7 @@ def _construct_samplers_id_table(samplers: List[BaseSampler]) -> Dict[str, int]: return samplers_id_table - def set_samplers(self, samplers: List[BaseSampler]) -> None: + def set_samplers(self, samplers: list[BaseSampler]) -> None: """Set the samplers list of the calibrator. This method overwrites the samplers of the calibrator object with a custom list of samplers. @@ -239,7 +240,7 @@ def restore_from_checkpoint( cls, checkpoint_path: str, model: Callable, - ) -> "Calibrator": + ) -> Calibrator: """Return an instantiated class from a database file and a model simulator. Args: @@ -340,7 +341,7 @@ def simulate_model(self, params: NDArray) -> NDArray: return simulated_data - def calibrate(self, n_batches: int) -> Tuple[NDArray, NDArray]: + def calibrate(self, n_batches: int) -> tuple[NDArray, NDArray]: """Run calibration for n batches. Args: @@ -482,7 +483,7 @@ def check_convergence( ) return converged - def create_checkpoint(self, file_name: Union[str, os.PathLike]) -> None: + def create_checkpoint(self, file_name: str | os.PathLike) -> None: """Save the current state of the object. Args: diff --git a/black_it/loss_functions/base.py b/black_it/loss_functions/base.py index 73f3487f..d8e2a479 100644 --- a/black_it/loss_functions/base.py +++ b/black_it/loss_functions/base.py @@ -15,9 +15,10 @@ # along with this program. If not, see . """This module defines the 'BaseLoss' base class.""" +from __future__ import annotations from abc import ABC, abstractmethod -from typing import Callable, List, Optional +from typing import Callable, Optional import numpy as np from numpy.typing import NDArray @@ -33,8 +34,8 @@ class BaseLoss(ABC): def __init__( self, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, ) -> None: """Initialize the loss function. @@ -75,7 +76,7 @@ def compute_loss( @staticmethod def _filter_data( - filters: List[Optional[Callable]], + filters: list[Callable | None], sim_data_ensemble: NDArray[np.float64], ) -> NDArray[np.float64]: """Filter the simulated time series.""" @@ -115,9 +116,9 @@ def _check_coordinate_weights(self, num_coords: int) -> NDArray[np.float64]: return weights - def _check_coordinate_filters(self, num_coords: int) -> List[Optional[Callable]]: + def _check_coordinate_filters(self, num_coords: int) -> list[Callable | None]: """Check self.coordinate_filters and return usable filters.""" - filters: List[Optional[Callable]] + filters: list[Callable | None] if self.coordinate_filters is None: # a list of identity functions diff --git a/black_it/loss_functions/fourier.py b/black_it/loss_functions/fourier.py index d23f0cf8..8f5a9623 100644 --- a/black_it/loss_functions/fourier.py +++ b/black_it/loss_functions/fourier.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """This module contains the implementation of the Fast Fourier Transform loss.""" -from typing import Callable, List, Optional +from __future__ import annotations + +from typing import Callable import numpy as np from numpy.typing import NDArray @@ -90,8 +92,8 @@ def __init__( self, frequency_filter: FrequencyFilter = gaussian_low_pass_filter, f: float = 0.8, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, ) -> None: """Loss computed using a distance in the Fourier space of the time series. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 2d0a9c04..af8aa0e6 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -51,7 +51,9 @@ **Note**: b,L don't increase much the comp power required (ie from (2,2) to (19,19) +20% time). """ -from typing import Callable, List, Optional, Union +from __future__ import annotations + +from typing import Callable import numpy as np from numpy.typing import NDArray @@ -77,10 +79,10 @@ class GslDivLoss(BaseLoss): def __init__( self, - nb_values: Optional[int] = None, - nb_word_lengths: Optional[int] = None, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + nb_values: int | None = None, + nb_word_lengths: int | None = None, + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, ) -> None: """Initialize the GSL-div loss object. @@ -215,8 +217,8 @@ def gsl_div_1d_1_sample( def discretize( time_series: NDArray[np.float64], nb_values: int, - start_index: Union[np.float64, float], - stop_index: Union[np.float64, float], + start_index: np.float64 | float, + stop_index: np.float64 | float, ) -> NDArray[np.int64]: """Discretize the TS in 'nb_values' finite states. diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 3c8f3912..45f8b7fd 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -15,8 +15,10 @@ # along with this program. If not, see . """This module defines the 'LikelihoodLoss' class.""" +from __future__ import annotations + import warnings -from typing import Callable, List, Optional, Union +from typing import Callable import numpy as np from numpy.typing import NDArray @@ -40,9 +42,9 @@ class LikelihoodLoss(BaseLoss): def __init__( self, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, - h: Union[str, float] = "silverman", + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, + h: str | float = "silverman", ) -> None: """Initialize the loss function. diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index 33fd88c1..2a624094 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """This module contains the implementation of the quadratic loss.""" -from typing import Callable, List, Optional +from __future__ import annotations + +from typing import Callable import numpy as np from numpy.typing import NDArray @@ -30,8 +32,8 @@ class MinkowskiLoss(BaseLoss): def __init__( self, p: int = 2, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, ) -> None: """Loss computed using a Minkowski distance. diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 49e278f4..c225b3c1 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -19,8 +19,10 @@ This module contains the implementation of the loss function based on the 'method of moments'. """ +from __future__ import annotations + from enum import Enum -from typing import Callable, List, Optional, Union, cast +from typing import Callable, cast import numpy as np from numpy.typing import NDArray @@ -52,10 +54,10 @@ class MethodOfMomentsLoss(BaseLoss): def __init__( self, - covariance_mat: Union[str, NDArray[np.float64]] = "identity", - coordinate_weights: Optional[NDArray[np.float64]] = None, + covariance_mat: str | NDArray[np.float64] = "identity", + coordinate_weights: NDArray[np.float64] | None = None, moment_calculator: MomentCalculator = get_mom_ts_1d, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + coordinate_filters: list[Callable | None] | None = None, standardise_moments: bool = False, ) -> None: """Initialize the loss function based on the 'method of moments'. @@ -98,7 +100,7 @@ def __init__( @staticmethod def _validate_covariance_and_calculator( moment_calculator: MomentCalculator, - covariance_mat: Union[NDArray[np.float64], str], + covariance_mat: NDArray[np.float64] | str, ) -> None: """Validate the covariance matrix. diff --git a/black_it/plot/plot_descriptive_statistics.py b/black_it/plot/plot_descriptive_statistics.py index 9b11d97d..062641cf 100644 --- a/black_it/plot/plot_descriptive_statistics.py +++ b/black_it/plot/plot_descriptive_statistics.py @@ -16,14 +16,14 @@ # """This module contains helper functions for plotting TS descriptive statistics.""" -from typing import List +from __future__ import annotations import matplotlib.pyplot as plt import numpy as np import statsmodels.api as sm -def ts_stats(time_series_raw: List[float]) -> None: +def ts_stats(time_series_raw: list[float]) -> None: """Show TS graphical descriptive statistics.""" color = "darkslateblue" alpha = 0.8 diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 9bb81cac..fb3fde69 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -15,9 +15,11 @@ # along with this program. If not, see . """This module contains utilities for plotting results.""" +from __future__ import annotations + import os import pickle # nosec B403 -from typing import Collection, Dict, List, Union +from typing import Collection import matplotlib.pyplot as plt import numpy as np @@ -28,7 +30,7 @@ from black_it.calibrator import Calibrator -def _get_samplers_id_table(saving_folder: Union[str, os.PathLike]) -> Dict[str, int]: +def _get_samplers_id_table(saving_folder: str | os.PathLike) -> dict[str, int]: """Get the id table of the samplers from the checkpoint. Args: @@ -46,9 +48,9 @@ def _get_samplers_id_table(saving_folder: Union[str, os.PathLike]) -> Dict[str, def _get_samplers_names( - saving_folder: Union[str, os.PathLike], - ids: List[int], -) -> List[str]: + saving_folder: str | os.PathLike, + ids: list[int], +) -> list[str]: """Get the names of the samplers from their ids and from the checkpoint of the calibration. Args: @@ -69,7 +71,7 @@ def _get_samplers_names( return sampler_names -def plot_convergence(saving_folder: Union[str, os.PathLike]) -> None: +def plot_convergence(saving_folder: str | os.PathLike) -> None: """Plot the loss values sampled by the various methods as a function of the batch number. Args: @@ -107,7 +109,7 @@ def plot_convergence(saving_folder: Union[str, os.PathLike]) -> None: plt.legend(handles, labels, loc="upper right") -def plot_losses(saving_folder: Union[str, os.PathLike]) -> None: +def plot_losses(saving_folder: str | os.PathLike) -> None: """Plot the parameter sampled colored according to their loss value. Args: @@ -136,7 +138,7 @@ def plot_losses(saving_folder: Union[str, os.PathLike]) -> None: g._legend.set_bbox_to_anchor((0.8, 0.5)) -def plot_sampling(saving_folder: Union[str, os.PathLike]) -> None: +def plot_sampling(saving_folder: str | os.PathLike) -> None: """Plot the parameter sampled colored according to the sampling method used to sample them. Args: @@ -170,7 +172,7 @@ def plot_sampling(saving_folder: Union[str, os.PathLike]) -> None: def plot_losses_method_num( - saving_folder: Union[str, os.PathLike], + saving_folder: str | os.PathLike, method_num: int, ) -> None: """Plot the parameter sampled by a specific sampling method, and color them according to their loss value. @@ -208,7 +210,7 @@ def plot_losses_method_num( g._legend.set_bbox_to_anchor((0.8, 0.5)) -def plot_losses_interact(saving_folder: Union[str, os.PathLike]) -> None: +def plot_losses_interact(saving_folder: str | os.PathLike) -> None: """Plot the parameter sampled colored according to their loss value. This plot allows to interactively choose the sampling method. @@ -235,7 +237,7 @@ def plot_losses_interact(saving_folder: Union[str, os.PathLike]) -> None: def plot_sampling_batch_nums( - saving_folder: Union[str, os.PathLike], + saving_folder: str | os.PathLike, batch_nums: Collection[int], ) -> None: """Plot the parameter sampled in specific batches colored according to the sampling method used to sample them. @@ -279,7 +281,7 @@ def plot_sampling_batch_nums( plt.legend(loc=2, handles=handles, labels=sampler_names, bbox_to_anchor=(0.0, 1.8)) -def plot_sampling_interact(saving_folder: Union[str, os.PathLike]) -> None: +def plot_sampling_interact(saving_folder: str | os.PathLike) -> None: """Plot the parameter sampled colored according to the sampling method used to sample them. The method allows to interactively choose the batch numbers included in the plot. diff --git a/black_it/samplers/base.py b/black_it/samplers/base.py index 2c777417..e5ad8d4c 100644 --- a/black_it/samplers/base.py +++ b/black_it/samplers/base.py @@ -15,9 +15,9 @@ # along with this program. If not, see . """This module defines the 'BaseSampler' base class.""" +from __future__ import annotations from abc import ABC, abstractmethod -from typing import List, Optional import numpy as np from numpy.typing import NDArray @@ -35,7 +35,7 @@ class BaseSampler(BaseSeedable, ABC): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, ) -> None: """Initialize the sampler. @@ -120,7 +120,7 @@ def sample( def find_and_get_duplicates( new_points: NDArray[np.float64], existing_points: NDArray[np.float64], - ) -> List: + ) -> list: """Find the points in 'new_points' that are already present in 'existing_points'. Args: diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index 0c2e84fb..e2351ce0 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains the implementation of the best-batch sampler.""" - -from typing import Optional +from __future__ import annotations import numpy as np from numpy.typing import NDArray @@ -41,7 +40,7 @@ class BestBatchSampler(BaseSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, a: float = 3.0, b: float = 1.0, diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index 1a0c5f93..8689c1ee 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -20,8 +20,10 @@ using radial basis functions." Journal of Global optimization 31.1 (2005): 153-171. """ +from __future__ import annotations + from math import factorial -from typing import Callable, Optional, cast +from typing import Callable, cast import numpy as np import scipy.optimize as op @@ -134,7 +136,7 @@ def __init__( max_samples: int, rho0: float = 0.5, p: float = 1.0, - random_state: Optional[int] = None, + random_state: int | None = None, verbose: bool = False, ) -> None: """Initialize the CORS sampler. diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index f61d9db4..2f5d2293 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -16,9 +16,11 @@ """This module contains the implementation of the Gaussian process-based sampling.""" +from __future__ import annotations + import warnings from enum import Enum -from typing import Optional, Tuple, Union, cast +from typing import cast import numpy as np from numpy.typing import NDArray @@ -50,9 +52,9 @@ class GaussianProcessSampler(MLSurrogateSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, - candidate_pool_size: Optional[int] = None, + candidate_pool_size: int | None = None, optimize_restarts: int = 5, acquisition: str = "expected_improvement", jitter: float = 0.1, @@ -79,8 +81,8 @@ def __init__( self.optimize_restarts = optimize_restarts self.acquisition = acquisition self.jitter = jitter - self._gpmodel: Optional[GaussianProcessRegressor] = None - self._fmin: Optional[Union[np.double, float]] = None + self._gpmodel: GaussianProcessRegressor | None = None + self._fmin: np.double | float | None = None @staticmethod def _validate_acquisition(acquisition: str) -> None: @@ -147,7 +149,7 @@ def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 def _predict_mean_std( self, X: NDArray[np.float64], # noqa: N803 - ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: + ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: """Predict mean and standard deviation of a fitted GP. Args: @@ -192,7 +194,7 @@ def get_quantiles( fmin: float, m: NDArray[np.float64], s: NDArray[np.float64], - ) -> Tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]: + ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]: """Quantiles of the Gaussian distribution useful to determine the acquisition function values. Args: diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index f5b4ed6b..3b65013c 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -15,8 +15,10 @@ # along with this program. If not, see . """This module contains the implementation for the Halton sampler.""" +from __future__ import annotations + import itertools -from typing import Iterator, List, Optional +from typing import Iterator import numpy as np from numpy.typing import NDArray @@ -39,7 +41,7 @@ class HaltonSampler(BaseSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, ) -> None: """Initialize the sampler. @@ -55,7 +57,7 @@ def __init__( # drop first N entries to avoid linear correlation self._reset_sequence_index() - def _set_random_state(self, random_state: Optional[int]) -> None: + def _set_random_state(self, random_state: int | None) -> None: """Set the random state (private use). For the Halton sampler, it also resets the sequence index. @@ -165,7 +167,7 @@ class _CachedPrimesCalculator: def __init__(self) -> None: """Initialize the object.""" self._primes_iterator = _PrimesIterator() - self._cached_primes: List[int] = [2] + self._cached_primes: list[int] = [2] def get_n_primes(self, n: int) -> NDArray[np.int64]: """Get the first n primes. diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index a21a6469..e5bc0d89 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """Implementation of the particle swarm sampler.""" -from typing import Optional, cast +from __future__ import annotations + +from typing import cast import numpy as np from numpy.typing import NDArray @@ -59,7 +61,7 @@ class ParticleSwarmSampler(BaseSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, inertia: float = 0.9, c1: float = 0.1, c2: float = 0.1, @@ -95,20 +97,20 @@ def __init__( self._global_minimum_across_samplers = global_minimum_across_samplers # all current particle positions; shape=(nb_particles, space dimensions) - self._curr_particle_positions: Optional[NDArray] = None + self._curr_particle_positions: NDArray | None = None # all current particle velocities; shape=(nb_particles, space dimensions) - self._curr_particle_velocities: Optional[NDArray] = None + self._curr_particle_velocities: NDArray | None = None # best particle positions, i.e. ; shape=(nb_particles, space dimensions) - self._best_particle_positions: Optional[NDArray] = None + self._best_particle_positions: NDArray | None = None # losses of the best positions - self._best_position_losses: Optional[NDArray] = None + self._best_position_losses: NDArray | None = None # particle id of the global best particle position - self._global_best_particle_id: Optional[int] = None + self._global_best_particle_id: int | None = None # best point in parameter space - could be the best across samplers - self._best_point: Optional[NDArray] = None + self._best_point: NDArray | None = None - self._previous_batch_index_start: Optional[int] = None + self._previous_batch_index_start: int | None = None @property def is_set_up(self) -> bool: diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index c4415acf..f7976cac 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -15,7 +15,7 @@ # along with this program. If not, see . """This module contains the implementation of the R-sequence sampler.""" -from typing import Optional +from __future__ import annotations import numpy as np from numpy.typing import NDArray @@ -34,7 +34,7 @@ class RSequenceSampler(BaseSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, ) -> None: """Initialize the sampler. @@ -69,7 +69,7 @@ def compute_phi(cls, nb_dims: int) -> float: phi = pow(1 + phi, 1.0 / (nb_dims + 1)) return phi - def _set_random_state(self, random_state: Optional[int]) -> None: + def _set_random_state(self, random_state: int | None) -> None: """Set the random state (private use). For the RSequence sampler, it also resets the sequence index and the sequence start. diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 2816be40..3f1b4c6d 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """This module contains the implementation of the random forest sampling.""" -from typing import Optional, Tuple, cast +from __future__ import annotations + +from typing import cast import numpy as np from numpy.typing import NDArray @@ -31,9 +33,9 @@ class RandomForestSampler(MLSurrogateSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, - candidate_pool_size: Optional[int] = None, + candidate_pool_size: int | None = None, n_estimators: int = 500, criterion: str = "gini", n_classes: int = 10, @@ -67,7 +69,7 @@ def __init__( self._n_estimators = n_estimators self._criterion = criterion self._n_classes = n_classes - self._classifier: Optional[RandomForestClassifier] = None + self._classifier: RandomForestClassifier | None = None @property def n_estimators(self) -> int: @@ -115,7 +117,7 @@ def prepare_data_for_classifier( existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], num_bins: int, - ) -> Tuple[NDArray[np.float64], NDArray[np.int64], NDArray[np.float64]]: + ) -> tuple[NDArray[np.float64], NDArray[np.int64], NDArray[np.float64]]: """Prepare data for the classifier. Args: diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index 70bbc532..b7e33c13 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -15,9 +15,9 @@ # along with this program. If not, see . """This module defines the 'MLSurrogateSampler' base class.""" +from __future__ import annotations from abc import abstractmethod -from typing import Optional import numpy as np from numpy.typing import NDArray @@ -37,9 +37,9 @@ class MLSurrogateSampler(BaseSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, - candidate_pool_size: Optional[int] = None, + candidate_pool_size: int | None = None, ) -> None: """Initialize the sampler. diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 3e954af5..a493aa65 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -15,8 +15,10 @@ # along with this program. If not, see . """This module contains the implementation of the XGBoost sampling.""" +from __future__ import annotations + import warnings -from typing import Optional, cast +from typing import cast import numpy as np import xgboost as xgb @@ -35,9 +37,9 @@ class XGBoostSampler(MLSurrogateSampler): def __init__( self, batch_size: int, - random_state: Optional[int] = None, + random_state: int | None = None, max_deduplication_passes: int = 5, - candidate_pool_size: Optional[int] = None, + candidate_pool_size: int | None = None, colsample_bytree: float = 0.3, learning_rate: float = 0.1, max_depth: int = 5, @@ -75,7 +77,7 @@ def __init__( self._max_depth = max_depth self._alpha = alpha self._n_estimators = n_estimators - self._xg_regressor: Optional[xgb.XGBRegressor] = None + self._xg_regressor: xgb.XGBRegressor | None = None @property def colsample_bytree(self) -> float: diff --git a/black_it/schedulers/base.py b/black_it/schedulers/base.py index 0fb7cbb3..faf02b88 100644 --- a/black_it/schedulers/base.py +++ b/black_it/schedulers/base.py @@ -15,9 +15,11 @@ # along with this program. If not, see . """This module defines the 'BaseScheduler' base class.""" +from __future__ import annotations + import contextlib from abc import ABC, abstractmethod -from typing import Generator, Optional, Sequence, Tuple +from typing import Generator, Sequence import numpy as np from numpy.typing import NDArray @@ -35,7 +37,7 @@ class BaseScheduler(BaseSeedable, ABC): def __init__( self, samplers: Sequence[BaseSampler], - random_state: Optional[int] = None, + random_state: int | None = None, ) -> None: """Initialize the scheduler. @@ -48,11 +50,11 @@ def __init__( BaseSeedable.__init__(self, random_state) @property - def samplers(self) -> Tuple[BaseSampler, ...]: + def samplers(self) -> tuple[BaseSampler, ...]: """Get the sequence of samplers.""" return self._samplers - def _set_random_state(self, random_state: Optional[int]) -> None: + def _set_random_state(self, random_state: int | None) -> None: """Set the random state (private use).""" super()._set_random_state(random_state) for sampler in self.samplers: diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index 0a245c40..a6c82a84 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -15,8 +15,9 @@ # along with this program. If not, see . """This module includes the implementation of a non-stationary epsilon-greedy agent.""" +from __future__ import annotations -from typing import Optional, SupportsFloat, cast +from typing import SupportsFloat, cast import numpy as np @@ -32,7 +33,7 @@ def __init__( alpha: float, eps: float, initial_values: float = 0.0, - random_state: Optional[int] = None, + random_state: int | None = None, ) -> None: """Initialize the agent object. diff --git a/black_it/schedulers/rl/envs/base.py b/black_it/schedulers/rl/envs/base.py index bbde83f4..963ab860 100644 --- a/black_it/schedulers/rl/envs/base.py +++ b/black_it/schedulers/rl/envs/base.py @@ -15,9 +15,11 @@ # along with this program. If not, see . """This package contains implementations of Gym environments for the RL Scheduler.""" +from __future__ import annotations + from abc import ABC, abstractmethod from queue import Queue -from typing import Any, Dict, List, Optional, SupportsFloat, Tuple, Union +from typing import Any, SupportsFloat import gymnasium as gym import numpy as np @@ -37,7 +39,7 @@ def __init__(self, nb_samplers: int) -> None: self._out_queue: Queue = Queue() self._in_queue: Queue = Queue() - self._curr_best_loss: Optional[float] = None + self._curr_best_loss: float | None = None self.action_space = Discrete(self._nb_samplers) @@ -56,16 +58,16 @@ def get_next_observation(self) -> ObsType: def reset( self, *, - seed: Optional[int] = None, - options: Optional[Dict[str, Any]] = None, - ) -> Tuple[ObsType, Dict[str, Any]]: + seed: int | None = None, + options: dict[str, Any] | None = None, + ) -> tuple[ObsType, dict[str, Any]]: """Reset the environment.""" return self.reset_state(), {} def step( self, action: np.int64, - ) -> Tuple[ObsType, SupportsFloat, bool, bool, Dict[str, Any]]: + ) -> tuple[ObsType, SupportsFloat, bool, bool, dict[str, Any]]: """Do a step.""" _assert(self.action_space.contains(action)) self._out_queue.put(action) @@ -78,6 +80,6 @@ def step( next_obs = self.get_next_observation() return next_obs, reward, False, False, {} - def render(self) -> Optional[Union[RenderFrame, List[RenderFrame]]]: + def render(self) -> RenderFrame | list[RenderFrame] | None: """Render the environment (not implemented).""" raise NotImplementedError diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index 0093ae31..241268e4 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -15,9 +15,11 @@ # along with this program. If not, see . """This module implements the 'RLScheduler' scheduler.""" +from __future__ import annotations + import threading from queue import Queue -from typing import List, Optional, Sequence, Tuple, cast +from typing import List, Sequence, cast import numpy as np from numpy._typing import NDArray @@ -40,7 +42,7 @@ def __init__( samplers: Sequence[BaseSampler], agent: Agent, env: CalibrationEnv, - random_state: Optional[int] = None, + random_state: int | None = None, ) -> None: """Initialize the scheduler.""" self._original_samplers = samplers @@ -56,13 +58,13 @@ def __init__( self._in_queue: Queue = self._env._out_queue self._out_queue: Queue = self._env._in_queue - self._best_param: Optional[float] = None - self._best_loss: Optional[float] = None + self._best_param: float | None = None + self._best_loss: float | None = None - self._agent_thread: Optional[threading.Thread] = None + self._agent_thread: threading.Thread | None = None self._stopped: bool = True - def _set_random_state(self, random_state: Optional[int]) -> None: + def _set_random_state(self, random_state: int | None) -> None: """Set the random state (private use).""" super()._set_random_state(random_state) for sampler in self.samplers: @@ -74,7 +76,7 @@ def _set_random_state(self, random_state: Optional[int]) -> None: def _add_or_get_bootstrap_sampler( cls, samplers: Sequence[BaseSampler], - ) -> Tuple[Sequence[BaseSampler], int]: + ) -> tuple[Sequence[BaseSampler], int]: """Add or retrieve a sampler for bootstrapping. Many samplers do require some "bootstrapping" of the calibration process, i.e. a set of parameters diff --git a/black_it/search_space.py b/black_it/search_space.py index 740dbe63..178d8d70 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains the definition of the search space abstractions.""" - -from typing import List, Union +from __future__ import annotations import numpy as np from numpy.typing import NDArray @@ -27,8 +26,8 @@ class SearchSpace: def __init__( self, - parameters_bounds: Union[NDArray[np.float64], List[List[float]]], - parameters_precision: Union[NDArray[np.float64], List[float]], + parameters_bounds: NDArray[np.float64] | list[list[float]], + parameters_precision: NDArray[np.float64] | list[float], verbose: bool, ) -> None: """Initialize the SearchSpace object. @@ -66,7 +65,7 @@ def __init__( self._parameters_precision = np.array(parameters_precision) # Initialize search grid - self._param_grid: List[NDArray[np.float64]] = [] + self._param_grid: list[NDArray[np.float64]] = [] self._space_size = 1 for i in range(self.dims): new_col = np.arange( @@ -86,8 +85,8 @@ def __init__( @staticmethod def _check_bounds( - parameters_bounds: Union[NDArray[np.float64], List[List[float]]], - parameters_precision: Union[NDArray[np.float64], List[float]], + parameters_bounds: NDArray[np.float64] | list[list[float]], + parameters_precision: NDArray[np.float64] | list[float], ) -> None: """Ensure parameter_bounds and parameter_precision have acceptable values. @@ -143,7 +142,7 @@ def _check_bounds( ) @property - def param_grid(self) -> List[NDArray[np.float64]]: + def param_grid(self) -> list[NDArray[np.float64]]: """Discretized parameter space containing all possible candidates for calibration.""" return self._param_grid diff --git a/black_it/utils/base.py b/black_it/utils/base.py index 70601da9..4006a970 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -15,9 +15,11 @@ # along with this program. If not, see . """This module contains generic utility functions for the library.""" +from __future__ import annotations + import os from json import JSONEncoder -from typing import Any, List, Optional, Type, Union +from typing import Any, Union import numpy as np from numpy.typing import NDArray @@ -27,8 +29,8 @@ def _assert( condition: bool, - error_message: Optional[str] = None, - exception_class: Type[Exception] = Exception, + error_message: str | None = None, + exception_class: type[Exception] = Exception, ) -> None: """Check condition; if false, raise exception with the provided error message.""" if not condition: @@ -59,7 +61,7 @@ def positive_float(arg: float) -> float: def digitize_data( data: NDArray[np.float64], - param_grid: List[NDArray[np.float64]], + param_grid: list[NDArray[np.float64]], ) -> NDArray[np.float64]: """Return a discretized version of the input sorted_array. diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index b466b98d..ff581af1 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -15,10 +15,12 @@ # along with this program. If not, see . """This module contains serialization and deserialization of calibration state with Pandas.""" +from __future__ import annotations + import json import pickle # nosec B403 from pathlib import Path -from typing import Mapping, Optional, Tuple +from typing import Mapping import numpy as np import pandas as pd @@ -30,7 +32,7 @@ from black_it.utils.base import NumpyArrayEncoder, PathLike -def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) -> Tuple: +def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) -> tuple: """Load calibrator data from a given folder. Args: @@ -100,10 +102,10 @@ def save_calibrator_state( ensemble_size: int, N: int, # noqa: N803 D: int, # noqa: N803 - convergence_precision: Optional[float], + convergence_precision: float | None, verbose: bool, - saving_file: Optional[str], - initial_random_seed: Optional[int], + saving_file: str | None, + initial_random_seed: int | None, random_generator_state: Mapping, model_name: str, scheduler: BaseScheduler, diff --git a/black_it/utils/seedable.py b/black_it/utils/seedable.py index 275c84fc..8cf9095d 100644 --- a/black_it/utils/seedable.py +++ b/black_it/utils/seedable.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains the definition of a 'seedable' base class.""" - -from typing import Optional +from __future__ import annotations import numpy as np from numpy.random import default_rng @@ -36,7 +35,7 @@ class BaseSeedable: def __init__( self, - random_state: Optional[int] = None, + random_state: int | None = None, ) -> None: """Initialize the sampler. @@ -44,23 +43,23 @@ def __init__( random_state: the internal state of the sampler, fixing this numbers the object (e.g. a calibrator, a sampler, or a scheduler) behaves deterministically """ - self.__random_state: Optional[int] + self.__random_state: int | None self.__random_generator: np.random.Generator # this triggers the property setter self.random_state = random_state @property - def random_state(self) -> Optional[int]: + def random_state(self) -> int | None: """Get the random state.""" return self.__random_state @random_state.setter - def random_state(self, random_state: Optional[int]) -> None: + def random_state(self, random_state: int | None) -> None: """Set the random state.""" self._set_random_state(random_state) - def _set_random_state(self, random_state: Optional[int]) -> None: + def _set_random_state(self, random_state: int | None) -> None: """Set the random state, private use.""" self.__random_state = random_state self.__random_generator = default_rng(self.random_state) diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index a0e71e96..fc8ca4b4 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -15,13 +15,15 @@ # along with this program. If not, see . """This module contains serialization and deserialization of calibration state with SQLite.""" +from __future__ import annotations + import gzip import io import json import pickle # nosec B403 import sqlite3 from pathlib import Path -from typing import Mapping, Optional, Sequence, Tuple +from typing import Mapping, Sequence import numpy as np from numpy.typing import NDArray @@ -167,7 +169,7 @@ class gz_ndarray(NDArray): # noqa: N801 def load_calibrator_state( checkpoint_path: PathLike, -) -> Tuple: +) -> tuple: """Load the calibration state. Checks that the schema version stored in checkpoint_path has the same value @@ -265,10 +267,10 @@ def save_calibrator_state( ensemble_size: int, N: int, # noqa: N803 D: int, # noqa: N803 - convergence_precision: Optional[float], + convergence_precision: float | None, verbose: bool, - saving_file: Optional[str], - initial_random_seed: Optional[int], + saving_file: str | None, + initial_random_seed: int | None, random_generator_state: Mapping, model_name: str, samplers: Sequence[BaseSampler], diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index e9947811..3e780500 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains utility functions to deal with time series.""" - -from typing import Tuple +from __future__ import annotations import numpy as np import scipy.sparse as sps @@ -88,7 +87,7 @@ def get_mom_ts_1d(time_series: NDArray[np.float64]) -> NDArray[np.float64]: def hp_filter( time_series: NDArray[np.float64], lamb: float = 1600, -) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: """Apply the HP filter to a time series. Args: diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index f472bdfd..87dd9ad3 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -15,8 +15,9 @@ # along with this program. If not, see . """Implementation of the model in (Brock and Hommes, 1998).""" +from __future__ import annotations -from typing import Optional, Sequence +from typing import Sequence import numpy as np from numpy.typing import NDArray @@ -26,7 +27,7 @@ def BH2( # noqa: N802 theta: Sequence[float], N: int, # noqa: N803 - seed: Optional[int], + seed: int | None, ) -> NDArray: """Model from Brock and Hommes 1998. 4.1.2. Fundamentalists versus trend chasers. @@ -88,7 +89,7 @@ def BH2( # noqa: N802 def BH4( # noqa: N802 theta: Sequence[float], N: int, # noqa: N803 - seed: Optional[int], + seed: int | None, ) -> NDArray: """Model from Brock and Hommes 1998. diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index d9a9ec15..2291b951 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -15,9 +15,9 @@ # along with this program. If not, see . """A simple model of a wildfire on a 2D grid.""" +from __future__ import annotations import subprocess -from typing import Optional import numpy as np from numpy.typing import NDArray @@ -26,7 +26,7 @@ def forest_fire( theta: NDArray, N: int, # noqa: N803 - seed: Optional[int] = 0, + seed: int | None = 0, ) -> NDArray: """A simple model of a wildfire on a 2D grid. diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 886e77f7..8d740e81 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -37,16 +37,16 @@ {"susceptible": 995, "infectious": 4, "recovered": 1}, {"susceptible": 995, "infectious": 2, "recovered": 3}] """ +from __future__ import annotations import json import re import subprocess import sys from itertools import chain -from typing import Dict, List -def parse_simulator_output(stdout: str) -> List[Dict[str, int]]: +def parse_simulator_output(stdout: str) -> list[dict[str, int]]: """Filter the output of a simulator execution and convert it to a python list.""" regex = re.compile(r"^EPOCH.*(initial status|ended with)") @@ -62,8 +62,8 @@ def parse_simulator_output(stdout: str) -> List[Dict[str, int]]: def _build_simulator_cmdline( docker_image_name: str, - sim_params: Dict[str, str], -) -> List[str]: + sim_params: dict[str, str], +) -> list[str]: """Convert a configuration object in a list of command line parameters. Accepts a configuration object for the simulation and returns a long term @@ -105,8 +105,8 @@ def _build_simulator_cmdline( def execute_simulator( path_to_simulator: str, - sim_params: Dict[str, str], -) -> List[Dict[str, int]]: + sim_params: dict[str, str], +) -> list[dict[str, int]]: """Execute the simulator with the given parameters, and return a structured output. - the simulator parameters are converted via _build_simulator_cmdline() @@ -133,7 +133,7 @@ def execute_simulator( return _execute_simulator_subprocess(args) -def _execute_simulator_subprocess(args: List[str]) -> List[Dict[str, int]]: +def _execute_simulator_subprocess(args: list[str]) -> list[dict[str, int]]: """Execute the simulator and convert its output in structured form via parse_simulator_output(). REMARKS: diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index dc51e0ad..4f84d711 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -15,7 +15,7 @@ # along with this program. If not, see . """SIR models written in C and run in Docker containers.""" -from typing import Optional +from __future__ import annotations import numpy as np from numpy.typing import NDArray @@ -23,7 +23,7 @@ from . import simlib -def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, N803 +def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N803 """SIR_docker. C++ SIR model run in Docker container. @@ -47,7 +47,7 @@ def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, def SIR_w_breaks( # noqa: N802 theta: NDArray, N: int, # noqa: N803 - seed: Optional[int] = None, + seed: int | None = None, ) -> NDArray: """SIR_docker_w_breaks.""" breaktime = int(theta[0]) diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index 46e38e87..a65e9c0d 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -15,7 +15,7 @@ # along with this program. If not, see . """SIR models written in Python.""" -from typing import Optional +from __future__ import annotations import ndlib.models.epidemics as ep import networkx as nx @@ -24,7 +24,7 @@ from numpy.typing import NDArray -def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, N803 +def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N803 """SIR model. 0 theta = [#LOC CONNECTIONS, @@ -73,7 +73,7 @@ def SIR(theta: NDArray, N: int, seed: Optional[int]) -> NDArray: # noqa: N802, def SIR_w_breaks( # noqa: N802 theta: NDArray, N: int, # noqa: N803 - seed: Optional[int], + seed: int | None, ) -> NDArray: """SIR model with structural breaks. diff --git a/tests/test_losses/test_base.py b/tests/test_losses/test_base.py index 4f365b10..2a3d0305 100644 --- a/tests/test_losses/test_base.py +++ b/tests/test_losses/test_base.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """This module contains tests for the base loss_functions module.""" -from typing import Callable, List, Optional +from __future__ import annotations + +from typing import Callable import numpy as np import pytest @@ -33,8 +35,8 @@ class MyCustomLoss(BaseLoss): def __init__( self, loss_constant: float, - coordinate_weights: Optional[NDArray] = None, - coordinate_filters: Optional[List[Optional[Callable]]] = None, + coordinate_weights: NDArray | None = None, + coordinate_filters: list[Callable | None] | None = None, ) -> None: """Initialize the custom loss.""" super().__init__(coordinate_weights, coordinate_filters) @@ -55,8 +57,8 @@ def compute_loss_1d( nb_sim_rows: int = 10 nb_real_rows: int = 10 - coordinate_weights: Optional[NDArray[np.float64]] - coordinate_filters: Optional[List[Optional[Callable]]] + coordinate_weights: NDArray[np.float64] | None + coordinate_filters: list[Callable | None] | None # instance attributes loss: MyCustomLoss diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index f87aab0f..e9e1d257 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains tests for the GSL-div loss.""" - -from typing import Tuple +from __future__ import annotations import numpy as np from hypothesis import given @@ -50,12 +49,12 @@ class TestDiscretize: """Test the 'discretize' function.""" @given(discretize_args()) - def test_discretize_time_series_any_args(self, args: Tuple) -> None: + def test_discretize_time_series_any_args(self, args: tuple) -> None: """Test the case with randomly generated args.""" GslDivLoss.discretize(*args) @given(discretize_args()) - def test_discretize_time_series_partition(self, args: Tuple) -> None: + def test_discretize_time_series_partition(self, args: tuple) -> None: """Test that discretize computes the right number of partitions.""" time_series, nb_values, start_index, stop_index = args actual = GslDivLoss.discretize(time_series, nb_values, start_index, stop_index) @@ -65,7 +64,7 @@ def test_discretize_time_series_partition(self, args: Tuple) -> None: assert (actual <= max_nb_values).all() @given(discretize_args()) - def test_ordering_is_preserved(self, args: Tuple) -> None: + def test_ordering_is_preserved(self, args: tuple) -> None: """Test that the time series ordering is preserved when discretized.""" time_series, nb_values, start_index, stop_index = args increasing_time_series = np.sort(time_series) @@ -91,7 +90,7 @@ class TestGetWords: """Test the 'get_words' function.""" @given(get_words_args()) - def test_get_words(self, args: Tuple) -> None: + def test_get_words(self, args: tuple) -> None: """Test the case with randomly generated args.""" GslDivLoss.get_words(*args) diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index 86b2d1f2..19aa61aa 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -15,10 +15,12 @@ # along with this program. If not, see . """This module contains utilities for the test_plot package.""" +from __future__ import annotations + import logging from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Callable, List, Optional +from typing import Any, Callable import matplotlib.pyplot as plt from matplotlib.testing.compare import compare_images @@ -29,8 +31,8 @@ class BasePlotTest: plotting_function: Callable expected_image: Path - tolerance: Optional[float] = 0.000 - args: List[Any] = [] + tolerance: float | None = 0.000 + args: list[Any] = [] def test_run(self) -> None: """Run the test.""" diff --git a/tests/test_plot/test_plot_descriptive_statistics.py b/tests/test_plot/test_plot_descriptive_statistics.py index 513c0c82..1cb3c90a 100644 --- a/tests/test_plot/test_plot_descriptive_statistics.py +++ b/tests/test_plot/test_plot_descriptive_statistics.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """This test module contains tests for the plot_descriptive_statistics.py module.""" -from typing import Any, List +from __future__ import annotations + +from typing import Any import numpy as np @@ -30,7 +32,7 @@ class TestTsStats(BasePlotTest): """Test 'ts_stats' plotting function.""" plotting_function = ts_stats - args: List[Any] = [] + args: list[Any] = [] expected_image = PLOT_DIR / "ts_stats-expected.png" def setup(self) -> None: diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 9e2a5e26..ca959b88 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -14,7 +14,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """This module contains tests for the Gaussian process sampler.""" -from typing import Optional, Tuple, cast +from __future__ import annotations + +from typing import cast import numpy as np import pytest @@ -35,7 +37,7 @@ def setup(self) -> None: def _construct_fake_grid( cls, n: int = 3, - ) -> Tuple[NDArray[np.float64], NDArray[np.float64]]: + ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: """Construct a fake grid of evaluated losses.""" xs = np.linspace(0, 1, n) ys = np.linspace(0, 1, n) @@ -81,7 +83,7 @@ def test_gaussian_process_2d( self, acquisition: str, optimize_restarts: int, - expected_params: Optional[NDArray], + expected_params: NDArray | None, ) -> None: """Test the Gaussian process sampler, 2d.""" sampler = GaussianProcessSampler( diff --git a/tests/test_search_space.py b/tests/test_search_space.py index 791a935f..3555a519 100644 --- a/tests/test_search_space.py +++ b/tests/test_search_space.py @@ -15,8 +15,7 @@ # along with this program. If not, see . """This module contains tests for the black_it.SearchSpace class.""" - -from typing import List +from __future__ import annotations import numpy as np import pytest @@ -42,9 +41,9 @@ ], ) def test_search_space_successful( - lower: List[float], - upper: List[float], - precision: List[float], + lower: list[float], + upper: list[float], + precision: list[float], expected_num_params: int, expected_cardinality: int, ) -> None: diff --git a/tests/utils/base.py b/tests/utils/base.py index ec6b65e3..6bfb418b 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -15,13 +15,15 @@ # along with this program. If not, see . """Generic utility functions.""" +from __future__ import annotations + import dataclasses import shutil import signal import subprocess # nosec B404 import sys from functools import wraps -from typing import Callable, List, Type, Union +from typing import Callable import pytest @@ -38,7 +40,7 @@ class PopenResult: def run_process( - command: List[str], + command: list[str], timeout: float = DEFAULT_SUBPROCESS_TIMEOUT, ) -> PopenResult: """Run a process, and wait for it to stop. @@ -132,7 +134,7 @@ def wrapper(*args, **kwargs): the Pytest wrapper for functions or classes. """ - def decorator(pytest_func_or_cls: Union[Callable, Type]) -> Callable: + def decorator(pytest_func_or_cls: Callable | type) -> Callable: """Implement the decorator. Args: diff --git a/tests/utils/docs.py b/tests/utils/docs.py index b5f03e8c..7daf622b 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -14,11 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """This module contains helper function to extract code from the .md files.""" +from __future__ import annotations import json from io import StringIO from pathlib import Path -from typing import Dict, List, Optional from unittest.mock import MagicMock import mistletoe @@ -27,7 +27,7 @@ MISTLETOE_CODE_BLOCK_ID = "CodeFence" -def code_block_filter(block_dict: Dict, language: Optional[str] = None) -> bool: +def code_block_filter(block_dict: dict, language: str | None = None) -> bool: """Check Mistletoe block is a code block. Args: @@ -42,12 +42,12 @@ def code_block_filter(block_dict: Dict, language: Optional[str] = None) -> bool: ) -def python_code_block_filter(block_dict: Dict) -> bool: +def python_code_block_filter(block_dict: dict) -> bool: """Filter Python code blocks.""" return code_block_filter(block_dict, language="python") -def code_block_extractor(child_dict: Dict) -> str: +def code_block_extractor(child_dict: dict) -> str: """Extract Mistletoe code block from Mistletoe CodeFence child.""" # we assume that 'children' of CodeFence child has only one child (may be wrong) assert len(child_dict["children"]) == 1 @@ -58,9 +58,9 @@ class BaseTestMarkdownDocs: """Base test class for testing Markdown documents.""" DOC_PATH: Path - blocks: List[Dict] - code_blocks: List[str] - python_blocks: List[str] + blocks: list[dict] + code_blocks: list[str] + python_blocks: list[str] @classmethod def setup_class(cls) -> None: @@ -81,8 +81,8 @@ def setup_class(cls) -> None: class BasePythonMarkdownDocs(BaseTestMarkdownDocs): """Test Markdown documentation by running Python snippets in sequence.""" - locals_dict: Dict - globals_dict: Dict + locals_dict: dict + globals_dict: dict @classmethod def setup_class(cls) -> None: @@ -94,7 +94,7 @@ def setup_class(cls) -> None: cls.locals_dict = {} cls.globals_dict = {} - def _assert(self, locals_: Dict, *mocks: MagicMock) -> None: + def _assert(self, locals_: dict, *mocks: MagicMock) -> None: """Do assertions after Python code execution.""" def test_python_blocks(self, *mocks: MagicMock) -> None: diff --git a/tests/utils/strategies.py b/tests/utils/strategies.py index f383df11..afefd7e6 100644 --- a/tests/utils/strategies.py +++ b/tests/utils/strategies.py @@ -15,7 +15,9 @@ # along with this program. If not, see . """Hypothesis strategies for testing.""" -from typing import Callable, Tuple +from __future__ import annotations + +from typing import Callable import hypothesis.extra.numpy import hypothesis.strategies as st @@ -48,7 +50,7 @@ def discretize_args( draw: Callable, max_length: int = MAX_TIME_SERIES_LENGTH, max_nb_values: int = MAX_NB_VALUES, -) -> Tuple: +) -> tuple: """Return the strategies for the arguments of the 'discretize' function. Args: @@ -71,7 +73,7 @@ def get_words_args( draw: Callable, max_length: int = MAX_TIME_SERIES_LENGTH, max_word_length: int = MAX_WORD_LENGTH, -) -> Tuple: +) -> tuple: """Return the strategies for the arguments of the 'discretize' function. Args: From fbdb708a2ad41529483509e9c651033d481ccb8e Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 11:08:24 +0200 Subject: [PATCH 28/56] lint: fix flake8-no-pep420 errors To reproduce the errors: ``` ruff check --fix --select "INP" black_it tests examples scripts ``` The error were fixed by adding __init__.py whenever was needed. In two cases, namely "examples/docker-sir.py" and "examples/models/simple_models.py", we ignored the INP001 errors. In particular 'docker-sir.py' and 'main.py' are supposed to be run as script, so it does not make sense to have them as part of a package. For some reason, simple_models.py was not considered part of the 'models' package, even after adding __init__.py; hence we added it to the ignore list. --- .ruff.toml | 8 +++++++- examples/models/__init__.py | 17 +++++++++++++++++ examples/models/economics/__init__.py | 17 +++++++++++++++++ examples/models/forest_fire/__init__.py | 17 +++++++++++++++++ examples/models/sir/__init__.py | 17 +++++++++++++++++ tests/fixtures/__init__.py | 17 +++++++++++++++++ 6 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 examples/models/__init__.py create mode 100644 examples/models/economics/__init__.py create mode 100644 examples/models/forest_fire/__init__.py create mode 100644 examples/models/sir/__init__.py create mode 100644 tests/fixtures/__init__.py diff --git a/.ruff.toml b/.ruff.toml index ceb164a3..55ff8fed 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] # Allow autofix for all enabled rules (when `--fix`) is provided. @@ -45,6 +45,12 @@ preview = true # Assume Python 3.8 target-version = "py38" +[per-file-ignores] +"examples/docker-sir.py" = ["INP001"] +"examples/main.py" = ["INP001"] +"examples/models/simple_models.py" = ["INP001"] + + [mccabe] # Unlike Flake8, default to a complexity level of 10. max-complexity = 10 diff --git a/examples/models/__init__.py b/examples/models/__init__.py new file mode 100644 index 00000000..67faa561 --- /dev/null +++ b/examples/models/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""This is a package containing example agent-based models for the package.""" diff --git a/examples/models/economics/__init__.py b/examples/models/economics/__init__.py new file mode 100644 index 00000000..74070f49 --- /dev/null +++ b/examples/models/economics/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""This is a package containing economic agent-based model.""" diff --git a/examples/models/forest_fire/__init__.py b/examples/models/forest_fire/__init__.py new file mode 100644 index 00000000..2025b2b5 --- /dev/null +++ b/examples/models/forest_fire/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""This is a package containing a forest fire model.""" diff --git a/examples/models/sir/__init__.py b/examples/models/sir/__init__.py new file mode 100644 index 00000000..d8bc05a5 --- /dev/null +++ b/examples/models/sir/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""This is a package containing SIR models.""" diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..d18986a8 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,17 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test fixtures for the black-it project.""" From 6b359689b326fdbafdb908b508b3200be4c214a8 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 12:29:14 +0200 Subject: [PATCH 29/56] dev: add examples/ folder in copyright-checked files --- scripts/check_copyright.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/check_copyright.py b/scripts/check_copyright.py index 84be3d72..84c2cf04 100755 --- a/scripts/check_copyright.py +++ b/scripts/check_copyright.py @@ -70,6 +70,8 @@ def check_copyright(file: Path) -> bool: if __name__ == "__main__": exclude_files = { Path("scripts", "__init__.py"), + Path("examples", "docker-sir.py"), + Path("examples", "main.py"), *Path("scripts", "whitelists/").glob("*.py"), } python_files = filter( @@ -78,6 +80,7 @@ def check_copyright(file: Path) -> bool: Path("black_it").glob("**/*.py"), Path("tests").glob("**/*.py"), Path("scripts").glob("**/*.py"), + Path("examples").glob("**/*.py"), ), ) From 785ee29b3b4510bca7555946fc585d4cbe5cfe20 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 12:56:39 +0200 Subject: [PATCH 30/56] lint: fix flake8-print errors To reproduce the errors: ``` ruff check --select "T20" black_it tests examples scripts ``` The flake8-print errors have been ignored for some files (e.g. the examples script and development scripts). However, fixing the errors in the library code would have implied conceptual changes that have to be handled separately from the current workline of fixing linting checks. Therefore, some files were added in the per-file-ignores section, but we also added "T" in the ignore list, denoting errors that we explicitly ignore but are of different nature from the errors in the examples/developmeent scripts. --- .ruff.toml | 11 +++++++---- tests/test_samplers/test_xgboost.py | 1 - 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 55ff8fed..9450cfc0 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,5 +1,5 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP"] -ignore = ["ANN101", "ANN102", "E203", "S", "FBT"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE"] +ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. #fixable = ["ALL"] @@ -46,9 +46,12 @@ preview = true target-version = "py38" [per-file-ignores] -"examples/docker-sir.py" = ["INP001"] -"examples/main.py" = ["INP001"] +"examples/docker-sir.py" = ["INP001", "T201"] +"examples/main.py" = ["INP001", "T201"] "examples/models/simple_models.py" = ["INP001"] +"examples/models/forest_fire/forest_fire.py" = ["T201"] +"examples/models/sir/simlib.py" = ["T201"] +"scripts/check_copyright.py" = ["T201"] [mccabe] diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index 0628c3dd..77151a71 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -62,7 +62,6 @@ def test_xgboost_2d() -> None: verbose=False, ) new_params = sampler.sample(param_grid, xys, losses) - print(new_params) assert np.allclose(expected_params, new_params) From b51dd16c5dcf8e85d75c5d13c1f4f22dc34a4c1f Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 14:23:27 +0200 Subject: [PATCH 31/56] lint: fix flake8-pytest-style errors To reproduce the errors: ``` ruff check --select "PT" black_it tests examples scripts ``` All minor fixes; some error were fixed automatically with the --fix flag. --- .ruff.toml | 2 +- tests/test_examples/test_docker_sir.py | 2 +- tests/test_losses/test_msm.py | 8 ++++---- tests/test_samplers/test_gaussian_process.py | 18 +++++++++--------- tests/test_search_space.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 9450cfc0..5337a2fc 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/tests/test_examples/test_docker_sir.py b/tests/test_examples/test_docker_sir.py index b78f69be..8b4eafc5 100644 --- a/tests/test_examples/test_docker_sir.py +++ b/tests/test_examples/test_docker_sir.py @@ -24,7 +24,7 @@ EXAMPLE_DOCKER_SIR_SCRIPT_PATH = ROOT_DIR / "examples" / "docker-sir.py" -@pytest.mark.e2e +@pytest.mark.e2e() @requires_docker @skip_on_windows class TestDockerSirMainExample(BaseMainExampleTestClass): diff --git a/tests/test_losses/test_msm.py b/tests/test_losses/test_msm.py index 5744d0c5..86cac771 100644 --- a/tests/test_losses/test_msm.py +++ b/tests/test_losses/test_msm.py @@ -107,14 +107,14 @@ def custom_moment_calculator(time_series: NDArray) -> NDArray: random_mat = np.random.rand(dimension, dimension) wrong_covariance_matrix = random_mat.T.dot(random_mat) + loss_func = MethodOfMomentsLoss( + moment_calculator=custom_moment_calculator, + covariance_mat=wrong_covariance_matrix, + ) with pytest.raises( ValueError, match=re.escape( "The size of the covariance matrix (3) and the number of moments (1) should be identical", ), ): - loss_func = MethodOfMomentsLoss( - moment_calculator=custom_moment_calculator, - covariance_mat=wrong_covariance_matrix, - ) loss_func.compute_loss(series_sim[None, :, :], series_real) diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index ca959b88..2074c66e 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -55,28 +55,28 @@ def _construct_fake_grid( return np.asarray(xys_list), np.asarray(losses_list) @pytest.mark.parametrize( - "acquisition,optimize_restarts,expected_params", + ("acquisition", "optimize_restarts", "expected_params"), [ - [ + ( "mean", 1, np.array([[0.0, 0.01], [0.01, 0.01], [0.0, 0.02], [0.01, 0.0]]), - ], - [ + ), + ( "mean", 3, np.array([[0.0, 0.01], [0.01, 0.01], [0.0, 0.02], [0.01, 0.0]]), - ], - [ + ), + ( "expected_improvement", 1, np.array([[0.0, 0.01], [0.0, 0.02], [0.01, 0.01], [0.01, 0.0]]), - ], - [ + ), + ( "expected_improvement", 3, np.array([[0.0, 0.01], [0.0, 0.02], [0.01, 0.01], [0.01, 0.0]]), - ], + ), ], ) def test_gaussian_process_2d( diff --git a/tests/test_search_space.py b/tests/test_search_space.py index 3555a519..54201958 100644 --- a/tests/test_search_space.py +++ b/tests/test_search_space.py @@ -33,7 +33,7 @@ @pytest.mark.parametrize( - "lower,upper,precision,expected_num_params,expected_cardinality", + ("lower", "upper", "precision", "expected_num_params", "expected_cardinality"), [ pytest.param([0], [1], [0.5], 1, 3, id="single_param"), pytest.param([0], [100], [1], 1, 101, id="show_that_size_is_off_by_one"), From 48024dafb5643b8f0ab7fe4e487d8236e5355c9c Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 14:25:52 +0200 Subject: [PATCH 32/56] lint: fix flake8-return errors To reproduce the errors: ``` ruff check --select "RET" black_it tests examples scripts ``` The errors were automatically fixed by running the command with the --fix flag. --- .ruff.toml | 2 +- black_it/calibrator.py | 7 ++----- black_it/loss_functions/fourier.py | 12 +++--------- black_it/loss_functions/gsl_div.py | 3 +-- black_it/loss_functions/likelihood.py | 9 +++------ black_it/loss_functions/minkowski.py | 4 +--- black_it/loss_functions/msm.py | 3 +-- black_it/plot/plot_results.py | 4 +--- black_it/samplers/cors.py | 6 ++---- black_it/samplers/gaussian_process.py | 4 +--- black_it/samplers/surrogate.py | 3 +-- black_it/samplers/xgboost.py | 4 +--- black_it/schedulers/rl/rl_scheduler.py | 3 +-- black_it/utils/time_series.py | 8 ++------ examples/models/forest_fire/forest_fire.py | 3 +-- examples/models/sir/simlib.py | 8 ++------ examples/models/sir/sir_docker.py | 8 ++------ tests/test_docs/base.py | 4 +--- tests/test_losses/test_msm.py | 6 ++---- 19 files changed, 29 insertions(+), 72 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 5337a2fc..1bf13fb8 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 2e3247dc..19e54bab 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -334,13 +334,11 @@ def simulate_model(self, params: NDArray) -> NDArray: simulated_data = np.array(simulated_data_list) - simulated_data = np.reshape( + return np.reshape( simulated_data, (params.shape[0], self.ensemble_size, self.N, self.D), ) - return simulated_data - def calibrate(self, n_batches: int) -> tuple[NDArray, NDArray]: """Run calibration for n batches. @@ -477,11 +475,10 @@ def check_convergence( Returns: True if the calibration converged, False otherwise. """ - converged = ( + return ( np.round(np.min(losses_samp[:n_sampled_params]), convergence_precision) == 0.0 ) - return converged def create_checkpoint(self, file_name: str | os.PathLike) -> None: """Save the current state of the object. diff --git a/black_it/loss_functions/fourier.py b/black_it/loss_functions/fourier.py index 8f5a9623..962e0639 100644 --- a/black_it/loss_functions/fourier.py +++ b/black_it/loss_functions/fourier.py @@ -55,9 +55,7 @@ def ideal_low_pass_filter( # ideal low-pass filter mask = np.zeros(signal_frequencies.shape[0]) mask[:n] = 1.0 - filtered_frequencies = signal_frequencies * mask - - return filtered_frequencies + return signal_frequencies * mask def gaussian_low_pass_filter( @@ -80,9 +78,7 @@ def gaussian_low_pass_filter( # gaussian low-pass filter mask = np.exp(-np.arange(signal_frequencies.shape[0]) ** 2 / (2 * sigma**2)) - filtered_frequencies = signal_frequencies * mask - - return filtered_frequencies + return signal_frequencies * mask class FourierLoss(BaseLoss): @@ -147,6 +143,4 @@ def compute_loss_1d( f_sim_data = np.array(f_sim_data).mean(0) - loss_1d = np.sqrt(np.sum((abs(f_sim_data - f_real_data)) ** 2) / ts_length) - - return loss_1d + return np.sqrt(np.sum((abs(f_sim_data - f_real_data)) ** 2) / ts_length) diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index af8aa0e6..877c493d 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -287,8 +287,7 @@ def get_words_est_prob(time_series: NDArray[np.float64]) -> NDArray[np.float64]: estimate of probabilities """ _, count = np.unique(time_series, return_counts=True) - est_p = np.divide(count, np.sum(count)) - return est_p + return np.divide(count, np.sum(count)) @staticmethod def get_sh_entr(probs: NDArray[np.float64], log_base: float) -> float: diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 45f8b7fd..2796a3de 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -28,8 +28,7 @@ def kernel(sq_dist: NDArray[np.float64], h: float, d: int) -> NDArray[np.float64]: """Compute a kernel density estimation using a Gaussian kernel.""" - k = np.exp(-(sq_dist / (2 * h**2))) / (h**d * (2 * np.pi) ** (d / 2.0)) - return k + return np.exp(-(sq_dist / (2 * h**2))) / (h**d * (2 * np.pi) ** (d / 2.0)) class LikelihoodLoss(BaseLoss): @@ -60,14 +59,12 @@ def __init__( @staticmethod def _get_bandwidth_silverman(n: int, d: int) -> float: """Return a reasonable bandwidth value computed using the Silverman's rule of thumb.""" - h = ((n * (d + 2)) / 4) ** (-1 / (d + 4)) - return h + return ((n * (d + 2)) / 4) ** (-1 / (d + 4)) @staticmethod def _get_bandwidth_scott(n: int, d: int) -> float: """Return a reasonable bandwidth value computed using the Scott's rule of thumb.""" - h = n ** (-1 / (d + 4)) - return h + return n ** (-1 / (d + 4)) def compute_loss( self, diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index 2a624094..3376dddd 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -70,6 +70,4 @@ def compute_loss_1d( # average simulated time series sim_data_ensemble = sim_data_ensemble.mean(axis=0) - loss_1d = minkowski(sim_data_ensemble, real_data, p=self.p) - - return loss_1d + return minkowski(sim_data_ensemble, real_data, p=self.p) diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index c225b3c1..e93fba2e 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -186,8 +186,7 @@ def compute_loss_1d( g = real_mom_1d - sim_mom_1d if self._covariance_mat == _CovarianceMatrixType.IDENTITY.value: - loss_1d = g.dot(g) - return loss_1d + return g.dot(g) if self._covariance_mat == _CovarianceMatrixType.INVERSE_VARIANCE.value: W = np.diag( # noqa: N806 1.0 diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index fb3fde69..45aabf75 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -42,9 +42,7 @@ def _get_samplers_id_table(saving_folder: str | os.PathLike) -> dict[str, int]: with open(os.path.join(saving_folder, "samplers_pickled.pickle"), "rb") as f: method_list = pickle.load(f) # nosec B301 - samplers_id_table = Calibrator._construct_samplers_id_table(method_list) - - return samplers_id_table + return Calibrator._construct_samplers_id_table(method_list) def _get_samplers_names( diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index 8689c1ee..d2e4a806 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -52,8 +52,7 @@ def cubetobox( space_bounds: NDArray, ) -> NDArray[np.float64]: """Go from normalized values (unit cube) to absolute values (box).""" - box_points = space_bounds[0] + X * (space_bounds[1] - space_bounds[0]) - return box_points + return space_bounds[0] + X * (space_bounds[1] - space_bounds[0]) def boxtocube( @@ -61,8 +60,7 @@ def boxtocube( space_bounds: NDArray, ) -> NDArray[np.float64]: """Go from absolute values (box) to normalized values (unit cube).""" - cube_points = (X - space_bounds[0]) / (space_bounds[1] - space_bounds[0]) - return cube_points + return (X - space_bounds[0]) / (space_bounds[1] - space_bounds[0]) def rbf(points: NDArray[np.float64], losses: NDArray[np.float64]) -> Callable: diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 2f5d2293..0fefe81d 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -184,9 +184,7 @@ def _predict_EI( # noqa: N802 phi, Phi, u = self.get_quantiles(jitter, fmin, m, s) # noqa: N806 - f_acqu = s * (u * Phi + phi) - - return f_acqu + return s * (u * Phi + phi) @staticmethod def get_quantiles( diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index b7e33c13..dd7cb254 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -69,7 +69,7 @@ def sample_candidates( existing_losses: NDArray[np.float64], ) -> NDArray[np.float64]: """Get a large pool of candidate parameters.""" - candidates = RandomUniformSampler( + return RandomUniformSampler( candidate_pool_size, random_state=self._get_random_seed(), ).sample_batch( @@ -78,7 +78,6 @@ def sample_candidates( existing_points, existing_losses, ) - return candidates @abstractmethod def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: N803 diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index a493aa65..ab83d970 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -150,6 +150,4 @@ def predict(self, X: NDArray[np.float64]) -> NDArray[np.float64]: # noqa: N803 _ = xgb.DMatrix(data=X) self._xg_regressor = cast(xgb.XGBRegressor, self._xg_regressor) - predictions = self._xg_regressor.predict(X) - - return predictions + return self._xg_regressor.predict(X) diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index 241268e4..64b08081 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -130,8 +130,7 @@ def get_next_sampler(self) -> BaseSampler: # first call, return halton sampler return self.samplers[self._halton_sampler_id] chosen_sampler_id = self._in_queue.get() - chosen_sampler = self.samplers[chosen_sampler_id] - return chosen_sampler + return self.samplers[chosen_sampler_id] def update( self, diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index 3e780500..e358642d 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -134,10 +134,7 @@ def log_and_hp_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: Returns: the filtered time series """ - transformed_series = ( - np.log(time_series) - hp_filter(np.log(time_series), lamb=1600)[1] - ) - return transformed_series + return np.log(time_series) - hp_filter(np.log(time_series), lamb=1600)[1] def diff_log_demean_filter(time_series: NDArray[np.float64]) -> NDArray[np.float64]: @@ -151,5 +148,4 @@ def diff_log_demean_filter(time_series: NDArray[np.float64]) -> NDArray[np.float """ log = np.log(time_series) diff_log = np.diff(log, prepend=log[0]) - transformed_series = diff_log - np.mean(diff_log) - return transformed_series + return diff_log - np.mean(diff_log) diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 2291b951..7a51a934 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -76,8 +76,7 @@ def forest_fire( splitted_line = line.split() results.append(float(splitted_line[-1])) - results = np.array([results]).T - return results + return np.array([results]).T if __name__ == "__main__": diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 8d740e81..209e611b 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -91,7 +91,7 @@ def _build_simulator_cmdline( Returns: the arguments for the Docker CLI """ - args = ["docker", "run", "--rm", docker_image_name] + list( + return ["docker", "run", "--rm", docker_image_name] + list( chain.from_iterable( ( (f"--{argname}", str(argvalue)) @@ -100,8 +100,6 @@ def _build_simulator_cmdline( ), ) - return args - def execute_simulator( path_to_simulator: str, @@ -157,9 +155,7 @@ def _execute_simulator_subprocess(args: list[str]) -> list[dict[str, int]]: exit(res.returncode) - simulation_output = parse_simulator_output(res.stdout) - - return simulation_output + return parse_simulator_output(res.stdout) def run_single_simulation() -> None: diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index 4f84d711..dee06983 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -39,9 +39,7 @@ def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N80 } res = simlib.execute_simulator("bancaditalia/abmsimulator", sim_params) - ret = np.array([(x["susceptible"], x["infectious"], x["recovered"]) for x in res]) - - return ret + return np.array([(x["susceptible"], x["infectious"], x["recovered"]) for x in res]) def SIR_w_breaks( # noqa: N802 @@ -68,6 +66,4 @@ def SIR_w_breaks( # noqa: N802 } res = simlib.execute_simulator("bancaditalia/abmsimulator", sim_params) - ret = np.array([(x["susceptible"], x["infectious"], x["recovered"]) for x in res]) - - return ret + return np.array([(x["susceptible"], x["infectious"], x["recovered"]) for x in res]) diff --git a/tests/test_docs/base.py b/tests/test_docs/base.py index 9f8e754a..05c4dc69 100644 --- a/tests/test_docs/base.py +++ b/tests/test_docs/base.py @@ -42,6 +42,4 @@ def extract_example_code(self) -> str: example_code = re.sub('""".*"""\n', "", example_code) # remove if __name__ == "__main__": - example_code = example_code.replace('if __name__ == "__main__":\n', "") - - return example_code + return example_code.replace('if __name__ == "__main__":\n', "") diff --git a/tests/test_losses/test_msm.py b/tests/test_losses/test_msm.py index 86cac771..de8c3797 100644 --- a/tests/test_losses/test_msm.py +++ b/tests/test_losses/test_msm.py @@ -86,8 +86,7 @@ def test_msm_custom_calculator() -> None: series_real = np.array([[1, 2]]).T def custom_moment_calculator(time_series: NDArray) -> NDArray: - moments = np.array([np.mean(time_series)]) - return moments + return np.array([np.mean(time_series)]) loss_func = MethodOfMomentsLoss(moment_calculator=custom_moment_calculator) loss = loss_func.compute_loss(series_sim[None, :, :], series_real) @@ -100,8 +99,7 @@ def test_msm_custom_calculator_wrong_shape_covariance_matrix() -> None: series_real = np.array([[1, 2]]).T def custom_moment_calculator(time_series: NDArray) -> NDArray: - moments = np.array([np.mean(time_series)]) - return moments + return np.array([np.mean(time_series)]) dimension = 3 random_mat = np.random.rand(dimension, dimension) From 7d36d14faa11c73d0019510583709686f2ca8215 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 14:55:56 +0200 Subject: [PATCH 33/56] lint: fix flake8-self errors To reproduce the errors: ``` ruff check --select "SLF" black_it tests examples scripts ``` Some errors were fixed automatically by using the flag --fix, while others manually (e.g. g.legend in place of g._legend in plot module). --- .ruff.toml | 2 +- black_it/calibrator.py | 2 +- black_it/plot/plot_results.py | 10 +++++----- black_it/schedulers/rl/rl_scheduler.py | 6 +++--- tests/test_samplers/test_gaussian_process.py | 4 +++- tests/test_samplers/test_xgboost.py | 2 +- tests/test_utils/test_seedable.py | 8 ++++++-- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 1bf13fb8..6311c152 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 19e54bab..7d9a6802 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -214,7 +214,7 @@ def set_samplers(self, samplers: list[BaseSampler]) -> None: """ # overwrite the list of samplers - self.scheduler._samplers = tuple(samplers) + self.scheduler._samplers = tuple(samplers) # noqa: SLF001 self.update_samplers_id_table(samplers) def set_scheduler(self, scheduler: BaseScheduler) -> None: diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 45aabf75..8180a81a 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -42,7 +42,7 @@ def _get_samplers_id_table(saving_folder: str | os.PathLike) -> dict[str, int]: with open(os.path.join(saving_folder, "samplers_pickled.pickle"), "rb") as f: method_list = pickle.load(f) # nosec B301 - return Calibrator._construct_samplers_id_table(method_list) + return Calibrator._construct_samplers_id_table(method_list) # noqa: SLF001 def _get_samplers_names( @@ -133,7 +133,7 @@ def plot_losses(saving_folder: str | os.PathLike) -> None: }, ) - g._legend.set_bbox_to_anchor((0.8, 0.5)) + g.legend.set_bbox_to_anchor((0.8, 0.5)) def plot_sampling(saving_folder: str | os.PathLike) -> None: @@ -164,7 +164,7 @@ def plot_sampling(saving_folder: str | os.PathLike) -> None: # take legend of the plot in the last row and first column, to be sure it's a scatter plot handles, _ = g.axes[-1][0].get_legend_handles_labels() - g._legend.remove() + g.legend.remove() plt.legend(loc=2, handles=handles, labels=sampler_names, bbox_to_anchor=(0.0, 1.8)) @@ -205,7 +205,7 @@ def plot_losses_method_num( }, ) - g._legend.set_bbox_to_anchor((0.8, 0.5)) + g.legend.set_bbox_to_anchor((0.8, 0.5)) def plot_losses_interact(saving_folder: str | os.PathLike) -> None: @@ -274,7 +274,7 @@ def plot_sampling_batch_nums( # take legend of the plot in the last row and first column, to be sure it's a scatter plot handles, _ = g.axes[-1][0].get_legend_handles_labels() - g._legend.remove() + g.legend.remove() plt.legend(loc=2, handles=handles, labels=sampler_names, bbox_to_anchor=(0.0, 1.8)) diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index 64b08081..7a2e558a 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -55,8 +55,8 @@ def __init__( super().__init__(new_samplers, random_state) - self._in_queue: Queue = self._env._out_queue - self._out_queue: Queue = self._env._in_queue + self._in_queue: Queue = self._env._out_queue # noqa: SLF001 + self._out_queue: Queue = self._env._in_queue # noqa: SLF001 self._best_param: float | None = None self._best_loss: float | None = None @@ -144,7 +144,7 @@ def update( if self._best_loss is None: self._best_loss = best_new_loss self._best_param = new_params[np.argmin(new_losses)] - self._env._curr_best_loss = best_new_loss + self._env._curr_best_loss = best_new_loss # noqa: SLF001 return if best_new_loss < cast(float, self._best_loss): self._best_loss = best_new_loss diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 2074c66e..1c7161a9 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -114,7 +114,9 @@ def test_gaussian_process_sample_warning_too_large_dataset() -> None: ( xys, losses, - ) = TestGaussianProcess2D._construct_fake_grid(n=23) + ) = TestGaussianProcess2D._construct_fake_grid( # noqa: SLF001 + n=23, + ) with pytest.warns( RuntimeWarning, match="Standard GP evaluations can be expensive " diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index 77151a71..84531338 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -125,7 +125,7 @@ def test_clip_losses() -> None: # verify that _clip_losses works as expected y = np.array([0.0, -1e40, 1e40]) - y2 = xgboost._clip_losses(y) + y2 = xgboost._clip_losses(y) # noqa: SLF001 assert ( y2 == np.array([0.0, MIN_FLOAT32 + EPS_FLOAT32, MAX_FLOAT32 - EPS_FLOAT32]) diff --git a/tests/test_utils/test_seedable.py b/tests/test_utils/test_seedable.py index 09d91470..5c8c6aa7 100644 --- a/tests/test_utils/test_seedable.py +++ b/tests/test_utils/test_seedable.py @@ -78,10 +78,14 @@ def test_get_random_seed(random_seed: int) -> None: nb_iterations = 1000 seedable.random_state = random_seed - expected_values_1 = [seedable._get_random_seed() for _ in range(nb_iterations)] + expected_values_1 = [ + seedable._get_random_seed() for _ in range(nb_iterations) # noqa: SLF001 + ] seedable.random_state = random_seed - expected_values_2 = [seedable._get_random_seed() for _ in range(nb_iterations)] + expected_values_2 = [ + seedable._get_random_seed() for _ in range(nb_iterations) # noqa: SLF001 + ] assert expected_values_1 == expected_values_2 assert all(0 <= value <= 2**32 - 1 for value in expected_values_1) From eef7cc90e3068e57db14d61f80b7ef1515ba2080 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 15:03:17 +0200 Subject: [PATCH 34/56] lint: fix flake8-simplify errors To reproduce the errors: ``` ruff check --select "SIM" black_it tests examples scripts ``` All the errors were automatically fixed with --fix flag. --- .ruff.toml | 2 +- black_it/loss_functions/gsl_div.py | 18 +++++++++--------- black_it/samplers/r_sequence.py | 2 +- .../schedulers/rl/agents/epsilon_greedy.py | 6 +----- tests/test_losses/test_gsl.py | 5 +---- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 6311c152..5784fca8 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 877c493d..4a17799e 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -121,15 +121,15 @@ def compute_loss_1d( ts_length = len(real_data) ensemble_size = sim_data_ensemble.shape[0] - if self.nb_values is None: - nb_values = int((ts_length - 1) / 2.0) - else: - nb_values = self.nb_values - - if self.nb_word_lengths is None: - nb_word_lengths = int((ts_length - 1) / 2.0) - else: - nb_word_lengths = self.nb_word_lengths + nb_values = ( + int((ts_length - 1) / 2.0) if self.nb_values is None else self.nb_values + ) + + nb_word_lengths = ( + int((ts_length - 1) / 2.0) + if self.nb_word_lengths is None + else self.nb_word_lengths + ) # discretize real time series obs_xd = self.discretize( diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index f7976cac..37e8d7c1 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -61,7 +61,7 @@ def compute_phi(cls, nb_dims: int) -> float: Returns: phi^nb_dims """ - check_arg(1 <= nb_dims, f"nb_dims should be greater than 0, got {nb_dims}") + check_arg(nb_dims >= 1, f"nb_dims should be greater than 0, got {nb_dims}") phi: float = 2.0 old_phi = None while old_phi != phi: diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index a6c82a84..7f130a9b 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -55,11 +55,7 @@ def __init__( def get_step_size(self, action: int) -> float: """Get the step size.""" - if self.alpha == -1: - step_size = 1 / self.actions_count[action] - else: - step_size = self.alpha - return step_size + return 1 / self.actions_count[action] if self.alpha == -1 else self.alpha def learn( self, diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index e9e1d257..c870bdb8 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -39,10 +39,7 @@ def is_sorted(array: np.ndarray, *, reverse: bool = False) -> bool: def fail_check(left: float, right: float) -> bool: return left > right if not reverse else left < right - for i in range(array.size - 1): - if fail_check(array[i], array[i + 1]): - return False - return True + return all(not fail_check(array[i], array[i + 1]) for i in range(array.size - 1)) class TestDiscretize: From 99222dc2bde6cbb64befb246e305936db24479bc Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 15:12:32 +0200 Subject: [PATCH 35/56] lint: fix flake8-type-checking errors To reproduce the errors: ``` ruff check --select "TCH" black_it tests examples scripts ``` All the errors were fixed by using the --fix flag. --- .ruff.toml | 2 +- black_it/calibrator.py | 11 +++++++---- black_it/loss_functions/gsl_div.py | 6 ++++-- black_it/loss_functions/likelihood.py | 6 ++++-- black_it/loss_functions/minkowski.py | 8 +++++--- black_it/samplers/base.py | 8 ++++++-- black_it/samplers/best_batch.py | 9 +++++++-- black_it/samplers/cors.py | 6 ++++-- black_it/samplers/gaussian_process.py | 6 ++++-- black_it/samplers/halton.py | 9 ++++++--- black_it/samplers/particle_swarm.py | 6 ++++-- black_it/samplers/r_sequence.py | 9 +++++++-- black_it/samplers/random_forest.py | 6 ++++-- black_it/samplers/surrogate.py | 8 ++++++-- black_it/samplers/xgboost.py | 6 ++++-- black_it/schedulers/base.py | 12 +++++++----- black_it/schedulers/rl/envs/base.py | 6 ++++-- black_it/schedulers/rl/rl_scheduler.py | 14 +++++++++----- black_it/search_space.py | 6 +++++- black_it/utils/base.py | 6 ++++-- black_it/utils/json_pandas_checkpointing.py | 11 +++++++---- black_it/utils/seedable.py | 6 +++++- black_it/utils/sqlite3_checkpointing.py | 9 +++++---- black_it/utils/time_series.py | 6 +++++- examples/models/economics/brock_hommes.py | 6 ++++-- examples/models/forest_fire/forest_fire.py | 5 ++++- examples/models/sir/sir_docker.py | 6 +++++- examples/models/sir/sir_python.py | 6 +++++- tests/test_losses/test_base.py | 6 ++++-- tests/utils/docs.py | 7 +++++-- 30 files changed, 151 insertions(+), 67 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 5784fca8..8d115268 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 7d9a6802..e0c80a41 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -22,14 +22,11 @@ import textwrap import time import warnings -from typing import Callable, Sequence, cast +from typing import TYPE_CHECKING, Callable, Sequence, cast import numpy as np from joblib import Parallel, delayed -from numpy.typing import NDArray -from black_it.loss_functions.base import BaseLoss -from black_it.samplers.base import BaseSampler from black_it.schedulers.base import BaseScheduler from black_it.schedulers.round_robin import RoundRobinScheduler from black_it.search_space import SearchSpace @@ -40,6 +37,12 @@ ) from black_it.utils.seedable import BaseSeedable +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.loss_functions.base import BaseLoss + from black_it.samplers.base import BaseSampler + class Calibrator(BaseSeedable): """The class used to perform a calibration.""" diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 4a17799e..9e2ad587 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -53,14 +53,16 @@ """ from __future__ import annotations -from typing import Callable +from typing import TYPE_CHECKING, Callable import numpy as np -from numpy.typing import NDArray from black_it.loss_functions.base import BaseLoss from black_it.utils.base import _assert +if TYPE_CHECKING: + from numpy.typing import NDArray + EPS = 0.00001 # np.finfo(float).eps diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index 2796a3de..c2a6e5e3 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -18,13 +18,15 @@ from __future__ import annotations import warnings -from typing import Callable +from typing import TYPE_CHECKING, Callable import numpy as np -from numpy.typing import NDArray from black_it.loss_functions.base import BaseLoss +if TYPE_CHECKING: + from numpy.typing import NDArray + def kernel(sq_dist: NDArray[np.float64], h: float, d: int) -> NDArray[np.float64]: """Compute a kernel density estimation using a Gaussian kernel.""" diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index 3376dddd..2e575db2 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -17,14 +17,16 @@ """This module contains the implementation of the quadratic loss.""" from __future__ import annotations -from typing import Callable +from typing import TYPE_CHECKING, Callable -import numpy as np -from numpy.typing import NDArray from scipy.spatial.distance import minkowski from black_it.loss_functions.base import BaseLoss +if TYPE_CHECKING: + import numpy as np + from numpy.typing import NDArray + class MinkowskiLoss(BaseLoss): """Class for the Minkowski loss.""" diff --git a/black_it/samplers/base.py b/black_it/samplers/base.py index e5ad8d4c..f3dd38a5 100644 --- a/black_it/samplers/base.py +++ b/black_it/samplers/base.py @@ -18,13 +18,17 @@ from __future__ import annotations from abc import ABC, abstractmethod +from typing import TYPE_CHECKING import numpy as np -from numpy.typing import NDArray -from black_it.search_space import SearchSpace from black_it.utils.seedable import BaseSeedable +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.search_space import SearchSpace + class BaseSampler(BaseSeedable, ABC): """BaseSampler interface. diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index e2351ce0..d3bb6c22 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -17,14 +17,19 @@ """This module contains the implementation of the best-batch sampler.""" from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np -from numpy.typing import NDArray from scipy.stats import betabinom from black_it.samplers.base import BaseSampler -from black_it.search_space import SearchSpace from black_it.utils.base import _assert +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.search_space import SearchSpace + class BestBatchSampler(BaseSampler): """This class implements the best-batch sampler. diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index d2e4a806..1878c558 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -23,7 +23,7 @@ from __future__ import annotations from math import factorial -from typing import Callable, cast +from typing import TYPE_CHECKING, Callable, cast import numpy as np import scipy.optimize as op @@ -31,9 +31,11 @@ from numpy.typing import NDArray from black_it.samplers.base import BaseSampler -from black_it.search_space import SearchSpace from black_it.utils.base import digitize_data, positive_float +if TYPE_CHECKING: + from black_it.search_space import SearchSpace + def volume_d_dimensional_ball_radius_1(dims: int) -> float: """Compute the volume of a d-dimensional ball with radius 1.""" diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 0fefe81d..6e352c3a 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -20,15 +20,17 @@ import warnings from enum import Enum -from typing import cast +from typing import TYPE_CHECKING, cast import numpy as np -from numpy.typing import NDArray from scipy.special import erfc from sklearn.gaussian_process import GaussianProcessRegressor, kernels from black_it.samplers.surrogate import MLSurrogateSampler +if TYPE_CHECKING: + from numpy.typing import NDArray + class _AcquisitionTypes(Enum): """Enumeration of allowed acquisition types.""" diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index 3b65013c..49bf49dc 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -18,15 +18,18 @@ from __future__ import annotations import itertools -from typing import Iterator +from typing import TYPE_CHECKING, Iterator import numpy as np -from numpy.typing import NDArray from black_it.samplers.base import BaseSampler -from black_it.search_space import SearchSpace from black_it.utils.base import check_arg, digitize_data +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.search_space import SearchSpace + _MIN_SEQUENCE_START_INDEX = 20 _MAX_SEQUENCE_START_INDEX = 2**16 diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index e5bc0d89..bba4e93c 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -17,15 +17,17 @@ """Implementation of the particle swarm sampler.""" from __future__ import annotations -from typing import cast +from typing import TYPE_CHECKING, cast import numpy as np from numpy.typing import NDArray from black_it.samplers.base import BaseSampler -from black_it.search_space import SearchSpace from black_it.utils.base import _assert, digitize_data, positive_float +if TYPE_CHECKING: + from black_it.search_space import SearchSpace + class ParticleSwarmSampler(BaseSampler): """Implementation of a particle swarm sampler. diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index 37e8d7c1..a699c1e1 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -17,13 +17,18 @@ """This module contains the implementation of the R-sequence sampler.""" from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np -from numpy.typing import NDArray from black_it.samplers.base import BaseSampler -from black_it.search_space import SearchSpace from black_it.utils.base import check_arg, digitize_data +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.search_space import SearchSpace + _MIN_SEQUENCE_START_INDEX = 20 _MAX_SEQUENCE_START_INDEX = 2**16 diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 3f1b4c6d..a6d08f7c 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -17,15 +17,17 @@ """This module contains the implementation of the random forest sampling.""" from __future__ import annotations -from typing import cast +from typing import TYPE_CHECKING, cast import numpy as np -from numpy.typing import NDArray from sklearn.ensemble import RandomForestClassifier from black_it.samplers.surrogate import MLSurrogateSampler from black_it.utils.base import _assert +if TYPE_CHECKING: + from numpy.typing import NDArray + class RandomForestSampler(MLSurrogateSampler): """This class implements random forest sampling.""" diff --git a/black_it/samplers/surrogate.py b/black_it/samplers/surrogate.py index dd7cb254..ce453625 100644 --- a/black_it/samplers/surrogate.py +++ b/black_it/samplers/surrogate.py @@ -18,15 +18,19 @@ from __future__ import annotations from abc import abstractmethod +from typing import TYPE_CHECKING import numpy as np -from numpy.typing import NDArray from black_it.samplers.base import BaseSampler from black_it.samplers.random_uniform import RandomUniformSampler -from black_it.search_space import SearchSpace from black_it.utils.base import digitize_data +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.search_space import SearchSpace + class MLSurrogateSampler(BaseSampler): """MLSurrogateSampler interface. diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index ab83d970..19e3409b 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -18,14 +18,16 @@ from __future__ import annotations import warnings -from typing import cast +from typing import TYPE_CHECKING, cast import numpy as np import xgboost as xgb -from numpy.typing import NDArray from black_it.samplers.surrogate import MLSurrogateSampler +if TYPE_CHECKING: + from numpy.typing import NDArray + MAX_FLOAT32 = np.finfo(np.float32).max MIN_FLOAT32 = np.finfo(np.float32).min EPS_FLOAT32 = np.finfo(np.float32).eps diff --git a/black_it/schedulers/base.py b/black_it/schedulers/base.py index faf02b88..88edf2a5 100644 --- a/black_it/schedulers/base.py +++ b/black_it/schedulers/base.py @@ -19,14 +19,16 @@ import contextlib from abc import ABC, abstractmethod -from typing import Generator, Sequence +from typing import TYPE_CHECKING, Generator, Sequence -import numpy as np -from numpy.typing import NDArray - -from black_it.samplers.base import BaseSampler from black_it.utils.seedable import BaseSeedable +if TYPE_CHECKING: + import numpy as np + from numpy.typing import NDArray + + from black_it.samplers.base import BaseSampler + class BaseScheduler(BaseSeedable, ABC): """BaseScheduler interface. diff --git a/black_it/schedulers/rl/envs/base.py b/black_it/schedulers/rl/envs/base.py index 963ab860..eacb18ef 100644 --- a/black_it/schedulers/rl/envs/base.py +++ b/black_it/schedulers/rl/envs/base.py @@ -19,16 +19,18 @@ from abc import ABC, abstractmethod from queue import Queue -from typing import Any, SupportsFloat +from typing import TYPE_CHECKING, Any, SupportsFloat import gymnasium as gym import numpy as np from gymnasium.core import ObsType, RenderFrame from gymnasium.spaces import Discrete -from numpy._typing import NDArray from black_it.utils.base import _assert +if TYPE_CHECKING: + from numpy._typing import NDArray + class CalibrationEnv(gym.Env[ObsType, np.int64], ABC): """A Gym environment that wraps the calibration task.""" diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index 7a2e558a..f4b5a82b 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -18,17 +18,21 @@ from __future__ import annotations import threading -from queue import Queue -from typing import List, Sequence, cast +from typing import TYPE_CHECKING, List, Sequence, cast import numpy as np -from numpy._typing import NDArray from black_it.samplers.base import BaseSampler from black_it.samplers.halton import HaltonSampler from black_it.schedulers.base import BaseScheduler -from black_it.schedulers.rl.agents.base import Agent -from black_it.schedulers.rl.envs.base import CalibrationEnv + +if TYPE_CHECKING: + from queue import Queue + + from numpy._typing import NDArray + + from black_it.schedulers.rl.agents.base import Agent + from black_it.schedulers.rl.envs.base import CalibrationEnv class RLScheduler(BaseScheduler): diff --git a/black_it/search_space.py b/black_it/search_space.py index 178d8d70..6ff9a250 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -17,8 +17,12 @@ """This module contains the definition of the search space abstractions.""" from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np -from numpy.typing import NDArray + +if TYPE_CHECKING: + from numpy.typing import NDArray class SearchSpace: diff --git a/black_it/utils/base.py b/black_it/utils/base.py index 4006a970..d1fb4586 100644 --- a/black_it/utils/base.py +++ b/black_it/utils/base.py @@ -19,10 +19,12 @@ import os from json import JSONEncoder -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union import numpy as np -from numpy.typing import NDArray + +if TYPE_CHECKING: + from numpy.typing import NDArray PathLike = Union[str, os.PathLike] diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index ff581af1..dbcf2866 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -20,17 +20,20 @@ import json import pickle # nosec B403 from pathlib import Path -from typing import Mapping +from typing import TYPE_CHECKING, Mapping import numpy as np import pandas as pd import tables -from numpy.typing import NDArray -from black_it.loss_functions.base import BaseLoss -from black_it.schedulers.base import BaseScheduler from black_it.utils.base import NumpyArrayEncoder, PathLike +if TYPE_CHECKING: + from numpy.typing import NDArray + + from black_it.loss_functions.base import BaseLoss + from black_it.schedulers.base import BaseScheduler + def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) -> tuple: """Load calibrator data from a given folder. diff --git a/black_it/utils/seedable.py b/black_it/utils/seedable.py index 8cf9095d..81242af5 100644 --- a/black_it/utils/seedable.py +++ b/black_it/utils/seedable.py @@ -17,9 +17,13 @@ """This module contains the definition of a 'seedable' base class.""" from __future__ import annotations -import numpy as np +from typing import TYPE_CHECKING + from numpy.random import default_rng +if TYPE_CHECKING: + import numpy as np + class BaseSeedable: """BaseSeedable base class. diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index fc8ca4b4..6a6d5060 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -23,14 +23,15 @@ import pickle # nosec B403 import sqlite3 from pathlib import Path -from typing import Mapping, Sequence +from typing import TYPE_CHECKING, Mapping, Sequence import numpy as np from numpy.typing import NDArray -from black_it.loss_functions.base import BaseLoss -from black_it.samplers.base import BaseSampler -from black_it.utils.base import PathLike +if TYPE_CHECKING: + from black_it.loss_functions.base import BaseLoss + from black_it.samplers.base import BaseSampler + from black_it.utils.base import PathLike SCHEMA_VERSION = 3 """ diff --git a/black_it/utils/time_series.py b/black_it/utils/time_series.py index e358642d..f81de90d 100644 --- a/black_it/utils/time_series.py +++ b/black_it/utils/time_series.py @@ -17,12 +17,16 @@ """This module contains utility functions to deal with time series.""" from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np import scipy.sparse as sps import statsmodels.api as sm -from numpy.typing import NDArray from scipy.stats import kurtosis, skew +if TYPE_CHECKING: + from numpy.typing import NDArray + def get_mom_ts(time_series: NDArray[np.float64]) -> NDArray[np.float64]: """Compute specific moments from a time series. diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index 87dd9ad3..e8fa8c29 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -17,12 +17,14 @@ """Implementation of the model in (Brock and Hommes, 1998).""" from __future__ import annotations -from typing import Sequence +from typing import TYPE_CHECKING, Sequence import numpy as np -from numpy.typing import NDArray from scipy.special import softmax +if TYPE_CHECKING: + from numpy.typing import NDArray + def BH2( # noqa: N802 theta: Sequence[float], diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 7a51a934..82f5055b 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -18,9 +18,12 @@ from __future__ import annotations import subprocess +from typing import TYPE_CHECKING import numpy as np -from numpy.typing import NDArray + +if TYPE_CHECKING: + from numpy.typing import NDArray def forest_fire( diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index dee06983..f865b16b 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -17,11 +17,15 @@ """SIR models written in C and run in Docker containers.""" from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np -from numpy.typing import NDArray from . import simlib +if TYPE_CHECKING: + from numpy.typing import NDArray + def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N803 """SIR_docker. diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index a65e9c0d..7154afdf 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -17,11 +17,15 @@ """SIR models written in Python.""" from __future__ import annotations +from typing import TYPE_CHECKING + import ndlib.models.epidemics as ep import networkx as nx import numpy as np from ndlib.models import ModelConfig -from numpy.typing import NDArray + +if TYPE_CHECKING: + from numpy.typing import NDArray def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N803 diff --git a/tests/test_losses/test_base.py b/tests/test_losses/test_base.py index 2a3d0305..53dc1f5d 100644 --- a/tests/test_losses/test_base.py +++ b/tests/test_losses/test_base.py @@ -17,14 +17,16 @@ """This module contains tests for the base loss_functions module.""" from __future__ import annotations -from typing import Callable +from typing import TYPE_CHECKING, Callable import numpy as np import pytest -from numpy.typing import NDArray from black_it.loss_functions.base import BaseLoss +if TYPE_CHECKING: + from numpy.typing import NDArray + class TestComputeLoss: """Tests for BaseLoss.compute_loss.""" diff --git a/tests/utils/docs.py b/tests/utils/docs.py index 7daf622b..ca1d96f4 100644 --- a/tests/utils/docs.py +++ b/tests/utils/docs.py @@ -18,12 +18,15 @@ import json from io import StringIO -from pathlib import Path -from unittest.mock import MagicMock +from typing import TYPE_CHECKING import mistletoe from mistletoe.ast_renderer import ASTRenderer +if TYPE_CHECKING: + from pathlib import Path + from unittest.mock import MagicMock + MISTLETOE_CODE_BLOCK_ID = "CodeFence" From 583723790c4e5b987bdb2f4e2b0931f5d5afcc57 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 15:35:20 +0200 Subject: [PATCH 36/56] lint: fix flake8-unused-arguments errors To reproduce the errors: ``` ruff check --select "ARG" black_it tests examples scripts ``` Errors were mostly fixed manually. --- .ruff.toml | 2 +- black_it/loss_functions/likelihood.py | 4 ++-- black_it/loss_functions/minkowski.py | 2 +- black_it/samplers/halton.py | 4 ++-- black_it/samplers/particle_swarm.py | 2 +- black_it/samplers/r_sequence.py | 4 ++-- black_it/samplers/random_uniform.py | 4 ++-- black_it/schedulers/rl/agents/epsilon_greedy.py | 4 ++-- black_it/schedulers/rl/envs/base.py | 4 ++-- black_it/schedulers/rl/envs/mab.py | 6 +++++- black_it/schedulers/rl/rl_scheduler.py | 4 ++-- black_it/schedulers/round_robin.py | 8 ++++---- examples/models/sir/sir_docker.py | 8 ++++++-- tests/test_calibrator.py | 8 ++++---- tests/test_losses/test_base.py | 2 +- tests/test_samplers/test_base.py | 4 ++-- 16 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 8d115268..2d764e82 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index c2a6e5e3..c4ed70ad 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -137,8 +137,8 @@ def _check_bandwidth(self, s: int, d: int) -> float: def compute_loss_1d( self, - sim_data_ensemble: NDArray[np.float64], - real_data: NDArray[np.float64], + sim_data_ensemble: NDArray[np.float64], # noqa: ARG002 + real_data: NDArray[np.float64], # noqa: ARG002 ) -> float: """Compute likelihood loss on a single dimension, not available.""" msg = "The likelihood cannot be currently computed on a single dimension." diff --git a/black_it/loss_functions/minkowski.py b/black_it/loss_functions/minkowski.py index 2e575db2..4bf8670f 100644 --- a/black_it/loss_functions/minkowski.py +++ b/black_it/loss_functions/minkowski.py @@ -35,7 +35,7 @@ def __init__( self, p: int = 2, coordinate_weights: NDArray | None = None, - coordinate_filters: list[Callable | None] | None = None, + coordinate_filters: list[Callable | None] | None = None, # noqa: ARG002 ) -> None: """Loss computed using a Minkowski distance. diff --git a/black_it/samplers/halton.py b/black_it/samplers/halton.py index 49bf49dc..97dac4d2 100644 --- a/black_it/samplers/halton.py +++ b/black_it/samplers/halton.py @@ -82,8 +82,8 @@ def sample_batch( self, batch_size: int, search_space: SearchSpace, - existing_points: NDArray[np.float64], - existing_losses: NDArray[np.float64], + existing_points: NDArray[np.float64], # noqa: ARG002 + existing_losses: NDArray[np.float64], # noqa: ARG002 ) -> NDArray[np.float64]: """Sample points using Halton sequence. diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index bba4e93c..b835dbe8 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -185,7 +185,7 @@ def reset(self) -> None: def sample_batch( self, - batch_size: int, + batch_size: int, # noqa: ARG002 search_space: SearchSpace, existing_points: NDArray[np.float64], existing_losses: NDArray[np.float64], diff --git a/black_it/samplers/r_sequence.py b/black_it/samplers/r_sequence.py index a699c1e1..070f7301 100644 --- a/black_it/samplers/r_sequence.py +++ b/black_it/samplers/r_sequence.py @@ -97,8 +97,8 @@ def sample_batch( self, batch_size: int, search_space: SearchSpace, - existing_points: NDArray[np.float64], - existing_losses: NDArray[np.float64], + existing_points: NDArray[np.float64], # noqa: ARG002 + existing_losses: NDArray[np.float64], # noqa: ARG002 ) -> NDArray[np.float64]: """Sample points using the R-sequence. diff --git a/black_it/samplers/random_uniform.py b/black_it/samplers/random_uniform.py index d050b5f6..8a817151 100644 --- a/black_it/samplers/random_uniform.py +++ b/black_it/samplers/random_uniform.py @@ -30,8 +30,8 @@ def sample_batch( self, batch_size: int, search_space: SearchSpace, - existing_points: NDArray[np.float64], - existing_losses: NDArray[np.float64], + existing_points: NDArray[np.float64], # noqa: ARG002 + existing_losses: NDArray[np.float64], # noqa: ARG002 ) -> NDArray[np.float64]: """Sample uniformly from the search space. diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index 7f130a9b..73d93219 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -59,10 +59,10 @@ def get_step_size(self, action: int) -> float: def learn( self, - state: int, + state: int, # noqa: ARG002 action: int, reward: SupportsFloat, - next_state: int, + next_state: int, # noqa: ARG002 ) -> None: """Learn from an agent-environment interaction timestep.""" self.actions_count[action] += 1 diff --git a/black_it/schedulers/rl/envs/base.py b/black_it/schedulers/rl/envs/base.py index eacb18ef..13957268 100644 --- a/black_it/schedulers/rl/envs/base.py +++ b/black_it/schedulers/rl/envs/base.py @@ -60,8 +60,8 @@ def get_next_observation(self) -> ObsType: def reset( self, *, - seed: int | None = None, - options: dict[str, Any] | None = None, + seed: int | None = None, # noqa: ARG002 + options: dict[str, Any] | None = None, # noqa: ARG002 ) -> tuple[ObsType, dict[str, Any]]: """Reset the environment.""" return self.reset_state(), {} diff --git a/black_it/schedulers/rl/envs/mab.py b/black_it/schedulers/rl/envs/mab.py index 1c85d7f2..b7828ac4 100644 --- a/black_it/schedulers/rl/envs/mab.py +++ b/black_it/schedulers/rl/envs/mab.py @@ -32,7 +32,11 @@ def reset_state(self) -> int: """Get the initial state.""" return 0 - def get_reward(self, best_param: NDArray, best_loss: float) -> float: + def get_reward( + self, + best_param: NDArray, # noqa: ARG002 + best_loss: float, + ) -> float: """Get the reward.""" if self._curr_best_loss is None: msg = "cannot get reward, curr_best_loss should be already set" diff --git a/black_it/schedulers/rl/rl_scheduler.py b/black_it/schedulers/rl/rl_scheduler.py index f4b5a82b..7074ea87 100644 --- a/black_it/schedulers/rl/rl_scheduler.py +++ b/black_it/schedulers/rl/rl_scheduler.py @@ -138,10 +138,10 @@ def get_next_sampler(self) -> BaseSampler: def update( self, - batch_id: int, + batch_id: int, # noqa: ARG002 new_params: NDArray[np.float64], new_losses: NDArray[np.float64], - new_simulated_data: NDArray[np.float64], + new_simulated_data: NDArray[np.float64], # noqa: ARG002 ) -> None: """Update the RL scheduler.""" best_new_loss = float(np.min(new_losses)) diff --git a/black_it/schedulers/round_robin.py b/black_it/schedulers/round_robin.py index 75bc25d6..dc6a9a5f 100644 --- a/black_it/schedulers/round_robin.py +++ b/black_it/schedulers/round_robin.py @@ -46,10 +46,10 @@ def get_next_sampler(self) -> BaseSampler: def update( self, - batch_id: int, - new_params: NDArray[np.float64], - new_losses: NDArray[np.float64], - new_simulated_data: NDArray[np.float64], + batch_id: int, # noqa: ARG002 + new_params: NDArray[np.float64], # noqa: ARG002 + new_losses: NDArray[np.float64], # noqa: ARG002 + new_simulated_data: NDArray[np.float64], # noqa: ARG002 ) -> None: """Update the state of the scheduler after each batch.""" self._batch_id += 1 diff --git a/examples/models/sir/sir_docker.py b/examples/models/sir/sir_docker.py index f865b16b..aad3328c 100644 --- a/examples/models/sir/sir_docker.py +++ b/examples/models/sir/sir_docker.py @@ -27,7 +27,11 @@ from numpy.typing import NDArray -def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N803 +def SIR( # noqa: N802 + theta: NDArray, + N: int, # noqa: N803 + seed: int | None, # noqa: ARG001 +) -> NDArray: """SIR_docker. C++ SIR model run in Docker container. @@ -49,7 +53,7 @@ def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N80 def SIR_w_breaks( # noqa: N802 theta: NDArray, N: int, # noqa: N803 - seed: int | None = None, + seed: int | None = None, # noqa: ARG001 ) -> NDArray: """SIR_docker_w_breaks.""" breaktime = int(theta[0]) diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index b2f46d19..aabe18d3 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -312,10 +312,10 @@ class MyCustomSampler(BaseSampler): def sample_batch( self, - batch_size: int, - search_space: SearchSpace, - existing_points: NDArray[np.float64], - existing_losses: NDArray[np.float64], + batch_size: int, # noqa: ARG002 + search_space: SearchSpace, # noqa: ARG002 + existing_points: NDArray[np.float64], # noqa: ARG002 + existing_losses: NDArray[np.float64], # noqa: ARG002 ) -> NDArray[np.float64]: """Sample a batch of parameters.""" return [] # type: ignore[return-value] diff --git a/tests/test_losses/test_base.py b/tests/test_losses/test_base.py index 53dc1f5d..1e5c9978 100644 --- a/tests/test_losses/test_base.py +++ b/tests/test_losses/test_base.py @@ -47,7 +47,7 @@ def __init__( def compute_loss_1d( self, sim_data_ensemble: NDArray[np.float64], - real_data: NDArray[np.float64], + real_data: NDArray[np.float64], # noqa: ARG002 ) -> float: """Compute the loss (constant).""" return float(np.sum(sim_data_ensemble) * self.loss_constant) diff --git a/tests/test_samplers/test_base.py b/tests/test_samplers/test_base.py index d6cc0fc0..42a03561 100644 --- a/tests/test_samplers/test_base.py +++ b/tests/test_samplers/test_base.py @@ -49,8 +49,8 @@ def sample_batch( self, batch_size: int, search_space: SearchSpace, - existing_points: NDArray[np.float64], - existing_losses: NDArray[np.float64], + existing_points: NDArray[np.float64], # noqa: ARG002 + existing_losses: NDArray[np.float64], # noqa: ARG002 ) -> NDArray[np.float64]: """Sample a batch of parameters.""" return self.random_generator.random(size=(batch_size, search_space.dims)) From add94766dcf328b37b53aefebfbcbd168423c548 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 15:51:48 +0200 Subject: [PATCH 37/56] lint: fix flake8-use-pathlib errors To reproduce the errors: ``` ruff check --select "PTH" black_it tests examples scripts ``` Errors were mostly fixed manually. --- .ruff.toml | 2 +- black_it/calibrator.py | 6 ++-- black_it/plot/plot_results.py | 33 +++++++++++++------ black_it/utils/json_pandas_checkpointing.py | 12 +++---- .../test_pandas_json_checkpointing.py | 9 ++--- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 2d764e82..669e42be 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index e0c80a41..e0dd0d26 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -18,10 +18,10 @@ from __future__ import annotations import multiprocessing -import os import textwrap import time import warnings +from pathlib import Path from typing import TYPE_CHECKING, Callable, Sequence, cast import numpy as np @@ -38,6 +38,8 @@ from black_it.utils.seedable import BaseSeedable if TYPE_CHECKING: + import os + from numpy.typing import NDArray from black_it.loss_functions.base import BaseLoss @@ -489,7 +491,7 @@ def create_checkpoint(self, file_name: str | os.PathLike) -> None: Args: file_name: the name of the folder where the data will be saved """ - checkpoint_path: str = os.path.join(os.path.realpath(file_name)) + checkpoint_path: Path = Path(file_name).resolve() t_start = time.time() diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 8180a81a..d1e2c7cd 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -17,9 +17,9 @@ """This module contains utilities for plotting results.""" from __future__ import annotations -import os import pickle # nosec B403 -from typing import Collection +from pathlib import Path +from typing import TYPE_CHECKING, Collection import matplotlib.pyplot as plt import numpy as np @@ -29,6 +29,9 @@ from black_it.calibrator import Calibrator +if TYPE_CHECKING: + import os + def _get_samplers_id_table(saving_folder: str | os.PathLike) -> dict[str, int]: """Get the id table of the samplers from the checkpoint. @@ -39,7 +42,8 @@ def _get_samplers_id_table(saving_folder: str | os.PathLike) -> dict[str, int]: Returns: the id table of the samplers """ - with open(os.path.join(saving_folder, "samplers_pickled.pickle"), "rb") as f: + output_file = Path(saving_folder) / "samplers_pickled.pickle" + with output_file.open("rb") as f: method_list = pickle.load(f) # nosec B301 return Calibrator._construct_samplers_id_table(method_list) # noqa: SLF001 @@ -76,7 +80,8 @@ def plot_convergence(saving_folder: str | os.PathLike) -> None: saving_folder: the folder where the calibration results were saved """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) losses_cummin = df.groupby("batch_num_samp").min()["losses_samp"].cummin() @@ -113,7 +118,8 @@ def plot_losses(saving_folder: str | os.PathLike) -> None: Args: saving_folder: the folder where the calibration results were saved """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) num_params = sum("params_samp_" in c_str for c_str in df.columns) @@ -142,7 +148,8 @@ def plot_sampling(saving_folder: str | os.PathLike) -> None: Args: saving_folder: the folder where the calibration results were saved """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) num_params = sum("params_samp_" in c_str for c_str in df.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] @@ -179,7 +186,8 @@ def plot_losses_method_num( saving_folder: the folder where the calibration results were saved method_num: the integer value defining a specific sampling method """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) if method_num not in set(df["method_samp"]): msg = f"Samplers with method_num = {method_num} was never used" @@ -216,7 +224,8 @@ def plot_losses_interact(saving_folder: str | os.PathLike) -> None: Args: saving_folder: the folder where the calibration results were saved """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) method_nums = set(df["method_samp"]) samplers_id_table = _get_samplers_id_table(saving_folder) @@ -244,9 +253,11 @@ def plot_sampling_batch_nums( saving_folder: the folder where the calibration results were saved batch_nums: a list of batch number """ + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) + # deduplicate batch numbers batch_nums = set(batch_nums) - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) filter_bns = [bn in batch_nums for bn in df["batch_num_samp"]] @@ -287,7 +298,9 @@ def plot_sampling_interact(saving_folder: str | os.PathLike) -> None: Args: saving_folder: the folder where the calibration results were saved """ - df = pd.read_csv(os.path.join(saving_folder, "calibration_results.csv")) + calibration_results_file = Path(saving_folder) / "calibration_results.csv" + df = pd.read_csv(calibration_results_file) + max_bn = int(max(df["batch_num_samp"])) all_bns = np.arange(max_bn + 1, dtype=int) indices_bns = np.array_split(all_bns, min(max_bn, 3)) diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index dbcf2866..32271a02 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -46,7 +46,7 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - all the data needed to reconstruct the calibrator state """ checkpoint_path = Path(checkpoint_path) - with open(checkpoint_path / "calibration_params.json") as f: + with (checkpoint_path / "calibration_params.json").open() as f: cp = json.load(f) cr = pd.read_csv(checkpoint_path / "calibration_results.csv") @@ -58,10 +58,10 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - params_samp = np.vstack(params_samp_list).T - with open(checkpoint_path / "scheduler_pickled.pickle", "rb") as fb: + with (checkpoint_path / "scheduler_pickled.pickle").open("rb") as fb: scheduler = pickle.load(fb) # nosec B301 - with open(checkpoint_path / "loss_function_pickled.pickle", "rb") as fb: + with (checkpoint_path / "loss_function_pickled.pickle").open("rb") as fb: loss_function = pickle.load(fb) # nosec B301 series_filename = checkpoint_path / "series_samp.h5" @@ -172,14 +172,14 @@ def save_calibrator_state( "n_jobs": n_jobs, } # save calibration parameters in a json dictionary - with open(checkpoint_path / "calibration_params.json", "w") as f: + with (checkpoint_path / "calibration_params.json").open("w") as f: json.dump(calibration_params, f, cls=NumpyArrayEncoder) # save instantiated scheduler and loss functions - with open(checkpoint_path / "scheduler_pickled.pickle", "wb") as fb: + with (checkpoint_path / "scheduler_pickled.pickle").open("wb") as fb: pickle.dump(scheduler, fb) - with open(checkpoint_path / "loss_function_pickled.pickle", "wb") as fb: + with (checkpoint_path / "loss_function_pickled.pickle").open("wb") as fb: pickle.dump(loss_function, fb) # save calibration results into pandas dataframe diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index bff8dcb0..4d5e31f1 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -14,8 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """This module contains tests for the pandas_json_checkpointing.py module.""" -import glob import os +from pathlib import Path import numpy as np @@ -108,7 +108,8 @@ def test_save_and_load_calibrator_state() -> None: assert np.allclose(loaded_state[21], method_samp) # remove the test folder - files = glob.glob("saving_folder/*") + saving_folder_path = Path("saving_folder") + files = saving_folder_path.glob("./*") for f in files: - os.remove(f) - os.rmdir("saving_folder") + f.unlink(missing_ok=False) + saving_folder_path.rmdir() From 7ed39aee90c59d1ec90f226b78793406e61e9aab Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 16:02:13 +0200 Subject: [PATCH 38/56] lint: fix flake8-todos To reproduce the errors: ``` ruff check --select "TD" black_it tests examples scripts ``` --- .ruff.toml | 2 +- tests/test_calibrator.py | 6 ++---- tests/test_samplers/test_xgboost.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 669e42be..02504fbf 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index aabe18d3..0bb2b71f 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -171,10 +171,8 @@ def test_calibrator_calibrate(self, n_jobs: int) -> None: params, losses = cal.calibrate(14) - # TODO: this is a temporary workaround to make tests to run also on Windows. - # See: https://github.com/bancaditalia/black-it/issues/49 - print(params.tolist()) - print(losses.tolist()) + # This is a temporary workaround to make tests to run also on Windows. + # See: https://github.com/bancaditalia/black-it/issues/49 if sys.platform == "darwin": assert np.allclose(params, self.darwin_expected_params) assert np.allclose(losses, self.darwin_expected_losses) diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index 84531338..a19567be 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -25,8 +25,8 @@ from black_it.search_space import SearchSpace from examples.models.economics.brock_hommes import BH4 -# TODO: this is a temporary workaround to make tests to run also on Windows and Mac. -# See: https://github.com/bancaditalia/black-it/issues/49 +# This is a temporary workaround to make tests to run also on Windows and Mac. +# See: https://github.com/bancaditalia/black-it/issues/49 if sys.platform == "win32": expected_params = np.array([[0.24, 0.26], [0.19, 0.11], [0.13, 0.22], [0.11, 0.05]]) elif sys.platform == "darwin": From 97a5d430c460227564c7f61108571f3647973599 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 16:07:47 +0200 Subject: [PATCH 39/56] lint: ignore eradicate errors To reproduce the errors: ``` ruff check --select "ERA" black_it tests examples scripts ``` We decided to ignore the errors in the examples/ code because they might be useful as inlined code documentation. --- .ruff.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 02504fbf..e5e8bb21 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. @@ -48,7 +48,8 @@ target-version = "py38" [per-file-ignores] "examples/docker-sir.py" = ["INP001", "T201"] "examples/main.py" = ["INP001", "T201"] -"examples/models/simple_models.py" = ["INP001"] +"examples/models/simple_models.py" = ["ERA001", "INP001"] +"examples/models/economics/brock_hommes.py" = ["ERA001"] "examples/models/forest_fire/forest_fire.py" = ["T201"] "examples/models/sir/simlib.py" = ["T201"] "scripts/check_copyright.py" = ["T201"] From e918605f26ec1c9e455f944c5d97464391246881 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 16:12:13 +0200 Subject: [PATCH 40/56] lint: fix pandas-vet errors To reproduce the errors: ``` ruff check --select "PD" black_it tests examples scripts ``` --- .ruff.toml | 2 +- black_it/plot/plot_results.py | 56 +++++++++++++++++------------------ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index e5e8bb21..9ae2df64 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index d1e2c7cd..468f9053 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -81,13 +81,13 @@ def plot_convergence(saving_folder: str | os.PathLike) -> None: """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) - losses_cummin = df.groupby("batch_num_samp").min()["losses_samp"].cummin() + losses_cummin = data_frame.groupby("batch_num_samp").min()["losses_samp"].cummin() plt.figure() g = sns.lineplot( - data=df, + data=data_frame, x="batch_num_samp", y="losses_samp", hue="method_samp", @@ -95,7 +95,7 @@ def plot_convergence(saving_folder: str | os.PathLike) -> None: ) g = sns.lineplot( - x=np.arange(max(df["batch_num_samp"]) + 1), + x=np.arange(max(data_frame["batch_num_samp"]) + 1), y=losses_cummin, color="black", ls="--", @@ -103,7 +103,7 @@ def plot_convergence(saving_folder: str | os.PathLike) -> None: label="min loss", ) - ids = df["method_samp"].unique() + ids = data_frame["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) handles, labels = g.get_legend_handles_labels() @@ -119,14 +119,14 @@ def plot_losses(saving_folder: str | os.PathLike) -> None: saving_folder: the folder where the calibration results were saved """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) - num_params = sum("params_samp_" in c_str for c_str in df.columns) + num_params = sum("params_samp_" in c_str for c_str in data_frame.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] g = sns.pairplot( - df, + data_frame, hue="losses_samp", vars=variables, diag_kind="hist", @@ -149,13 +149,13 @@ def plot_sampling(saving_folder: str | os.PathLike) -> None: saving_folder: the folder where the calibration results were saved """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) - num_params = sum("params_samp_" in c_str for c_str in df.columns) + num_params = sum("params_samp_" in c_str for c_str in data_frame.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] g = sns.pairplot( - df, + data_frame, hue="method_samp", vars=variables, diag_kind="hist", @@ -166,7 +166,7 @@ def plot_sampling(saving_folder: str | os.PathLike) -> None: "fill": False, }, ) - ids = df["method_samp"].unique() + ids = data_frame["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) # take legend of the plot in the last row and first column, to be sure it's a scatter plot @@ -187,20 +187,20 @@ def plot_losses_method_num( method_num: the integer value defining a specific sampling method """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) - if method_num not in set(df["method_samp"]): + if method_num not in set(data_frame["method_samp"]): msg = f"Samplers with method_num = {method_num} was never used" raise ValueError(msg) - df = df.loc[df["method_samp"] == method_num] + data_frame = data_frame.loc[data_frame["method_samp"] == method_num] - num_params = sum("params_samp_" in c_str for c_str in df.columns) + num_params = sum("params_samp_" in c_str for c_str in data_frame.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] g = sns.pairplot( - df, + data_frame, hue="losses_samp", vars=variables, diag_kind="hist", @@ -225,8 +225,8 @@ def plot_losses_interact(saving_folder: str | os.PathLike) -> None: saving_folder: the folder where the calibration results were saved """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) - method_nums = set(df["method_samp"]) + data_frame = pd.read_csv(calibration_results_file) + method_nums = set(data_frame["method_samp"]) samplers_id_table = _get_samplers_id_table(saving_folder) @@ -254,23 +254,23 @@ def plot_sampling_batch_nums( batch_nums: a list of batch number """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) # deduplicate batch numbers batch_nums = set(batch_nums) - filter_bns = [bn in batch_nums for bn in df["batch_num_samp"]] + filter_bns = [bn in batch_nums for bn in data_frame["batch_num_samp"]] - df["filter_bns"] = filter_bns + data_frame["filter_bns"] = filter_bns - df_ = df.loc[df["filter_bns"] == True] # noqa + data_frame_2 = data_frame.loc[data_frame["filter_bns"] == True] # noqa - num_params = sum("params_samp_" in c_str for c_str in df.columns) + num_params = sum("params_samp_" in c_str for c_str in data_frame.columns) variables = ["params_samp_" + str(i) for i in range(num_params)] g = sns.pairplot( - df_, + data_frame_2, hue="method_samp", vars=variables, diag_kind="hist", @@ -280,7 +280,7 @@ def plot_sampling_batch_nums( palette="tab10", ) - ids = df["method_samp"].unique() + ids = data_frame["method_samp"].unique() sampler_names = _get_samplers_names(saving_folder, ids) # take legend of the plot in the last row and first column, to be sure it's a scatter plot @@ -299,9 +299,9 @@ def plot_sampling_interact(saving_folder: str | os.PathLike) -> None: saving_folder: the folder where the calibration results were saved """ calibration_results_file = Path(saving_folder) / "calibration_results.csv" - df = pd.read_csv(calibration_results_file) + data_frame = pd.read_csv(calibration_results_file) - max_bn = int(max(df["batch_num_samp"])) + max_bn = int(max(data_frame["batch_num_samp"])) all_bns = np.arange(max_bn + 1, dtype=int) indices_bns = np.array_split(all_bns, min(max_bn, 3)) From 55d7395559a63ea7e4642670f56b00a2d7b53d78 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Tue, 29 Aug 2023 16:45:54 +0200 Subject: [PATCH 41/56] lint: fix pygrep-hooks errors To reproduce the errors: ``` ruff check --select "PGH" black_it tests examples scripts ``` --- .ruff.toml | 2 +- black_it/calibrator.py | 2 +- black_it/plot/plot_results.py | 2 +- black_it/samplers/cors.py | 4 ++-- black_it/samplers/particle_swarm.py | 2 +- examples/docker-sir.py | 2 +- examples/main.py | 2 +- tests/conftest.py | 2 +- tests/test_calibrator.py | 8 ++++---- tests/test_docs/test_readme.py | 4 ---- tests/test_plot/base.py | 2 +- tests/test_samplers/test_base.py | 2 +- tests/test_utils/test_pandas_json_checkpointing.py | 4 ++-- tests/test_utils/test_sqlite3_checkpointing.py | 4 ++-- tests/utils/base.py | 2 +- 15 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 9ae2df64..03a9e689 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/calibrator.py b/black_it/calibrator.py index e0dd0d26..0abbe8fb 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -439,7 +439,7 @@ def calibrate(self, n_batches: int) -> tuple[NDArray, NDArray]: self.scheduler.update( self.current_batch_index, new_params, - new_losses, # type: ignore + new_losses, # type: ignore[arg-type] new_simulated_data, ) diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 468f9053..42b183a6 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -263,7 +263,7 @@ def plot_sampling_batch_nums( data_frame["filter_bns"] = filter_bns - data_frame_2 = data_frame.loc[data_frame["filter_bns"] == True] # noqa + data_frame_2 = data_frame.loc[data_frame["filter_bns"] == True] # noqa: E712 num_params = sum("params_samp_" in c_str for c_str in data_frame.columns) diff --git a/black_it/samplers/cors.py b/black_it/samplers/cors.py index 1878c558..37a55288 100644 --- a/black_it/samplers/cors.py +++ b/black_it/samplers/cors.py @@ -86,7 +86,7 @@ def phi(r: float) -> float: return r * r * r phis = [ - [phi(np.linalg.norm(np.subtract(points[i], points[j]))) for j in range(n)] # type: ignore + [phi(np.linalg.norm(np.subtract(points[i], points[j]))) for j in range(n)] # type: ignore[arg-type] for i in range(n) ] @@ -117,7 +117,7 @@ def phi(r: float) -> float: def fit(x: float) -> float: return ( sum( - lam[i] * phi(np.linalg.norm(np.subtract(x, points[i]))) # type: ignore + lam[i] * phi(np.linalg.norm(np.subtract(x, points[i]))) # type: ignore[arg-type] for i in range(n) ) + np.dot(b, x) diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index b835dbe8..7b019929 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -259,7 +259,7 @@ def _do_step(self) -> None: * (best_particle_positions - self._curr_particle_positions) + self.c2 * r2_vec - * (self._get_best_position() - self._curr_particle_positions) # type: ignore + * (self._get_best_position() - self._curr_particle_positions) # type: ignore[operator] ) self._curr_particle_positions = np.clip( diff --git a/examples/docker-sir.py b/examples/docker-sir.py index dbfdc39f..716b3e44 100755 --- a/examples/docker-sir.py +++ b/examples/docker-sir.py @@ -16,7 +16,7 @@ # along with this program. If not, see . """SIR model calibration example using a Docker-based simulator.""" -from models.sir.sir_docker import SIR # type: ignore +from models.sir.sir_docker import SIR from black_it.calibrator import Calibrator from black_it.loss_functions.minkowski import MinkowskiLoss diff --git a/examples/main.py b/examples/main.py index 6dd47bdd..94748fd3 100755 --- a/examples/main.py +++ b/examples/main.py @@ -16,7 +16,7 @@ # along with this program. If not, see . """This is a simple example showing the main features of the library.""" -import models.simple_models as md # type: ignore +import models.simple_models as md from black_it.calibrator import Calibrator from black_it.loss_functions.msm import MethodOfMomentsLoss diff --git a/tests/conftest.py b/tests/conftest.py index 2b842c18..443d3faf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ import inspect from pathlib import Path -CUR_PATH = Path(inspect.getfile(inspect.currentframe())).parent # type: ignore +CUR_PATH = Path(inspect.getfile(inspect.currentframe())).parent # type: ignore[arg-type] ROOT_DIR = Path(CUR_PATH, "..").resolve().absolute() DOCS_DIR = ROOT_DIR / "docs" TEST_DIR = ROOT_DIR / "tests" diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index 0bb2b71f..6ca96a39 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -257,20 +257,20 @@ def test_calibrator_restore_from_checkpoint_and_set_sampler(tmp_path: Path) -> N vars_cal["samplers"], cal_restored.scheduler.samplers, ): - assert type(method1).__name__ == type(method2).__name__ # noqa + assert type(method1).__name__ == type(method2).__name__ if key == "scheduler": t1 = type(vars_cal["scheduler"]) t2 = type(cal_restored.scheduler) - assert t1 == t2 # noqa + assert t1 == t2 elif key == "loss_function": assert ( type(vars_cal["loss_function"]).__name__ - == type(cal_restored.loss_function).__name__ # noqa + == type(cal_restored.loss_function).__name__ ) elif key == "param_grid": assert ( type(vars_cal["param_grid"]).__name__ - == type(cal_restored.param_grid).__name__ # noqa + == type(cal_restored.param_grid).__name__ ) elif key == f"_{BaseSeedable.__name__}__random_generator": assert ( diff --git a/tests/test_docs/test_readme.py b/tests/test_docs/test_readme.py index c642a9f3..064f3415 100644 --- a/tests/test_docs/test_readme.py +++ b/tests/test_docs/test_readme.py @@ -32,10 +32,6 @@ def test_quickstart_same_code_of_example(self) -> None: example_code = self.extract_example_code() quickstart_code_snippet = self.python_blocks[0] - quickstart_code_snippet = quickstart_code_snippet.replace( - "import models.simple_models as md\n", - "import models.simple_models as md # type: ignore\n", - ) # test import lines are equal nb_import_lines = 8 quickstart_import_lines = quickstart_code_snippet.splitlines()[:nb_import_lines] diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index 19aa61aa..e65a464b 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -42,7 +42,7 @@ def run(self) -> None: """Run the plotting and the image comparison.""" logging.getLogger("matplotlib.font_manager").disabled = True plt.close() - self.plotting_function.__func__(*self.args) # type: ignore + self.plotting_function.__func__(*self.args) # type: ignore[attr-defined] with TemporaryDirectory() as tmpdir: actual_figure_path = Path(tmpdir) / "actual.png" plt.savefig(actual_figure_path) diff --git a/tests/test_samplers/test_base.py b/tests/test_samplers/test_base.py index 42a03561..13bbb1be 100644 --- a/tests/test_samplers/test_base.py +++ b/tests/test_samplers/test_base.py @@ -30,7 +30,7 @@ def test_find_and_get_duplicates() -> None: BaseSampler.__abstractmethods__ = frozenset() - sampler = BaseSampler(batch_size=1) # type: ignore + sampler = BaseSampler(batch_size=1) # type: ignore[abstract] duplicates = sampler.find_and_get_duplicates(new_points, existing_points) assert duplicates == [0, 1, 2, 4, 5] diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index 4d5e31f1..4ee430ff 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -67,8 +67,8 @@ def test_save_and_load_calibrator_state() -> None: initial_random_seed, random_generator_state, model_name, - samplers, # type: ignore - loss_function, # type: ignore + samplers, # type: ignore[arg-type] + loss_function, # type: ignore[arg-type] current_batch_index, n_sampled_params, n_jobs, diff --git a/tests/test_utils/test_sqlite3_checkpointing.py b/tests/test_utils/test_sqlite3_checkpointing.py index bf016954..b01e235f 100644 --- a/tests/test_utils/test_sqlite3_checkpointing.py +++ b/tests/test_utils/test_sqlite3_checkpointing.py @@ -69,8 +69,8 @@ def test_sqlite3_checkpointing() -> None: initial_random_seed, random_generator_state, model_name, - samplers, # type: ignore - loss_function, # type: ignore + samplers, # type: ignore[arg-type] + loss_function, # type: ignore[arg-type] current_batch_index, params_samp, losses_samp, diff --git a/tests/utils/base.py b/tests/utils/base.py index 6bfb418b..48dd9686 100644 --- a/tests/utils/base.py +++ b/tests/utils/base.py @@ -156,7 +156,7 @@ def wrapper( # type: ignore[no-untyped-def] # noqa: ANN202 # here if the wrappee is a test class # wrap setup_class with 'action' if setup_class is defined if hasattr(pytest_func_or_cls, "setup_class"): - new_setup_class = decorator(pytest_func_or_cls.setup_class) # type: ignore + new_setup_class = decorator(pytest_func_or_cls.setup_class) else: new_setup_class = action # return a new subclass with same name, parent test methods, From a4a50ad42f09a076e73811066f575432ead95281 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 10:02:56 +0200 Subject: [PATCH 42/56] lint: fix PLR2004 errors To reproduce the errors: ``` ruff check --select "PLR2004" black_it tests examples scripts ``` We tried to not ignore errors as much as possible. This kind of error in the test files do not make much sense (expected values are usually "magic values" in code to make the tests more readable). Other errors were fixed by either replacing floats to ints (e.g. 0.0 -> 0, 1.0 -> 1), or moving the magic value to a module-level private constant. --- .ruff.toml | 3 ++- black_it/calibrator.py | 5 ++--- black_it/loss_functions/fourier.py | 2 +- black_it/loss_functions/msm.py | 8 ++++++-- black_it/samplers/best_batch.py | 4 ++-- black_it/samplers/gaussian_process.py | 8 ++++++-- black_it/samplers/random_forest.py | 5 ++++- black_it/search_space.py | 2 +- examples/models/economics/brock_hommes.py | 4 ++-- 9 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 03a9e689..47fe38e2 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PLR2004"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. @@ -53,6 +53,7 @@ target-version = "py38" "examples/models/forest_fire/forest_fire.py" = ["T201"] "examples/models/sir/simlib.py" = ["T201"] "scripts/check_copyright.py" = ["T201"] +"tests/**.py" = ["PLR2004"] [mccabe] diff --git a/black_it/calibrator.py b/black_it/calibrator.py index 0abbe8fb..55efcc7f 100644 --- a/black_it/calibrator.py +++ b/black_it/calibrator.py @@ -51,7 +51,7 @@ class Calibrator(BaseSeedable): STATE_VERSION = 0 - def __init__( + def __init__( # noqa: PLR0913 self, loss_function: BaseLoss, real_data: NDArray[np.float64], @@ -481,8 +481,7 @@ def check_convergence( True if the calibration converged, False otherwise. """ return ( - np.round(np.min(losses_samp[:n_sampled_params]), convergence_precision) - == 0.0 + np.round(np.min(losses_samp[:n_sampled_params]), convergence_precision) == 0 ) def create_checkpoint(self, file_name: str | os.PathLike) -> None: diff --git a/black_it/loss_functions/fourier.py b/black_it/loss_functions/fourier.py index 962e0639..b37ab7f8 100644 --- a/black_it/loss_functions/fourier.py +++ b/black_it/loss_functions/fourier.py @@ -110,7 +110,7 @@ def __init__( the loss computation. """ _assert( - 0.0 < f <= 1.0, + 0 < f <= 1, "'f' must be in the interval (0.0, 1.0]", ) self.frequency_filter = frequency_filter diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index e93fba2e..a94b8012 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -37,6 +37,8 @@ an NDArray. """ +_NB_MOMENTS = 18 + class _CovarianceMatrixType(Enum): """Enumeration of allowed covariance matrix types.""" @@ -136,10 +138,12 @@ def _validate_covariance_and_calculator( raise ValueError( msg, ) - if (moment_calculator is get_mom_ts_1d) and (covariance_mat.shape[0] != 18): + if (moment_calculator is get_mom_ts_1d) and ( + covariance_mat.shape[0] != _NB_MOMENTS + ): msg = ( "the provided covariance matrix is not valid as it has a wrong shape: " - f"expected 18, got {covariance_mat.shape[0]}" + f"expected {_NB_MOMENTS}, got {covariance_mat.shape[0]}" ) raise ValueError( msg, diff --git a/black_it/samplers/best_batch.py b/black_it/samplers/best_batch.py index d3bb6c22..ff1ec714 100644 --- a/black_it/samplers/best_batch.py +++ b/black_it/samplers/best_batch.py @@ -63,11 +63,11 @@ def __init__( plus/minus the perturbation_range times the precision of the specific parameter coordinate """ _assert( - a > 0.0, + a > 0, "'a' should be greater than zero", ) _assert( - b > 0.0, + b > 0, "'b' should be greater than zero", ) _assert( diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index 6e352c3a..c9003a0b 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -32,6 +32,10 @@ from numpy.typing import NDArray +_BIG_DATASET_SIZE_WARNING_THRESHOLD = 500 +_SMALL_VARIANCE_VALUES = 1e-5 + + class _AcquisitionTypes(Enum): """Enumeration of allowed acquisition types.""" @@ -111,7 +115,7 @@ def fit(self, X: NDArray[np.float64], y: NDArray[np.float64]) -> None: # noqa: """Fit a gaussian process surrogate model.""" y = np.atleast_2d(y).T - if X.shape[0] > 500: + if X.shape[0] > _BIG_DATASET_SIZE_WARNING_THRESHOLD: warnings.warn( # noqa: B028 "Standard GP evaluations can be expensive for large datasets, consider implementing a sparse GP", RuntimeWarning, @@ -207,7 +211,7 @@ def get_quantiles( the quantiles. """ # remove values of variance that are too small - s[s < 1e-5] = 1e-5 + s[s < _SMALL_VARIANCE_VALUES] = _SMALL_VARIANCE_VALUES u: NDArray[np.float64] = (fmin - m - acquisition_par) / s phi: NDArray[np.float64] = np.exp(-0.5 * u**2) / np.sqrt(2 * np.pi) diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index a6d08f7c..9697fc51 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -29,6 +29,9 @@ from numpy.typing import NDArray +_MIN_RF_CLASSES = 2 + + class RandomForestSampler(MLSurrogateSampler): """This class implements random forest sampling.""" @@ -57,7 +60,7 @@ def __init__( of the distribution of loss values. """ _assert( - n_classes > 2, + n_classes > _MIN_RF_CLASSES, "'n_classes' should be at least 2 to provide meaningful results", ) diff --git a/black_it/search_space.py b/black_it/search_space.py index 6ff9a250..b6f67589 100644 --- a/black_it/search_space.py +++ b/black_it/search_space.py @@ -101,7 +101,7 @@ def _check_bounds( parameters_precision: resolution of the grid of parameters """ # ensure parameters_bounds is a two-elements array - if len(parameters_bounds) != 2: + if len(parameters_bounds) != 2: # noqa: PLR2004 raise BoundsNotOfSizeTwoError(len(parameters_bounds)) # ensure the two sub-arrays of parameter_bounds (which are the min and diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index e8fa8c29..c9eae5b4 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -137,9 +137,9 @@ def BH4( # noqa: N802 g[i] = theta[i * 2] b[i] = theta[i * 2 + 1] - if len(theta) >= 9: + if len(theta) >= 9: # noqa: PLR2004 R = 1.0 + theta[8] # noqa: N806 - if len(theta) >= 10: + if len(theta) >= 10: # noqa: PLR2004 beta = theta[9] for t in range(2, N + 1): From b0e1d6dbf143ad8ad8d8a01422254148ccd27c32 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 10:15:40 +0200 Subject: [PATCH 43/56] lint: fix all remaining pylint errors To reproduce errors: ``` ruff check --select "PL" black_it tests examples scripts\n ``` --- .ruff.toml | 5 ++++- black_it/loss_functions/gsl_div.py | 2 +- black_it/loss_functions/likelihood.py | 2 +- black_it/loss_functions/msm.py | 2 +- black_it/samplers/gaussian_process.py | 2 +- black_it/samplers/random_forest.py | 2 +- black_it/samplers/xgboost.py | 2 +- black_it/schedulers/rl/agents/epsilon_greedy.py | 2 +- black_it/schedulers/rl/envs/mab.py | 4 ++-- black_it/utils/json_pandas_checkpointing.py | 2 +- black_it/utils/sqlite3_checkpointing.py | 2 +- examples/models/forest_fire/forest_fire.py | 2 +- examples/models/sir/simlib.py | 4 ++-- tests/test_calibrator.py | 2 +- tests/test_losses/test_gsl.py | 14 ++++++++++---- tests/test_samplers/test_gaussian_process.py | 8 ++++---- tests/test_utils/test_pandas_json_checkpointing.py | 2 +- 17 files changed, 34 insertions(+), 25 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 47fe38e2..1500aa35 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PLR2004"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. @@ -62,3 +62,6 @@ max-complexity = 10 [pydocstyle] convention = "google" + +[pylint] +max-args = 7 diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index 9e2ad587..a9279111 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -165,7 +165,7 @@ def compute_loss_1d( return gsl_loss / ensemble_size - def gsl_div_1d_1_sample( + def gsl_div_1d_1_sample( # noqa: PLR0913 self, sim_xd: NDArray, obs_xd: NDArray, diff --git a/black_it/loss_functions/likelihood.py b/black_it/loss_functions/likelihood.py index c4ed70ad..bb90602c 100644 --- a/black_it/loss_functions/likelihood.py +++ b/black_it/loss_functions/likelihood.py @@ -136,7 +136,7 @@ def _check_bandwidth(self, s: int, d: int) -> float: return h def compute_loss_1d( - self, + self, # noqa: PLR6301 sim_data_ensemble: NDArray[np.float64], # noqa: ARG002 real_data: NDArray[np.float64], # noqa: ARG002 ) -> float: diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index a94b8012..54bf5a49 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -54,7 +54,7 @@ def __str__(self) -> str: class MethodOfMomentsLoss(BaseLoss): """Class for the 'method of moments' loss.""" - def __init__( + def __init__( # noqa: PLR0913 self, covariance_mat: str | NDArray[np.float64] = "identity", coordinate_weights: NDArray[np.float64] | None = None, diff --git a/black_it/samplers/gaussian_process.py b/black_it/samplers/gaussian_process.py index c9003a0b..7583a292 100644 --- a/black_it/samplers/gaussian_process.py +++ b/black_it/samplers/gaussian_process.py @@ -55,7 +55,7 @@ class GaussianProcessSampler(MLSurrogateSampler): Note: this class is a wrapper of the GaussianProcessRegressor model of the scikit-learn package. """ - def __init__( + def __init__( # noqa: PLR0913 self, batch_size: int, random_state: int | None = None, diff --git a/black_it/samplers/random_forest.py b/black_it/samplers/random_forest.py index 9697fc51..f525bda1 100644 --- a/black_it/samplers/random_forest.py +++ b/black_it/samplers/random_forest.py @@ -35,7 +35,7 @@ class RandomForestSampler(MLSurrogateSampler): """This class implements random forest sampling.""" - def __init__( + def __init__( # noqa: PLR0913 self, batch_size: int, random_state: int | None = None, diff --git a/black_it/samplers/xgboost.py b/black_it/samplers/xgboost.py index 19e3409b..ffb70aa2 100644 --- a/black_it/samplers/xgboost.py +++ b/black_it/samplers/xgboost.py @@ -36,7 +36,7 @@ class XGBoostSampler(MLSurrogateSampler): """This class implements xgboost sampling.""" - def __init__( + def __init__( # noqa: PLR0913 self, batch_size: int, random_state: int | None = None, diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index 73d93219..79121751 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -27,7 +27,7 @@ class MABEpsilonGreedy(Agent[int, int]): """Implementation of a MAB eps-greedy algorithm.""" - def __init__( + def __init__( # noqa: PLR0913 self, n_actions: int, alpha: float, diff --git a/black_it/schedulers/rl/envs/mab.py b/black_it/schedulers/rl/envs/mab.py index b7828ac4..7304dfbf 100644 --- a/black_it/schedulers/rl/envs/mab.py +++ b/black_it/schedulers/rl/envs/mab.py @@ -24,11 +24,11 @@ class MABCalibrationEnv(CalibrationEnv[int]): """A calibration environment for MAB algorithms.""" - def get_next_observation(self) -> int: + def get_next_observation(self) -> int: # noqa: PLR6301 """Get the next observation.""" return 0 - def reset_state(self) -> int: + def reset_state(self) -> int: # noqa: PLR6301 """Get the initial state.""" return 0 diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index 32271a02..fd6b4e1d 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -97,7 +97,7 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - ) -def save_calibrator_state( +def save_calibrator_state( # noqa: PLR0913 checkpoint_path: PathLike, parameters_bounds: NDArray[np.float64], parameters_precision: NDArray[np.float64], diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index 6a6d5060..d4111d2b 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -260,7 +260,7 @@ def load_calibrator_state( connection.close() -def save_calibrator_state( +def save_calibrator_state( # noqa: PLR0913 checkpoint_path: PathLike, parameters_bounds: NDArray[np.float64], parameters_precision: NDArray[np.float64], diff --git a/examples/models/forest_fire/forest_fire.py b/examples/models/forest_fire/forest_fire.py index 82f5055b..86e33e7a 100644 --- a/examples/models/forest_fire/forest_fire.py +++ b/examples/models/forest_fire/forest_fire.py @@ -58,7 +58,7 @@ def forest_fire( command = f"julia forest_fire_julia.jl {density} {n} {xsize} {ysize} {seed}" - res = subprocess.run( + res = subprocess.run( # noqa: PLW1510 command.split(), shell=False, stdout=subprocess.PIPE, diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index 209e611b..c2a1b457 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -144,7 +144,7 @@ def _execute_simulator_subprocess(args: list[str]) -> list[dict[str, int]]: Returns: the simulation output """ - res = subprocess.run(args, text=True, capture_output=True) + res = subprocess.run(args, text=True, capture_output=True) # noqa: PLW1510 if res.returncode != 0: # bail out in case of error. In a mature system we should return & @@ -153,7 +153,7 @@ def _execute_simulator_subprocess(args: list[str]) -> list[dict[str, int]]: print(f"stdout:\n{res.stdout}") print(f"stderr:\n{res.stderr}") - exit(res.returncode) + sys.exit(res.returncode) return parse_simulator_output(res.stdout) diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index 6ca96a39..03bb8bcd 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -309,7 +309,7 @@ class MyCustomSampler(BaseSampler): """Custom sampler.""" def sample_batch( - self, + self, # noqa: PLR6301 batch_size: int, # noqa: ARG002 search_space: SearchSpace, # noqa: ARG002 existing_points: NDArray[np.float64], # noqa: ARG002 diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index c870bdb8..5c63720c 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -46,12 +46,18 @@ class TestDiscretize: """Test the 'discretize' function.""" @given(discretize_args()) - def test_discretize_time_series_any_args(self, args: tuple) -> None: + def test_discretize_time_series_any_args( + self, # noqa: PLR6301 + args: tuple, + ) -> None: """Test the case with randomly generated args.""" GslDivLoss.discretize(*args) @given(discretize_args()) - def test_discretize_time_series_partition(self, args: tuple) -> None: + def test_discretize_time_series_partition( + self, # noqa: PLR6301 + args: tuple, + ) -> None: """Test that discretize computes the right number of partitions.""" time_series, nb_values, start_index, stop_index = args actual = GslDivLoss.discretize(time_series, nb_values, start_index, stop_index) @@ -61,7 +67,7 @@ def test_discretize_time_series_partition(self, args: tuple) -> None: assert (actual <= max_nb_values).all() @given(discretize_args()) - def test_ordering_is_preserved(self, args: tuple) -> None: + def test_ordering_is_preserved(self, args: tuple) -> None: # noqa: PLR6301 """Test that the time series ordering is preserved when discretized.""" time_series, nb_values, start_index, stop_index = args increasing_time_series = np.sort(time_series) @@ -87,7 +93,7 @@ class TestGetWords: """Test the 'get_words' function.""" @given(get_words_args()) - def test_get_words(self, args: tuple) -> None: + def test_get_words(self, args: tuple) -> None: # noqa: PLR6301 """Test the case with randomly generated args.""" GslDivLoss.get_words(*args) diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 1c7161a9..57e95d73 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -47,10 +47,10 @@ def _construct_fake_grid( np.random.seed(0) for x in xs: for y in ys: - x = x + np.random.normal(0, 1e-2) - y = y + np.random.normal(0, 1e-2) - xys_list.append([x, y]) - losses_list.append(x**2 + y**2) + px = x + np.random.normal(0, 1e-2) + py = y + np.random.normal(0, 1e-2) + xys_list.append([px, py]) + losses_list.append(px**2 + py**2) return np.asarray(xys_list), np.asarray(losses_list) diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index 4ee430ff..74308dd2 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -25,7 +25,7 @@ ) -def test_save_and_load_calibrator_state() -> None: +def test_save_and_load_calibrator_state() -> None: # noqa: PLR0915 """Test the 'save_calibrator_state' and 'load_calibrator_state' functions.""" parameters_bounds = np.array([[0, 1], [0, 1]]).T parameters_precision = np.array([0.01, 0.01]) From 95d0f16690b4de03fe124602376a211e7a4270ff Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 10:54:09 +0200 Subject: [PATCH 44/56] lint: fix tryceratops errors To reproduce the errors: ``` ruff check --select "TRY" black_it tests examples scripts ``` --- .ruff.toml | 2 +- black_it/utils/sqlite3_checkpointing.py | 36 +++++++++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 1500aa35..b0ad28e5 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/utils/sqlite3_checkpointing.py b/black_it/utils/sqlite3_checkpointing.py index d4111d2b..9e82bce8 100644 --- a/black_it/utils/sqlite3_checkpointing.py +++ b/black_it/utils/sqlite3_checkpointing.py @@ -168,6 +168,31 @@ class gz_ndarray(NDArray): # noqa: N801 """ +class SchemaVersionMismatchError(Exception): + """Exception raised when the schema version of the checkpoint does not match the current schema version.""" + + def __init__( + self, + checkpoint_schema_version: int, + current_schema_version: int, + ) -> None: + """Initialize the exception. + + Args: + checkpoint_schema_version: the schema version of the checkpoint + current_schema_version: the schema version of the current code + """ + self.checkpoint_schema_version = checkpoint_schema_version + self.current_schema_version = current_schema_version + self.message = ( + "The checkpoint you want to load has been generated with another version of the code" + ":\n\tCheckpoint schema version:" + f" {checkpoint_schema_version}" + f"\tSchema version of the current code: {current_schema_version}" + ) + super().__init__(self.message) + + def load_calibrator_state( checkpoint_path: PathLike, ) -> tuple: @@ -195,14 +220,9 @@ def load_calibrator_state( SQL_LOAD_USER_VERSION, ).fetchone()[0] if checkpoint_schema_version != SCHEMA_VERSION: - msg = ( - "The checkpoint you want to load has been generated with another version of the code" - ":\n\tCheckpoint schema version:" - f" {checkpoint_schema_version}" - f"\tSchema version of the current code: {SCHEMA_VERSION}" - ) - raise Exception( - msg, + raise SchemaVersionMismatchError( # noqa: TRY301 + checkpoint_schema_version, + SCHEMA_VERSION, ) ( From a67a1f62c05c8609d00fa6e36484fcad1a6197a9 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 11:39:29 +0200 Subject: [PATCH 45/56] lint: ignore NumPy-specific rules errors for specific modules To reproduce the above errors (the output will include also others): ``` ruff check --select "NPY" black_it tests examples scripts\n ``` In this commit, we explicitly ignored these errors: - examples/models/economics/boltzmann_wealth.py:93, we kept np.random.binomial since would have required a change in the APIs. This could be addressed separately in another commit. --- .ruff.toml | 2 +- examples/models/economics/boltzmann_wealth.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index b0ad28e5..f23b71ad 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index 9c5f5411..601e61cf 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -90,7 +90,10 @@ class MoneyAgent(Agent): def __init__(self, unique_id: int, model: BoltzmannWealthModel) -> None: """Initialize the agent.""" super().__init__(unique_id, model) - self.wealth = np.random.binomial(n=2 * self.model.mean_init_wealth, p=0.5) + self.wealth = np.random.binomial( # noqa: NPY002 + n=2 * self.model.mean_init_wealth, + p=0.5, + ) def move(self) -> None: """Move the agent.""" From b21d25933a05b3e9d39d2f3028fb393161af2441 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 12:28:50 +0200 Subject: [PATCH 46/56] lint: fix NPY002 for Brock-Hommes model To reproduce the errors: ``` ruff check --select "NPY" examples/models/economics/brock_hommes.py ``` We also added tests to ensure reproducibility with fixed seed. --- examples/models/economics/brock_hommes.py | 14 +- tests/test_examples/test_brock_hommes.py | 198 +++++++++++----------- tests/test_samplers/test_xgboost.py | 9 +- 3 files changed, 107 insertions(+), 114 deletions(-) diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index c9eae5b4..197b3843 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -49,7 +49,7 @@ def BH2( # noqa: N802 Returns: simulated series """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) R = 1.10 # noqa: N806 beta = 3.6 @@ -78,7 +78,7 @@ def BH2( # noqa: N802 for t in range(2, N + 1): x[t] = n[1] * g[1] * x[t - 1] / R - x[t] = x[t] + np.random.uniform(low=div_eps_min, high=div_eps_max, size=1) + x[t] = x[t] + rng.uniform(low=div_eps_min, high=div_eps_max, size=1) n[0] = np.exp(bsa * (R * x[t - 1] * (R * x[t - 1] - x[t])) - beta * C) n[1] = np.exp(bsa * (x[t] - R * x[t - 1]) * (g[1] * x[t - 2] - R * x[t - 1])) @@ -115,10 +115,10 @@ def BH4( # noqa: N802 Returns: simulated series """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - R = 1.01 # noqa: N806 - beta = 120 + R = 1.0 + theta[-2] # noqa: N806 + beta = theta[-1] sigma = 0.04 # BH noise: @@ -133,7 +133,7 @@ def BH4( # noqa: N802 x[0] = x_lag2 x[1] = x_lag1 - for i in range(min(int(len(theta) / 2), 4)): + for i in range((len(theta) // 2) - 1): g[i] = theta[i * 2] b[i] = theta[i * 2 + 1] @@ -150,7 +150,7 @@ def BH4( # noqa: N802 # divEpsMax = 0 # 0.05 # dividend_noise = np.random.uniform(low=divEpsMin, high=divEpsMax, size=1) # DP noise: - dividend_noise = np.random.normal(0, sigma, 1) + dividend_noise = rng.normal(0, sigma, 1) x[t] = (np.sum(weighted_exp) + dividend_noise) / R left_factor = x[t] - R * x[t - 1] diff --git a/tests/test_examples/test_brock_hommes.py b/tests/test_examples/test_brock_hommes.py index dacd5682..2f061c68 100644 --- a/tests/test_examples/test_brock_hommes.py +++ b/tests/test_examples/test_brock_hommes.py @@ -142,105 +142,105 @@ def test_bh4_model() -> None: seed = 42 expected_output = np.array( [ - [0.0892263], - [0.0563433], - [0.06414067], - [0.1050808], - [0.06465298], - [0.03484719], - [0.08615989], - [0.09137854], - [0.04505338], - [0.05185943], - [0.0178482], - [-0.00671861], - [0.00442111], - [-0.07247632], - [-0.12025483], - [-0.10683951], - [-0.11411574], - [-0.06706414], - [-0.08159742], - [-0.11298336], - [-0.02118343], - [-0.02181211], - [-0.01250533], - [-0.06493467], - [-0.06777318], - [-0.04278995], - [-0.07482654], - [-0.0378188], - [-0.04933496], - [-0.04609427], - [-0.05581557], - [0.03434773], - [0.02512761], - [-0.02459935], - [0.01448361], - [-0.0374949], - [-0.01884807], - [-0.09034455], - [-0.11689355], - [-0.07402553], - [-0.02132221], - [-0.00696484], - [-0.00913479], - [-0.01832021], - [-0.07147812], - [-0.07928009], - [-0.073528], - [-0.00914052], - [0.00854694], - [-0.06352846], - [-0.03278851], - [-0.03743684], - [-0.05293504], - [-0.01289774], - [0.03266696], - [0.06049519], - [0.00938186], - [-0.00674687], - [0.00810651], - [0.04456955], - [0.01274162], - [0.00086996], - [-0.04344781], - [-0.07846817], - [-0.02308353], - [0.03877772], - [0.0253172], - [0.05708022], - [0.05465977], - [0.01240909], - [0.02209176], - [0.07646935], - [0.05285044], - [0.09823849], - [-0.03447823], - [0.00596483], - [0.00840848], - [-0.00594631], - [-0.00078766], - [-0.07916013], - [-0.06531819], - [-0.03099877], - [0.03767566], - [0.00701667], - [-0.0277527], - [-0.03985857], - [0.00829645], - [0.01974978], - [-0.00702008], - [0.01491091], - [0.01465264], - [0.04854568], - [0.00663405], - [-0.00920254], - [-0.0222435], - [-0.07368844], - [-0.04054655], - [-0.0171874], - [-0.01127672], + [0.08162246], + [0.01518795], + [0.0389381], + [0.06481314], + [-0.03167937], + [-0.07550877], + [-0.04828303], + [-0.04554649], + [-0.03227944], + [-0.05596049], + [-0.00456788], + [0.02865612], + [0.02319651], + [0.06066374], + [0.06145787], + [0.00871906], + [0.01960806], + [-0.02412119], + [0.01714444], + [0.01077186], + [4.172e-05], + [-0.02715309], + [0.02899408], + [0.01516408], + [-0.00669485], + [-0.01903755], + [0.00759759], + [0.02028846], + [0.03070637], + [0.03862254], + [0.11183309], + [0.0631743], + [0.02264867], + [-0.01730684], + [0.01156946], + [0.05333543], + [0.03340287], + [-0.01044686], + [-0.04078947], + [-0.00319678], + [0.02796493], + [0.04157364], + [0.00282316], + [0.01038224], + [0.01199221], + [0.01703188], + [0.04645603], + [0.04174959], + [0.05582337], + [0.04178035], + [0.04021709], + [0.05293433], + [-0.02064256], + [-0.0284747], + [-0.03858314], + [-0.05233422], + [-0.04756549], + [0.02622807], + [-0.01459457], + [0.02737786], + [-0.0467665], + [-0.04727375], + [-0.02643611], + [0.00525546], + [0.0324541], + [0.05452664], + [0.02454655], + [-0.00184605], + [0.03216834], + [0.01547663], + [-0.04009435], + [-0.07386154], + [-0.08844246], + [-0.0421084], + [-0.02268923], + [0.01196387], + [-0.00791187], + [0.00037916], + [0.02520534], + [0.00577081], + [0.02171413], + [-0.01079571], + [-0.02253499], + [-0.0310218], + [-0.06910108], + [-0.02953476], + [-0.03832997], + [-0.02633743], + [0.00096724], + [0.01890278], + [0.03985309], + [0.02423046], + [-0.00022935], + [-0.0038057], + [-0.06954288], + [-0.10698327], + [-0.12752616], + [-0.12860932], + [-0.07361663], [0.0], ], ) diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index a19567be..34fb541d 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -112,14 +112,7 @@ def test_clip_losses() -> None: assert np.allclose( losses, np.array( - [ - 7.46098998e02, - 5.80544566e17, - 3.40282347e38, - 3.40282347e38, - 3.40282347e38, - 2.94273501e41, - ], + [0.18388932, 0.58118863, 0.84728975, 0.87882275, 0.88818152, 1.2508034], ), ) From bca1344c33f6d40229d24f47634d4e1025af8ee9 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 17:00:31 +0200 Subject: [PATCH 47/56] lint: fix NPY002 for examples/models/simple_models.py The fixes changed the behaviour of the models even if the same seed was fed before and after the changes. To reproduce: ``` import numpy as np from scipy.stats import alpha, bernoulli np.random.seed(0) print("NP.RANDOM") print(bernoulli.rvs(0.5, size=1)) print(bernoulli.rvs(0.5, size=1)) print("RNG") rng = np.random.default_rng(seed=0) print(bernoulli.rvs(0.5, size=1, random_state=rng)) print(bernoulli.rvs(0.5, size=1, random_state=rng)) ``` Output: ``` NP.RANDOM [1] [1] RNG [1] [0] ``` --- README.md | 2 +- examples/models/simple_models.py | 119 ++++++++++++++++--------------- tests/test_calibrator.py | 108 ++++++++++++++-------------- tests/test_examples/test_main.py | 2 +- 4 files changed, 117 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 5802441f..9c22e1fb 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ When the calibration terminates (~half a minute), towards the end of the output you should see the following messages: ``` True parameters: [0.2, 0.2, 0.75] -Best parameters found: [0.19 0.19 0.74] +Best parameters found: [0.19 0.21 0.68] ``` ## Docs diff --git a/examples/models/simple_models.py b/examples/models/simple_models.py index d1e014f7..47aeb78d 100644 --- a/examples/models/simple_models.py +++ b/examples/models/simple_models.py @@ -26,22 +26,25 @@ - RWSB1 - RWSB2 """ +from __future__ import annotations -from typing import Sequence +from typing import TYPE_CHECKING, Sequence import numpy as np -from numpy.typing import NDArray from scipy.stats import alpha, bernoulli +if TYPE_CHECKING: + from numpy.typing import NDArray + def NormalM( # noqa: N802 theta: Sequence[float], N: int, # noqa: N803 - seed: int, + seed: int | None, ) -> NDArray[np.float64]: """Normal samples with adjustable mean.""" - np.random.seed(seed=seed) - y = np.random.normal(theta[0], 1, N) + rng = np.random.default_rng(seed=seed) + y = rng.normal(theta[0], 1, N) return np.atleast_2d(y).T @@ -51,9 +54,9 @@ def NormalMV( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Normal samples with adjustable mean and variance.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - y = np.random.normal(theta[0], theta[1], N) + y = rng.normal(theta[0], theta[1], N) return np.atleast_2d(y).T @@ -63,15 +66,15 @@ def NormalBer_3P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) y = np.zeros(N) - b = bernoulli.rvs(theta[0], size=N) + b = bernoulli.rvs(theta[0], size=N, random_state=rng) for i in range(N): if b[i] == 1: - y[i] = np.random.normal(theta[1], 0.1, 1) + y[i] = rng.normal(theta[1], 0.1, 1) else: - y[i] = np.random.normal(theta[2], 0.1, 1) + y[i] = rng.normal(theta[2], 0.1, 1) return np.atleast_2d(y).T @@ -82,15 +85,15 @@ def NormalBer_5P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Bernoulli + Normal samples.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) y = np.zeros(N) - b = bernoulli.rvs(theta[0], size=N) + b = bernoulli.rvs(theta[0], size=N, random_state=rng) for i in range(N): if b[i] == 1: - y[i] = np.random.normal(theta[1], theta[3], 1) + y[i] = rng.normal(theta[1], theta[3], 1) else: - y[i] = np.random.normal(theta[2], theta[4], 1) + y[i] = rng.normal(theta[2], theta[4], 1) return np.atleast_2d(y).T @@ -106,9 +109,9 @@ def Alpha( # noqa: N802 theta[1] position param theta[2] scale param """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - y = (alpha.rvs(theta[0], size=N) + theta[1]) * theta[2] + y = (alpha.rvs(theta[0], size=N, random_state=rng) + theta[1]) * theta[2] return np.atleast_2d(y).T @@ -119,7 +122,7 @@ def MarkovC_2P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) sigma = 0.0 y = np.zeros(N) @@ -128,19 +131,19 @@ def MarkovC_2P( # noqa: N802 s = 0 for i in range(N): if s == 0: - b = bernoulli.rvs(theta[0], size=1) + b = bernoulli.rvs(theta[0], size=1, random_state=rng) if b == 1: s = 1 - y[i] = np.random.normal(x[1], sigma, 1) + y[i] = rng.normal(x[1], sigma, 1) else: - y[i] = np.random.normal(x[0], sigma, 1) + y[i] = rng.normal(x[0], sigma, 1) if s == 1: - b = bernoulli.rvs(theta[1], size=1) + b = bernoulli.rvs(theta[1], size=1, random_state=rng) if b == 1: s = 0 - y[i] = np.random.normal(x[0], sigma, 1) + y[i] = rng.normal(x[0], sigma, 1) else: - y[i] = np.random.normal(x[1], sigma, 1) + y[i] = rng.normal(x[1], sigma, 1) return np.atleast_2d(y).T @@ -151,7 +154,7 @@ def MarkovC_3P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) sigma = 0.0 y = np.zeros(N) @@ -160,13 +163,13 @@ def MarkovC_3P( # noqa: N802 s = 0 for i in range(N): - b = bernoulli.rvs(theta[s], size=1) + b = bernoulli.rvs(theta[s], size=1, random_state=rng) if b == 1: s = s + 1 if s == len(x): s = 0 - y[i] = np.random.normal(x[s], sigma, 1) + y[i] = rng.normal(x[s], sigma, 1) return np.atleast_2d(y).T @@ -177,7 +180,7 @@ def MarkovC_KP( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """Markov chain samples.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) sigma = 0.0 y = np.zeros(N) @@ -185,13 +188,13 @@ def MarkovC_KP( # noqa: N802 s = 0 for i in range(N): - b = bernoulli.rvs(theta[s], size=1) + b = bernoulli.rvs(theta[s], size=1, random_state=rng) if b == 1: s = s + 1 if s == len(theta): s = 0 - y[i] = np.random.normal(s, sigma, 1) + y[i] = rng.normal(s, sigma, 1) return np.atleast_2d(y).T @@ -205,14 +208,14 @@ def AR1( # noqa: N802 Model 1 in Platt (2019) """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) # AR(1) # Y_t = p0*Y_t-1 + eps y = np.zeros(N) - y[0] = np.random.normal(0, 1, 1) + y[0] = rng.normal(0, 1, 1) for i in range(1, N): - y[i] = theta[0] * y[i - 1] + np.random.normal(0, 1, 1) + y[i] = theta[0] * y[i - 1] + rng.normal(0, 1, 1) return np.atleast_2d(y).T @@ -223,11 +226,11 @@ def AR1Ber_3P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """AR(1) + Bernoulli.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) y = np.zeros(N) - e = np.random.normal(0, 0.1, N) - b = bernoulli.rvs(theta[0], size=N) + e = rng.normal(0, 0.1, N) + b = bernoulli.rvs(theta[0], size=N, random_state=rng) # s = 0 y[0] = e[0] @@ -248,14 +251,14 @@ def AR1_2P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """AR(1) with 2 parameters.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) # AR(1) # Y_t = p0*Y_t-1 + eps y = np.zeros(N) - y[0] = np.random.normal(0, 1, 1) + y[0] = rng.normal(0, 1, 1) for i in range(1, N): - y[i] = theta[0] * y[i - 1] + np.random.normal(0, theta[1], 1) + y[i] = theta[0] * y[i - 1] + rng.normal(0, theta[1], 1) return np.atleast_2d(y).T @@ -265,14 +268,14 @@ def AR1_3P( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """AR(1) with 3 parameters.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) # AR(1) # Y_t = p0*Y_t-1 + eps y = np.zeros(N) - y[0] = np.random.normal(0, 1, 1) + y[0] = rng.normal(0, 1, 1) for i in range(1, N): - y[i] = theta[0] * y[i - 1] + np.random.normal(theta[2], theta[1], 1) + y[i] = theta[0] * y[i - 1] + rng.normal(theta[2], theta[1], 1) return np.atleast_2d(y).T @@ -283,9 +286,9 @@ def ARMA2( # noqa: N802 seed: int, ) -> NDArray[np.float64]: """ARMA(1, 1) model.""" - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - e = np.random.normal(0, 1, N + 1) + e = rng.normal(0, 1, N + 1) y = np.zeros(N + 1) for i in range(1, N): y[i] = theta[0] * y[i - 1] + theta[1] * e[i - 1] + e[i] @@ -303,9 +306,9 @@ def ARMAARCH2( # noqa: N802 Model 2 of Platt (2019) - Param set 1 [a0,a1] [*a0,*a1,a2,b1,b2,c0,c1,c2] [*0,*0.7,0.1,0.2,0.2,0.25,0.5,0.3] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - e = np.random.normal(0, 1, N + 2) + e = rng.normal(0, 1, N + 2) s = np.zeros(N + 2) y = np.zeros(N + 2) for i in range(2, N + 1): @@ -333,9 +336,9 @@ def ARMAARCH4( # noqa: N802 DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - e = np.random.normal(0, 1, N + 2) + e = rng.normal(0, 1, N + 2) s = np.zeros(N + 2) y = np.zeros(N + 2) for i in range(2, N + 1): @@ -365,9 +368,9 @@ def ARMAARCH6( # noqa: N802 DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - e = np.random.normal(0, 1, N + 2) + e = rng.normal(0, 1, N + 2) s = np.zeros(N + 2) y = np.zeros(N + 2) for i in range(2, N + 1): @@ -399,9 +402,9 @@ def ARMAARCH4v2( # noqa: N802 DONOVAN MODEL 2 - PARAM SET 2 [b1,b2,c0,c1,c2] [a0,a1,a2,*b1,*b2,*c0,*c1,*c2] [0,0.7,0.1,*0.2,*0.2,*0.25,*0.5,*0.3] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) - e = np.random.normal(0, 1, N + 2) + e = rng.normal(0, 1, N + 2) s = np.zeros(N + 2) y = np.zeros(N + 2) for i in range(2, N + 1): @@ -429,11 +432,11 @@ def RWSB1( # noqa: N802 DONOVAN MODEL 3 - PARAM SET 1 [tau] [*tau,sigma1,sigma2,drift1,drift2] [*700,0.1,0.2,1,2] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) e = np.zeros(N) - e[0 : theta[0]] = np.random.normal(0, 0.1, theta[0]) - e[theta[0] :] = np.random.normal(0, 0.2, N - theta[0]) + e[0 : theta[0]] = rng.normal(0, 0.1, theta[0]) + e[theta[0] :] = rng.normal(0, 0.2, N - theta[0]) d = np.full(N, 1) d[theta[0] :] = 2 @@ -458,11 +461,11 @@ def RWSB2( # noqa: N802 DONOVAN MODEL 3 - PARAM SET 2 [sigma1,sigma2] [tau,*sigma1,*sigma2,drift1,drift2] [700,*0.1,*0.2,1,2] """ - np.random.seed(seed=seed) + rng = np.random.default_rng(seed=seed) e = np.zeros(N) - e[0:700] = np.random.normal(0, theta[0], 700) - e[700:] = np.random.normal(0, theta[1], N - 700) + e[0:700] = rng.normal(0, theta[0], 700) + e[700:] = rng.normal(0, theta[1], N - 700) d = np.full(N, 1) d[700:] = 2 diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index 03bb8bcd..edaef30d 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -45,74 +45,74 @@ class TestCalibrate: expected_params = np.array( [ - [0.59, 0.36], - [0.63, 0.41], - [0.18, 0.39], + [0.52, 0.01], [0.56, 0.37], - [0.61, 0.53], - [0.54, 0.32], - [0.74, 0.32], + [0.59, 0.36], + [0.55, 0.46], [0.53, 0.46], - [0.55, 0.62], - [0.71, 0.35], - [0.32, 0.93], [0.8, 0.06], [0.99, 1.0], - [0.06, 0.98], + [0.74, 0.32], + [0.6, 0.42], + [0.18, 0.39], + [0.51, 0.33], + [0.04, 0.99], + [0.32, 0.93], + [0.56, 0.24], ], ) expected_losses = [ - 0.33400294, - 0.55274918, - 0.55798021, - 0.61712034, - 0.78963342, - 1.31118518, - 1.51682355, - 1.55503666, - 1.68181078, - 1.70075834, - 1.79905545, - 2.07605975, - 2.25201126, - 2.97360386, + 0.87950070, + 0.99224516, + 1.15590624, + 1.24380484, + 1.76330622, + 1.88165325, + 2.30766018, + 2.55676207, + 2.86482802, + 2.88057794, + 2.90585611, + 3.77705872, + 4.47466328, + 5.79615295, ] - darwin_expected_params = np.array( + win32_expected_params = np.array( [ - [0.59, 0.36], - [0.63, 0.41], - [0.18, 0.39], + [0.15, 0.35], [0.56, 0.37], - [0.63, 0.31], - [0.54, 0.32], - [0.37, 0.59], - [0.74, 0.32], + [0.59, 0.36], + [0.57, 0.38], [0.53, 0.46], - [0.57, 0.39], + [0.80, 0.06], + [0.99, 1.00], + [0.74, 0.32], + [0.60, 0.42], + [0.18, 0.39], + [0.51, 0.33], + [0.04, 0.99], [0.32, 0.93], - [0.03, 0.36], - [0.8, 0.06], - [0.06, 0.98], + [0.70, 0.53], ], ) - darwin_expected_losses = [ - 0.33400294, - 0.55274918, - 0.55798021, - 0.61712034, - 0.89679611, - 1.31118518, - 1.3961825, - 1.51682355, - 1.55503666, - 1.65968375, - 1.79905545, - 1.92174866, - 2.07605975, - 2.97360386, + win32_expected_losses = [ + 0.43742215, + 0.99224516, + 1.15590624, + 1.25892187, + 1.76330622, + 1.88165325, + 2.30766018, + 2.55676207, + 2.86482802, + 2.88057794, + 2.90585611, + 3.77705872, + 4.47466328, + 5.67663570, ] def setup(self) -> None: @@ -173,9 +173,9 @@ def test_calibrator_calibrate(self, n_jobs: int) -> None: # This is a temporary workaround to make tests to run also on Windows. # See: https://github.com/bancaditalia/black-it/issues/49 - if sys.platform == "darwin": - assert np.allclose(params, self.darwin_expected_params) - assert np.allclose(losses, self.darwin_expected_losses) + if sys.platform == "win32": + assert np.allclose(params, self.win32_expected_params) + assert np.allclose(losses, self.win32_expected_losses) else: assert np.allclose(params, self.expected_params) assert np.allclose(losses, self.expected_losses) diff --git a/tests/test_examples/test_main.py b/tests/test_examples/test_main.py index b7e9cb28..639287e5 100644 --- a/tests/test_examples/test_main.py +++ b/tests/test_examples/test_main.py @@ -23,7 +23,7 @@ TRUE_PARAMETERS_STR = "True parameters: [0.2, 0.2, 0.75]" -BEST_PARAMETERS_STR = "Best parameters found: [0.19 0.19 0.74]" +BEST_PARAMETERS_STR = "Best parameters found: [0.19 0.21 0.68]" class TestMainExample(BaseMainExampleTestClass): From 2269dfa878f4366361ea9510aeb3d3db0339ea82 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Wed, 30 Aug 2023 17:21:09 +0200 Subject: [PATCH 48/56] lint: fix NPY002 for examples/models/sir/sir_python.py Fixed NPY002 issues in SIR models implemented in examples/models/sir/sir_python.py. Added tests to verify the new seed management works. --- .gitignore | 2 + examples/models/sir/sir_python.py | 8 +- tests/fixtures/data/test_sir_python.npy | Bin 0 -> 2528 bytes tests/test_examples/test_sir_python.py | 96 ++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/data/test_sir_python.npy create mode 100644 tests/test_examples/test_sir_python.py diff --git a/.gitignore b/.gitignore index 94ead7f7..3d49b035 100644 --- a/.gitignore +++ b/.gitignore @@ -362,3 +362,5 @@ Java\ model/ examples/output +!tests/fixtures/data/test_sir_python.npy +!tests/fixtures/data/test_sir_python_w_breaks_python.npy \ No newline at end of file diff --git a/examples/models/sir/sir_python.py b/examples/models/sir/sir_python.py index 7154afdf..676f7460 100644 --- a/examples/models/sir/sir_python.py +++ b/examples/models/sir/sir_python.py @@ -46,12 +46,10 @@ def SIR(theta: NDArray, N: int, seed: int | None) -> NDArray: # noqa: N802, N80 Returns: simulated series """ - np.random.seed(seed=seed) - num_agents = 100000 g = nx.watts_strogatz_graph(num_agents, int(theta[0]), theta[1], seed=theta[5]) - model = ep.SIRModel(g) + model = ep.SIRModel(g, seed=seed) cfg = ModelConfig.Configuration() cfg.add_model_parameter("beta", theta[3]) # infection rate @@ -102,12 +100,10 @@ def SIR_w_breaks( # noqa: N802 Returns: simulated series """ - np.random.seed(seed=seed) - num_agents = 100000 g = nx.watts_strogatz_graph(num_agents, int(theta[0]), theta[1], seed=theta[11]) - model = ep.SIRModel(g) + model = ep.SIRModel(g, seed=seed) cfg = ModelConfig.Configuration() cfg.add_model_parameter("beta", theta[3]) # infection rate diff --git a/tests/fixtures/data/test_sir_python.npy b/tests/fixtures/data/test_sir_python.npy new file mode 100644 index 0000000000000000000000000000000000000000..5825c50b3d91dda1c471078362d77927e30b0f1f GIT binary patch literal 2528 zcmeIxNoW%R6b9hei)iqWn>lzGBs3L*Of5=~Y6m205xa<$Sc07{iEWWKrged$4243F z3UN7j5IP`u@Y2DG;zdXF;EE$C6@(HMt%%FG)#AzIf8$}^AXM?#cX|9@ejdprIo;H@ zqh(JO9iov~PiCk)AKReB>a&ShT#01|^TYYhfsVm^Pp0BqI`Yi)0Qu8niHxAN=UCbFZixIg*9*VpSy$j)}x z=i0<`WOekj-w#eBTjTtG$s4We345RS`8>CmKZZNi->^LCNzPRyb{g(?01* zxE}D$d|$eu!CmRJ5w7;i`|IFVz}Lb{0k7rz0guC#K6$?et_8dr-V*RATo3q4croA+ yxVc~tqi+ZLOX1FfozO1@`hRk_fy(0s`hV+{E9L2z&zX7A7oUUo1bF?&-^nlYK|k66 literal 0 HcmV?d00001 diff --git a/tests/test_examples/test_sir_python.py b/tests/test_examples/test_sir_python.py new file mode 100644 index 00000000..3f489d64 --- /dev/null +++ b/tests/test_examples/test_sir_python.py @@ -0,0 +1,96 @@ +# Black-box ABM Calibration Kit (Black-it) +# Copyright (C) 2021-2023 Banca d'Italia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test the SIR model implementation in Python.""" + +import numpy as np +import pytest + +try: + from examples.models.sir.sir_python import SIR, SIR_w_breaks +except ModuleNotFoundError as e: + pytest.skip( + f"skipping tests for SIR python models, reason: {str(e)}", + allow_module_level=True, + ) + +from tests.conftest import TEST_DIR + + +def test_sir() -> None: + """Test the 'SIR' function in examples/models/sir/sir_python.py.""" + expected_output = np.load(TEST_DIR / "fixtures" / "data" / "test_sir_python.npy") + model_seed = 0 + + lattice_order = 20 + rewire_probability = 0.2 + percentage_infected = 0.05 + beta = 0.2 + gamma = 0.15 + networkx_seed = 0 + theta = [ + lattice_order, + rewire_probability, + percentage_infected, + beta, + gamma, + networkx_seed, + ] + + n = 100 + output = SIR(theta, n, seed=model_seed) + + assert np.isclose(output, expected_output).all() + + +def test_sir_w_breaks() -> None: + """Test the 'SIR_w_breaks' function in examples/models/sir/sir_python.py.""" + expected_output = np.load( + TEST_DIR / "fixtures" / "data" / "test_sir_w_breaks_python.npy", + ) + model_seed = 0 + + lattice_order = 20 + rewire_probability = 0.2 + percentage_infected = 0.05 + beta_1 = 0.2 + gamma_1 = 0.15 + beta_2 = 0.3 + beta_3 = 0.1 + beta_4 = 0.01 + t_break_1 = 10 + t_break_2 = 20 + t_break_3 = 30 + networkx_seed = 0 + theta = [ + lattice_order, + rewire_probability, + percentage_infected, + beta_1, + gamma_1, + beta_2, + beta_3, + beta_4, + t_break_1, + t_break_2, + t_break_3, + networkx_seed, + ] + + n = 100 + output = SIR_w_breaks(theta, n, seed=model_seed) + + assert np.isclose(output, expected_output).all() From 859f063678b3dd9e069e27096c14f02b58a3d840 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 10:55:49 +0200 Subject: [PATCH 49/56] lint: fix NPY002 in tests/* To reproduce the errors: ``` ruff check --select "NPY" tests ``` This commit updates the tests code so to make them to use np.random.default_rng with a certain seed, rather than relying on global random seed handling, i.e. using np.random.seed. To do so, a new fixture, 'rng', has been added so to avoid the tests code to initialize a np.random.Generator instance manually. This allowed to remove the special case for macOS platforms in TestCalibrate.test_calibrator_calibrate tests (but not for Windows). --- .ruff.toml | 2 +- tests/conftest.py | 9 ++ tests/test_calibrator.py | 2 +- tests/test_losses/test_fourier.py | 14 +- tests/test_losses/test_gsl.py | 27 ++-- tests/test_losses/test_likelihood.py | 21 ++- tests/test_losses/test_msm.py | 40 +++-- .../test_plot_descriptive_statistics.py | 4 +- tests/test_samplers/test_best_batch.py | 14 +- tests/test_samplers/test_gaussian_process.py | 10 +- tests/test_samplers/test_xgboost.py | 8 +- tests/test_utils/test_base.py | 4 +- .../test_pandas_json_checkpointing.py | 12 +- tests/test_utils/test_time_series.py | 149 +++++++++++++++--- 14 files changed, 218 insertions(+), 98 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index f23b71ad..75d63454 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY","NPY"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/tests/conftest.py b/tests/conftest.py index 443d3faf..c4357044 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,9 @@ import inspect from pathlib import Path +import numpy as np +import pytest + CUR_PATH = Path(inspect.getfile(inspect.currentframe())).parent # type: ignore[arg-type] ROOT_DIR = Path(CUR_PATH, "..").resolve().absolute() DOCS_DIR = ROOT_DIR / "docs" @@ -27,3 +30,9 @@ EXAMPLE_SAVING_FOLDER = ROOT_DIR / "examples" / "saving_folder" DEFAULT_SUBPROCESS_TIMEOUT = 100.0 + + +@pytest.fixture() +def rng() -> np.random.Generator: + """Return random number generator.""" + return np.random.default_rng(seed=11) diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index edaef30d..e0046fdb 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -79,7 +79,7 @@ class TestCalibrate: 5.79615295, ] - win32_expected_params = np.array( + win32_expected_params: NDArray = np.array( [ [0.15, 0.35], [0.56, 0.37], diff --git a/tests/test_losses/test_fourier.py b/tests/test_losses/test_fourier.py index ae0efe1e..a60b36de 100644 --- a/tests/test_losses/test_fourier.py +++ b/tests/test_losses/test_fourier.py @@ -23,32 +23,30 @@ ) -def test_fourier_ideal_low_pass() -> None: +def test_fourier_ideal_low_pass(rng: np.random.Generator) -> None: """Test the Fourier loss with the ideal low-pass filter.""" - np.random.seed(11) series_real = np.sin(np.linspace(0, 50, 1000))[:, None] - series_sim = series_real + np.random.normal(0, 0.1, series_real.shape) + series_sim = series_real + rng.normal(0, 0.1, series_real.shape) euclidean_loss = np.sqrt(np.sum((series_sim - series_real) ** 2)) # check that for no filter (f=1.0) this loss is approximately equivalent to # the Euclidean loss and that with increasingly aggressive filters (up to # f=0.01) the loss goes towards zero. - expected_losses = [euclidean_loss, 2.23, 0.97, 0.27] + expected_losses = [euclidean_loss, 2.21, 0.97, 0.187] for i, f in enumerate([1.0, 0.5, 0.1, 0.01]): loss_func = FourierLoss(f=f, frequency_filter=ideal_low_pass_filter) loss = loss_func.compute_loss(series_sim[None, :, :], series_real) assert np.isclose(expected_losses[i], loss, atol=0.01) -def test_fourier_gaussian_low_pass() -> None: +def test_fourier_gaussian_low_pass(rng: np.random.Generator) -> None: """Test the Fourier loss with the gaussian low-pass filter.""" - np.random.seed(11) series_real = np.sin(np.linspace(0, 50, 1000))[:, None] - series_sim = series_real + np.random.normal(0, 0.1, series_real.shape) + series_sim = series_real + rng.normal(0, 0.1, series_real.shape) # check that with increasingly aggressive filters (up to f=0.01) the loss # goes towards zero. - expected_losses = [2.75, 0.95, 0.27] + expected_losses = [2.73, 0.95, 0.19] for i, f in enumerate([1.0, 0.1, 0.01]): loss_func = FourierLoss(f=f, frequency_filter=gaussian_low_pass_filter) loss = loss_func.compute_loss(series_sim[None, :, :], series_real) diff --git a/tests/test_losses/test_gsl.py b/tests/test_losses/test_gsl.py index 5c63720c..d812a571 100644 --- a/tests/test_losses/test_gsl.py +++ b/tests/test_losses/test_gsl.py @@ -98,13 +98,12 @@ def test_get_words(self, args: tuple) -> None: # noqa: PLR6301 GslDivLoss.get_words(*args) -def test_gsl_default() -> None: +def test_gsl_default(rng: np.random.Generator) -> None: """Test the Gsl-div loss function.""" - expected_loss = 0.39737637181336855 + expected_loss = 0.3972285978726733 - np.random.seed(11) - series_sim = np.random.normal(0, 1, (100, 3)) - series_real = np.random.normal(0, 1, (100, 3)) + series_sim = rng.normal(0, 1, (100, 3)) + series_real = rng.normal(0, 1, (100, 3)) loss_func = GslDivLoss() loss = loss_func.compute_loss(series_sim[None, :, :], series_real) @@ -112,13 +111,12 @@ def test_gsl_default() -> None: assert np.isclose(expected_loss, loss) -def test_gsl_with_nb_values() -> None: +def test_gsl_with_nb_values(rng: np.random.Generator) -> None: """Test the Gsl-div loss function with nb_values set.""" - expected_loss = 0.4353415724764564 + expected_loss = 0.4354049587629579 - np.random.seed(11) - series_sim = np.random.normal(0, 1, (2, 100, 3)) - series_real = np.random.normal(0, 1, (100, 3)) + series_sim = rng.normal(0, 1, (2, 100, 3)) + series_real = rng.normal(0, 1, (100, 3)) loss_func = GslDivLoss(nb_values=10) loss = loss_func.compute_loss(series_sim, series_real) @@ -126,13 +124,12 @@ def test_gsl_with_nb_values() -> None: assert np.isclose(expected_loss, loss) -def test_gsl_with_nb_word_lengths() -> None: +def test_gsl_with_nb_word_lengths(rng: np.random.Generator) -> None: """Test the Gsl-div loss function with nb_word_lengths set.""" - expected_loss = 0.7210261201578492 + expected_loss = 0.7177347914787273 - np.random.seed(11) - series_sim = np.random.normal(0, 1, (100, 3)) - series_real = np.random.normal(0, 1, (100, 3)) + series_sim = rng.normal(0, 1, (100, 3)) + series_real = rng.normal(0, 1, (100, 3)) loss_func = GslDivLoss(nb_word_lengths=10) loss = loss_func.compute_loss(series_sim[None, :, :], series_real) diff --git a/tests/test_losses/test_likelihood.py b/tests/test_losses/test_likelihood.py index ef732007..e3a0dfcd 100644 --- a/tests/test_losses/test_likelihood.py +++ b/tests/test_losses/test_likelihood.py @@ -19,11 +19,10 @@ from black_it.loss_functions.likelihood import LikelihoodLoss -def test_likelihood_1d() -> None: +def test_likelihood_1d(rng: np.random.Generator) -> None: """Test the computation of the Likelihood in the Likelihood loss in 1d.""" # sample from a Gaussian distribution. - np.random.seed(11) - real_data = np.random.normal(0, 1, size=(7, 1)) + real_data = rng.normal(0, 1, size=(7, 1)) expected_neg_log_likelihood = -np.sum( -0.5 * np.sum(real_data**2, axis=1) - 0.5 * np.log(2.0 * np.pi), @@ -31,37 +30,35 @@ def test_likelihood_1d() -> None: ) expected_likelihood = np.exp(-expected_neg_log_likelihood) - sim_data_ensemble = np.random.normal(0, 1, size=(3, 100000, 1)) + sim_data_ensemble = rng.normal(0, 1, size=(3, 100000, 1)) loss = LikelihoodLoss(h="silverman") neg_log_lik = loss.compute_loss(sim_data_ensemble, real_data) lik = np.exp(-neg_log_lik) assert np.isclose(lik, expected_likelihood, rtol=0.1) -def test_likelihood_2d() -> None: +def test_likelihood_2d(rng: np.random.Generator) -> None: """Test the computation of the Likelihood in the Likelihood loss in 2d.""" # sample from a Gaussian distribution. - np.random.seed(11) - real_data = np.random.normal(0, 1, size=(10, 2)) + real_data = rng.normal(0, 1, size=(10, 2)) expected_neg_log_likelihood = -np.sum( -0.5 * np.sum(real_data**2, axis=1) - 2.0 / 2.0 * np.log(2.0 * np.pi), axis=0, ) expected_likelihood = np.exp(-expected_neg_log_likelihood) - sim_data_ensemble = np.random.normal(0, 1, size=(1, 1000000, 2)) + sim_data_ensemble = rng.normal(0, 1, size=(1, 1000000, 2)) loss = LikelihoodLoss(h=1.0) neg_log_lik = loss.compute_loss(sim_data_ensemble, real_data) lik = np.exp(-neg_log_lik) assert np.isclose(lik, expected_likelihood, rtol=0.1) -def test_likelihood_2d_wsigma() -> None: +def test_likelihood_2d_wsigma(rng: np.random.Generator) -> None: """Test the computation of the Likelihood in the Likelihood loss in 2d.""" # sample from a Gaussian distribution. - np.random.seed(11) sigma, d = 3.0, 2 - real_data = np.random.normal(0, sigma, size=(10, d)) + real_data = rng.normal(0, sigma, size=(10, d)) expected_neg_log_likelihood = -np.sum( -0.5 / sigma**2 * np.sum(real_data**2, axis=1) @@ -70,7 +67,7 @@ def test_likelihood_2d_wsigma() -> None: ) expected_likelihood = np.exp(-expected_neg_log_likelihood) - sim_data_ensemble = np.random.normal(0, sigma, size=(1, 1000000, d)) + sim_data_ensemble = rng.normal(0, sigma, size=(1, 1000000, d)) loss = LikelihoodLoss(h=sigma) neg_log_lik = loss.compute_loss(sim_data_ensemble, real_data) lik = np.exp(-neg_log_lik) diff --git a/tests/test_losses/test_msm.py b/tests/test_losses/test_msm.py index de8c3797..e0ca8a5e 100644 --- a/tests/test_losses/test_msm.py +++ b/tests/test_losses/test_msm.py @@ -23,14 +23,13 @@ from black_it.loss_functions.msm import MethodOfMomentsLoss -def test_msm_default() -> None: +def test_msm_default(rng: np.random.Generator) -> None: """Test the 'method of moments' loss.""" - expected_loss = 2.830647081075395 + expected_loss = 2.054721024744742 - np.random.seed(11) # ensemble size 2, time series length 100, number of variables 3 - series_sim = np.random.normal(0, 1, (2, 100, 3)) - series_real = np.random.normal(0, 1, (100, 3)) + series_sim = rng.normal(0, 1, (2, 100, 3)) + series_real = rng.normal(0, 1, (100, 3)) loss_func = MethodOfMomentsLoss() @@ -39,15 +38,16 @@ def test_msm_default() -> None: assert np.isclose(expected_loss, loss) -def test_msm_default_calculator_custom_covariance_matrix() -> None: +def test_msm_default_calculator_custom_covariance_matrix( + rng: np.random.Generator, +) -> None: """Test the MSM loss when the covariance matrix is not None.""" - expected_loss = 16.49853079135471 + expected_loss = 7.569748247731355 - np.random.seed(11) - series_sim = np.random.normal(0, 1, (2, 100, 3)) - series_real = np.random.normal(0, 1, (100, 3)) + series_sim = rng.normal(0, 1, (2, 100, 3)) + series_real = rng.normal(0, 1, (100, 3)) - random_mat = np.random.rand(18, 18) + random_mat = rng.random(size=(18, 18)) covariance_matrix = random_mat.T.dot(random_mat) loss_func = MethodOfMomentsLoss(covariance_mat=covariance_matrix) @@ -57,19 +57,23 @@ def test_msm_default_calculator_custom_covariance_matrix() -> None: assert np.isclose(expected_loss, loss) -def test_msm_default_calculator_non_symmetric_covariance_matrix() -> None: +def test_msm_default_calculator_non_symmetric_covariance_matrix( + rng: np.random.Generator, +) -> None: """Test the MSM loss raises error when the provided matrix is not symmetric.""" with pytest.raises( ValueError, match="the provided covariance matrix is not valid as it is not a symmetric matrix", ): - MethodOfMomentsLoss(covariance_mat=np.random.rand(2, 3)) + MethodOfMomentsLoss(covariance_mat=rng.random(size=(2, 3))) -def test_msm_default_calculator_wrong_shape_covariance_matrix() -> None: +def test_msm_default_calculator_wrong_shape_covariance_matrix( + rng: np.random.Generator, +) -> None: """Test the MSM loss raises error when the covariance matrix has wrong shape.""" dimension = 20 - random_mat = np.random.rand(dimension, dimension) + random_mat = rng.random(size=(dimension, dimension)) wrong_covariance_matrix = random_mat.T.dot(random_mat) with pytest.raises( ValueError, @@ -93,7 +97,9 @@ def custom_moment_calculator(time_series: NDArray) -> NDArray: assert np.isclose(expected_loss, loss) -def test_msm_custom_calculator_wrong_shape_covariance_matrix() -> None: +def test_msm_custom_calculator_wrong_shape_covariance_matrix( + rng: np.random.Generator, +) -> None: """Test the 'method of moments' loss with a custom calculator and a custom covariance of the wrong shape.""" series_sim = np.array([[0, 1]]).T series_real = np.array([[1, 2]]).T @@ -102,7 +108,7 @@ def custom_moment_calculator(time_series: NDArray) -> NDArray: return np.array([np.mean(time_series)]) dimension = 3 - random_mat = np.random.rand(dimension, dimension) + random_mat = rng.random(size=(dimension, dimension)) wrong_covariance_matrix = random_mat.T.dot(random_mat) loss_func = MethodOfMomentsLoss( diff --git a/tests/test_plot/test_plot_descriptive_statistics.py b/tests/test_plot/test_plot_descriptive_statistics.py index 1cb3c90a..ff3c1839 100644 --- a/tests/test_plot/test_plot_descriptive_statistics.py +++ b/tests/test_plot/test_plot_descriptive_statistics.py @@ -37,6 +37,6 @@ class TestTsStats(BasePlotTest): def setup(self) -> None: """Set up the test.""" - np.random.seed(42) - data = np.random.rand(100) + rng = np.random.default_rng(42) + data = rng.random(100) self.args = [data] diff --git a/tests/test_samplers/test_best_batch.py b/tests/test_samplers/test_best_batch.py index 1d5191fe..2f13c7d2 100644 --- a/tests/test_samplers/test_best_batch.py +++ b/tests/test_samplers/test_best_batch.py @@ -61,7 +61,7 @@ def test_best_batch_2d() -> None: assert np.allclose(expected_params, new_params) -def test_best_batch_clipping() -> None: +def test_best_batch_clipping(rng: np.random.Generator) -> None: """Test the best-batch clipping when sampling.""" lower_bound, upper_bound = 0.499, 0.5 sampler = BestBatchSampler(batch_size=8, random_state=0) @@ -73,19 +73,21 @@ def test_best_batch_clipping() -> None: verbose=False, ) - existing_points = np.random.rand(8, 2) - existing_losses = np.random.rand(8) + existing_points = rng.random(size=(8, 2)) + existing_losses = rng.random(8) new_params = sampler.sample(param_grid, existing_points, existing_losses) assert (new_params == lower_bound).any() or (new_params == upper_bound).any() -def test_best_batch_sample_requires_batch_size_existing_points() -> None: +def test_best_batch_sample_requires_batch_size_existing_points( + rng: np.random.Generator, +) -> None: """Test that BestBatch.sample must be greater or equal than batch size.""" nb_points = 8 batch_size = 16 - existing_points = np.random.rand(nb_points, 2) - existing_losses = np.random.rand(nb_points) + existing_points = rng.random(size=(nb_points, 2)) + existing_losses = rng.random(size=nb_points) param_grid = SearchSpace( parameters_bounds=np.array([[0, 1], [0, 1]]).T, parameters_precision=np.array([0.01, 0.01]), diff --git a/tests/test_samplers/test_gaussian_process.py b/tests/test_samplers/test_gaussian_process.py index 57e95d73..3f2a8d3a 100644 --- a/tests/test_samplers/test_gaussian_process.py +++ b/tests/test_samplers/test_gaussian_process.py @@ -31,24 +31,25 @@ class TestGaussianProcess2D: def setup(self) -> None: """Set up the test.""" - self.xys, self.losses = self._construct_fake_grid() + self.xys, self.losses = self._construct_fake_grid(seed=0) @classmethod def _construct_fake_grid( cls, + seed: int = 0, n: int = 3, ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: """Construct a fake grid of evaluated losses.""" + rng = np.random.default_rng(seed) xs = np.linspace(0, 1, n) ys = np.linspace(0, 1, n) xys_list = [] losses_list = [] - np.random.seed(0) for x in xs: for y in ys: - px = x + np.random.normal(0, 1e-2) - py = y + np.random.normal(0, 1e-2) + px = x + rng.normal(0, 1e-2) + py = y + rng.normal(0, 1e-2) xys_list.append([px, py]) losses_list.append(px**2 + py**2) @@ -115,6 +116,7 @@ def test_gaussian_process_sample_warning_too_large_dataset() -> None: xys, losses, ) = TestGaussianProcess2D._construct_fake_grid( # noqa: SLF001 + seed=0, n=23, ) with pytest.warns( diff --git a/tests/test_samplers/test_xgboost.py b/tests/test_samplers/test_xgboost.py index 34fb541d..5a7a3c4c 100644 --- a/tests/test_samplers/test_xgboost.py +++ b/tests/test_samplers/test_xgboost.py @@ -76,14 +76,14 @@ def test_clip_losses() -> None: 0.9, # g3 -0.2, # b3 1.01, # g4 - 0.01, + 0.01, # b4 ], - ) # b4 + ) parameter_bounds = [ [0.0, 0.0, 0.0, 0.0, 0.0, -1.0, 1.0, 0.0], # lower bounds - [0.1, 0.1, 1.0, 1.0, 1.0, 0.0, 1.1, 1.0], - ] # upper bounds + [0.1, 0.1, 1.0, 1.0, 1.0, 0.0, 1.1, 1.0], # upper bounds + ] precisions = [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01] diff --git a/tests/test_utils/test_base.py b/tests/test_utils/test_base.py index 5b599457..4090ce10 100644 --- a/tests/test_utils/test_base.py +++ b/tests/test_utils/test_base.py @@ -65,13 +65,13 @@ def test_digitize_data() -> None: assert output_array == pytest.approx(expected_output) -def test_is_symmetric() -> None: +def test_is_symmetric(rng: np.random.Generator) -> None: """Test the 'is_symmetric' function.""" assert is_symmetric(np.zeros((0, 0))) assert is_symmetric(np.zeros((4, 4))) assert is_symmetric(np.ones((4, 4))) - assert not is_symmetric(np.random.rand(4, 4)) + assert not is_symmetric(rng.random(size=(4, 4))) @given( diff --git a/tests/test_utils/test_pandas_json_checkpointing.py b/tests/test_utils/test_pandas_json_checkpointing.py index 74308dd2..f7333d3e 100644 --- a/tests/test_utils/test_pandas_json_checkpointing.py +++ b/tests/test_utils/test_pandas_json_checkpointing.py @@ -25,11 +25,13 @@ ) -def test_save_and_load_calibrator_state() -> None: # noqa: PLR0915 +def test_save_and_load_calibrator_state( # noqa: PLR0915 + rng: np.random.Generator, +) -> None: """Test the 'save_calibrator_state' and 'load_calibrator_state' functions.""" parameters_bounds = np.array([[0, 1], [0, 1]]).T parameters_precision = np.array([0.01, 0.01]) - real_data = np.random.randn(100, 5) + real_data = rng.standard_normal(size=(100, 5)) ensemble_size = 5 N = 30 # noqa: N806 D = 2 # noqa: N806 @@ -47,9 +49,9 @@ def test_save_and_load_calibrator_state() -> None: # noqa: PLR0915 n_sampled_params = 10 n_jobs = 1 - params_samp = np.random.rand(10, 2) - losses_samp = np.random.rand(10) - series_samp = np.random.rand(10, 100, 5) + params_samp = rng.random(size=(10, 2)) + losses_samp = rng.random(size=10) + series_samp = rng.random(size=(10, 100, 5)) batch_num_samp = np.arange(10) method_samp = np.arange(10) diff --git a/tests/test_utils/test_time_series.py b/tests/test_utils/test_time_series.py index 09f31a43..250d55e2 100644 --- a/tests/test_utils/test_time_series.py +++ b/tests/test_utils/test_time_series.py @@ -20,31 +20,138 @@ from black_it.utils.time_series import get_mom_ts -def test_get_mom_ts() -> None: +def test_get_mom_ts(rng: np.random.Generator) -> None: """Test 'get_mom_ts'.""" expected_mom_ts = np.asarray( [ - [0.50471071, 0.52523233, 0.48769797, 0.52125415, 0.45391341], - [0.29597164, 0.29840167, 0.29334348, 0.29175012, 0.30658573], - [-0.39247097, -0.53521998, -0.48164217, -0.41514771, 0.64604918], - [-1.04321643, -1.04806895, -1.05729681, -1.04166884, -1.07053431], - [-0.00826626, -0.1310761, -0.01714527, 0.05582763, -0.07196251], - [0.08511583, 0.05062704, -0.10239579, 0.05814917, -0.15244803], - [-0.1573025, -0.03237303, 0.14330391, 0.03944553, 0.03682953], - [-0.13205127, 0.02166404, -0.06635989, 0.01231925, 0.07872452], - [-0.0594593, -0.16828977, -0.08318478, 0.05319515, 0.00255055], - [0.3472227, 0.36920251, 0.33991259, 0.32411733, 0.36455635], - [0.23971816, 0.25552126, 0.24285517, 0.2348814, 0.25863802], - [0.90238066, 0.74258728, 0.80671623, 0.89465546, 0.59934486], - [-0.65135278, -0.96315459, -0.93646035, -0.78143557, -1.02558061], - [0.15463938, 0.19370484, -0.03971417, 0.04101357, -0.05957849], - [0.00551848, 0.05268421, -0.16417644, 0.11792084, -0.03113898], - [-0.11193418, 0.11907523, -0.04560956, 0.03323841, -0.11316762], - [-0.02379168, -0.00181507, -0.00476993, 0.13203911, -0.06712482], - [0.00665585, -0.06428739, -0.11528277, 0.02027778, -0.07112011], + [ + 0.5047481878242738, + 0.4577343452959496, + 0.45960556439046707, + 0.4939260177141695, + 0.49227447674825114, + ], + [ + 0.3043545480614458, + 0.2669628026226539, + 0.28423903693391567, + 0.30245153381716033, + 0.2819981640130968, + ], + [ + -0.4482110315558403, + 0.6542053297744806, + 0.5308172556250059, + -0.3849423235845297, + 0.31557366158556094, + ], + [ + -1.0657862342927136, + -1.0000694770963234, + -1.0101207985128777, + -1.0782685600527042, + -1.0462395326269132, + ], + [ + -0.09320904577520875, + 0.0686169052226043, + 0.1355394841358424, + 0.09653107323762894, + 0.05311636958390115, + ], + [ + 0.12169737118483734, + 0.0810227210130983, + 0.061958779523484865, + 0.08490375108820097, + 0.006039369703180401, + ], + [ + -0.2025899142653773, + -0.001258750058814992, + 0.12789551725231704, + -0.005223658816300541, + -0.051171362779900316, + ], + [ + 0.0895667296886336, + -0.1207015312253511, + 0.0636018350025852, + -0.028626794182293015, + -0.033236042041738946, + ], + [ + -0.03860009031959613, + -0.10752955274899662, + -0.13297311479661147, + 0.1362727715117892, + -0.20066288961998208, + ], + [ + 0.3705043372180348, + 0.2908837829651126, + 0.29957236943742727, + 0.32868245323966144, + 0.3219905265411198, + ], + [ + 0.2532444717435027, + 0.22136038120813056, + 0.22607782802456683, + 0.23746687234565775, + 0.21518362830554183, + ], + [ + 0.6878867186665477, + 0.8516810049012855, + 0.9249968652455888, + 0.8402319231441494, + 0.8433453672087052, + ], + [ + -0.9980698948119241, + -0.8313118851534257, + -0.7294913128812629, + -0.781988185137505, + -0.8992290494498107, + ], + [ + 0.15460527502011878, + 0.07324077575840597, + 0.129091118386242, + 0.12525686451619064, + 0.04116717905886786, + ], + [ + 0.132780107882644, + -0.0652821068259806, + -0.012893435354472009, + -0.038207396068454, + -0.05256884500600915, + ], + [ + 0.1309359433071446, + 0.09360205934372054, + 0.04414471989530231, + -0.0632467584785815, + 0.002402854248802498, + ], + [ + 0.016550810028943094, + 0.03214763642101067, + 0.0070285228164037265, + -0.03352305029619496, + -0.019493316543544734, + ], + [ + -0.08443798877034842, + -0.04227111216992749, + -0.053302703788843504, + 0.10373275400311846, + -0.06054093058906396, + ], ], ) - np.random.seed(42) - time_series = np.random.rand(100, 5) + time_series = rng.random(size=(100, 5)) mom_ts = get_mom_ts(time_series) assert np.allclose(mom_ts, expected_mom_ts) From a9b58ffbf2db832069bd4f62c3430bf63a6d776d Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 11:49:04 +0200 Subject: [PATCH 50/56] lint: fix Perflint errors To reproduce the errors: ``` ruff check --select "PERF" black_it tests examples scripts ``` --- .ruff.toml | 2 +- black_it/loss_functions/gsl_div.py | 2 +- black_it/plot/plot_results.py | 8 ++------ black_it/samplers/base.py | 2 +- black_it/samplers/particle_swarm.py | 2 +- black_it/utils/json_pandas_checkpointing.py | 7 +++---- examples/models/economics/brock_hommes.py | 2 +- 7 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 75d63454..0821a3c7 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY","NPY"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY","NPY","AIR","PERF"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index a9279111..e5880f42 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -111,7 +111,7 @@ def compute_loss_1d( > The information loss about the behaviour of the stochastic process due to the symbolization becomes smaller and smaller as b increases. On the other side, low values of b would likely wash away - processes’ noise the modeller might not be interested in. + processes' noise the modeller might not be interested in. Args: sim_data_ensemble: the ensemble of simulated data diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index 42b183a6..bf76444f 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -66,11 +66,7 @@ def _get_samplers_names( inv_samplers_id_table = {v: k for k, v in samplers_id_table.items()} - sampler_names = [] - for sampler_id in ids: - sampler_names.append(inv_samplers_id_table[sampler_id]) - - return sampler_names + return [inv_samplers_id_table[sampler_id] for sampler_id in ids] def plot_convergence(saving_folder: str | os.PathLike) -> None: @@ -234,7 +230,7 @@ def plot_losses_interact(saving_folder: str | os.PathLike) -> None: for sampler, sampler_id in samplers_id_table.items(): if sampler_id in method_nums: - id_samplers_cal[sampler] = sampler_id + id_samplers_cal[sampler] = sampler_id # noqa: PERF403 interact( plot_losses_method_num, diff --git a/black_it/samplers/base.py b/black_it/samplers/base.py index f3dd38a5..de1fd453 100644 --- a/black_it/samplers/base.py +++ b/black_it/samplers/base.py @@ -143,6 +143,6 @@ def find_and_get_duplicates( for repeated_group in repeated_groups: repeated_idx = np.argwhere(np.all(new_points == repeated_group, axis=1)) for index in repeated_idx: - repeated_pos.append(index[0]) + repeated_pos.append(index[0]) # noqa: PERF401 return repeated_pos diff --git a/black_it/samplers/particle_swarm.py b/black_it/samplers/particle_swarm.py index 7b019929..4e22dfb5 100644 --- a/black_it/samplers/particle_swarm.py +++ b/black_it/samplers/particle_swarm.py @@ -40,7 +40,7 @@ class ParticleSwarmSampler(BaseSampler): In a particle swarm optimizer, there is a set of particles that are "evolved" by cooperation and competition among the individuals themselves through generations. Each particle adjusts its flying according to its own - flying experience and its companions’ flying experience. Each particle, in fact, represents a potential solution + flying experience and its companions' flying experience. Each particle, in fact, represents a potential solution to a problem. Each particle is treated as a point in a D-dimensional space. The ith particle is represented as Xi = (x_{i1},...,x_{iD}). The best previous position (the position giving the best fitness value) of any particle is recorded and represented as Pi = (p_{i1},...,p_{iD}). The index of the best particle among all the particles diff --git a/black_it/utils/json_pandas_checkpointing.py b/black_it/utils/json_pandas_checkpointing.py index fd6b4e1d..a8584e7d 100644 --- a/black_it/utils/json_pandas_checkpointing.py +++ b/black_it/utils/json_pandas_checkpointing.py @@ -51,10 +51,9 @@ def load_calibrator_state(checkpoint_path: PathLike, _code_state_version: int) - cr = pd.read_csv(checkpoint_path / "calibration_results.csv") - params_samp_list = [] - - for i in range(len(cp["parameters_precision"])): - params_samp_list.append(cr[f"params_samp_{i}"]) + params_samp_list = [ + cr[f"params_samp_{i}"] for i in range(len(cp["parameters_precision"])) + ] params_samp = np.vstack(params_samp_list).T diff --git a/examples/models/economics/brock_hommes.py b/examples/models/economics/brock_hommes.py index 197b3843..7fc49155 100644 --- a/examples/models/economics/brock_hommes.py +++ b/examples/models/economics/brock_hommes.py @@ -104,7 +104,7 @@ def BH4( # noqa: N802 From Donovan Platt 2019: (experiments run) [g2, b2] and [g2, b2, g3, b3], with all parameters assumed to lie in the - interval [0, 1], with the exception of b3, which we assume lies in the interval [−1, 0]. + interval [0, 1], with the exception of b3, which we assume lies in the interval [-1, 0]. [g1,b1, g2,b2, g3,b3, g4,b4, r,β] = [0,0, 0.9,0.2, 0.9,-0.2, 1.01,0, 0.01,1] Args: From ef0a5a2247f1156a0cd7602d5e7463765c62a83e Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 12:18:11 +0200 Subject: [PATCH 51/56] lint: automatically fix RUF fixable errors To reproduce the changes: ``` ruff check --fix black_it tests examples scripts make black ruff check --fix black_it tests examples scripts ``` --- .ruff.toml | 2 +- black_it/loss_functions/gsl_div.py | 2 +- black_it/loss_functions/msm.py | 2 +- black_it/plot/plot_results.py | 2 +- black_it/schedulers/rl/agents/epsilon_greedy.py | 2 +- examples/models/economics/boltzmann_wealth.py | 2 +- examples/models/sir/simlib.py | 12 ++++++++---- tests/test_examples/base.py | 2 +- tests/test_examples/test_sir_python.py | 2 +- tests/test_plot/base.py | 2 +- 10 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 0821a3c7..a2adefc2 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY","NPY","AIR","PERF"] +select = ["F","W","E","C90","I","N","D","UP","YTT","ANN","ASYNC","BLE","B","A","COM","CPY","C4","DTZ","T10","DJ","EM","EXE","FA","ISC","ICN","G","INP","PIE","PYI","PT","Q","RSE","RET","SLF","SLOT","SIM","TID","TCH","INT","ARG","PTH","TD","FIX","ERA","PD","PGH","PL","TRY","FLY","NPY","AIR","PERF","FURB","LOG","RUF"] ignore = ["ANN101", "ANN102", "E203", "S", "FBT", "T20"] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/black_it/loss_functions/gsl_div.py b/black_it/loss_functions/gsl_div.py index e5880f42..052efcf4 100644 --- a/black_it/loss_functions/gsl_div.py +++ b/black_it/loss_functions/gsl_div.py @@ -165,7 +165,7 @@ def compute_loss_1d( return gsl_loss / ensemble_size - def gsl_div_1d_1_sample( # noqa: PLR0913 + def gsl_div_1d_1_sample( self, sim_xd: NDArray, obs_xd: NDArray, diff --git a/black_it/loss_functions/msm.py b/black_it/loss_functions/msm.py index 54bf5a49..a94b8012 100644 --- a/black_it/loss_functions/msm.py +++ b/black_it/loss_functions/msm.py @@ -54,7 +54,7 @@ def __str__(self) -> str: class MethodOfMomentsLoss(BaseLoss): """Class for the 'method of moments' loss.""" - def __init__( # noqa: PLR0913 + def __init__( self, covariance_mat: str | NDArray[np.float64] = "identity", coordinate_weights: NDArray[np.float64] | None = None, diff --git a/black_it/plot/plot_results.py b/black_it/plot/plot_results.py index bf76444f..ab663c42 100644 --- a/black_it/plot/plot_results.py +++ b/black_it/plot/plot_results.py @@ -103,7 +103,7 @@ def plot_convergence(saving_folder: str | os.PathLike) -> None: sampler_names = _get_samplers_names(saving_folder, ids) handles, labels = g.get_legend_handles_labels() - labels = sampler_names + [labels[-1]] + labels = [*sampler_names, labels[-1]] plt.legend(handles, labels, loc="upper right") diff --git a/black_it/schedulers/rl/agents/epsilon_greedy.py b/black_it/schedulers/rl/agents/epsilon_greedy.py index 79121751..73d93219 100644 --- a/black_it/schedulers/rl/agents/epsilon_greedy.py +++ b/black_it/schedulers/rl/agents/epsilon_greedy.py @@ -27,7 +27,7 @@ class MABEpsilonGreedy(Agent[int, int]): """Implementation of a MAB eps-greedy algorithm.""" - def __init__( # noqa: PLR0913 + def __init__( self, n_actions: int, alpha: float, diff --git a/examples/models/economics/boltzmann_wealth.py b/examples/models/economics/boltzmann_wealth.py index 601e61cf..ef87936f 100644 --- a/examples/models/economics/boltzmann_wealth.py +++ b/examples/models/economics/boltzmann_wealth.py @@ -128,5 +128,5 @@ def compute_gini(model: BoltzmannWealthModel) -> float: agent_wealths = [cast(MoneyAgent, agent).wealth for agent in model.schedule.agents] x = sorted(agent_wealths) n = model.num_agents - b = sum(xi * (n - i) for i, xi in enumerate(x)) / (n * sum(x)) # noqa: N806 + b = sum(xi * (n - i) for i, xi in enumerate(x)) / (n * sum(x)) return 1 + (1 / n) - 2 * b diff --git a/examples/models/sir/simlib.py b/examples/models/sir/simlib.py index c2a1b457..016ec815 100644 --- a/examples/models/sir/simlib.py +++ b/examples/models/sir/simlib.py @@ -91,14 +91,18 @@ def _build_simulator_cmdline( Returns: the arguments for the Docker CLI """ - return ["docker", "run", "--rm", docker_image_name] + list( - chain.from_iterable( - ( + return [ + "docker", + "run", + "--rm", + docker_image_name, + *list( + chain.from_iterable( (f"--{argname}", str(argvalue)) for argname, argvalue in sim_params.items() ), ), - ) + ] def execute_simulator( diff --git a/tests/test_examples/base.py b/tests/test_examples/base.py index e7c3a3f2..b7059835 100644 --- a/tests/test_examples/base.py +++ b/tests/test_examples/base.py @@ -47,7 +47,7 @@ def test_run(self) -> None: assert ( process_output.returncode == 0 - ), f"{str(self.script_path)} exited with error code {process_output.returncode}" + ), f"{self.script_path!s} exited with error code {process_output.returncode}" assert all( line in process_output.stdout for line in self.expected_lines diff --git a/tests/test_examples/test_sir_python.py b/tests/test_examples/test_sir_python.py index 3f489d64..0bbea2cf 100644 --- a/tests/test_examples/test_sir_python.py +++ b/tests/test_examples/test_sir_python.py @@ -23,7 +23,7 @@ from examples.models.sir.sir_python import SIR, SIR_w_breaks except ModuleNotFoundError as e: pytest.skip( - f"skipping tests for SIR python models, reason: {str(e)}", + f"skipping tests for SIR python models, reason: {e!s}", allow_module_level=True, ) diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index e65a464b..b776db78 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -68,4 +68,4 @@ class BasePlotResultsTest(BasePlotTest): @classmethod def setup(cls) -> None: """Set up the test.""" - cls.args = [cls.saving_folder] + cls.args + cls.args = [cls.saving_folder, *cls.args] From dc7c87ffdb4aba04472cf6316dcad205d3a221d5 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 12:40:18 +0200 Subject: [PATCH 52/56] fix: update coordinate_filters argument type annotation in BaseLoss.__init__ A good practice is to use the coarsest type as argument function, so to make it usable in more ways. In particular, in this commit, the argument coordinate_filters in the base constructor of the class BaseLoss has been changed from 'list' to 'Sequence'. This allows to use coordinate_filters as tuples. It is also a more correct type hint, since coordinate_filters should not change during the lifetime of BaseLoss. --- black_it/loss_functions/base.py | 10 +++++----- tests/test_losses/test_base.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/black_it/loss_functions/base.py b/black_it/loss_functions/base.py index d8e2a479..1272131e 100644 --- a/black_it/loss_functions/base.py +++ b/black_it/loss_functions/base.py @@ -18,7 +18,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Callable, Optional +from typing import Callable, Optional, Sequence import numpy as np from numpy.typing import NDArray @@ -35,7 +35,7 @@ class BaseLoss(ABC): def __init__( self, coordinate_weights: NDArray | None = None, - coordinate_filters: list[Callable | None] | None = None, + coordinate_filters: Sequence[Callable | None] | None = None, ) -> None: """Initialize the loss function. @@ -76,7 +76,7 @@ def compute_loss( @staticmethod def _filter_data( - filters: list[Callable | None], + filters: Sequence[Callable | None], sim_data_ensemble: NDArray[np.float64], ) -> NDArray[np.float64]: """Filter the simulated time series.""" @@ -116,9 +116,9 @@ def _check_coordinate_weights(self, num_coords: int) -> NDArray[np.float64]: return weights - def _check_coordinate_filters(self, num_coords: int) -> list[Callable | None]: + def _check_coordinate_filters(self, num_coords: int) -> Sequence[Callable | None]: """Check self.coordinate_filters and return usable filters.""" - filters: list[Callable | None] + filters: Sequence[Callable | None] if self.coordinate_filters is None: # a list of identity functions diff --git a/tests/test_losses/test_base.py b/tests/test_losses/test_base.py index 1e5c9978..438b4761 100644 --- a/tests/test_losses/test_base.py +++ b/tests/test_losses/test_base.py @@ -38,7 +38,7 @@ def __init__( self, loss_constant: float, coordinate_weights: NDArray | None = None, - coordinate_filters: list[Callable | None] | None = None, + coordinate_filters: tuple[Callable | None, ...] | None = None, ) -> None: """Initialize the custom loss.""" super().__init__(coordinate_weights, coordinate_filters) @@ -60,7 +60,7 @@ def compute_loss_1d( nb_real_rows: int = 10 coordinate_weights: NDArray[np.float64] | None - coordinate_filters: list[Callable | None] | None + coordinate_filters: tuple[Callable | None, ...] | None # instance attributes loss: MyCustomLoss @@ -129,7 +129,7 @@ class TestComputeLossWhenCoordFiltersIsNotNone(TestComputeLoss): """Test BaseLoss.compute_loss when coordinate filters is not None.""" coordinate_weights = None - coordinate_filters = [lambda x: x * 0, lambda x: x * 0] + coordinate_filters = (lambda x: x * 0, lambda x: x * 0) def test_run(self) -> None: """Test BaseLoss.compute_loss when coordinate filters is not None.""" @@ -141,7 +141,7 @@ class TestComputeLossWhenNbCoordFiltersIsWrong(TestComputeLoss): """Test BaseLoss.compute_loss when number of coordinate weights does not match data.""" coordinate_weights = None - coordinate_filters = [lambda x: x] + coordinate_filters: tuple[Callable] = (lambda x: x,) def test_run(self) -> None: """Test BaseLoss.compute_loss when number of coordinate filters does not match data.""" From f61be8b525c262152c0fc3bc87f72745f772d844 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 12:45:05 +0200 Subject: [PATCH 53/56] lint: fix remaining Ruff-specific-rules errors To reproduce errors: ``` ruff check --show-fixes --fix black_it tests examples scripts ``` --- tests/test_calibrator.py | 70 ++++++++++--------- tests/test_examples/test_main.py | 5 +- tests/test_plot/base.py | 4 +- .../test_plot_descriptive_statistics.py | 6 +- 4 files changed, 45 insertions(+), 40 deletions(-) diff --git a/tests/test_calibrator.py b/tests/test_calibrator.py index e0046fdb..e812bb53 100644 --- a/tests/test_calibrator.py +++ b/tests/test_calibrator.py @@ -43,7 +43,7 @@ class TestCalibrate: """Test the Calibrator.calibrate method.""" - expected_params = np.array( + expected_params: NDArray = np.array( [ [0.52, 0.01], [0.56, 0.37], @@ -62,22 +62,24 @@ class TestCalibrate: ], ) - expected_losses = [ - 0.87950070, - 0.99224516, - 1.15590624, - 1.24380484, - 1.76330622, - 1.88165325, - 2.30766018, - 2.55676207, - 2.86482802, - 2.88057794, - 2.90585611, - 3.77705872, - 4.47466328, - 5.79615295, - ] + expected_losses: NDArray = np.array( + [ + 0.87950070, + 0.99224516, + 1.15590624, + 1.24380484, + 1.76330622, + 1.88165325, + 2.30766018, + 2.55676207, + 2.86482802, + 2.88057794, + 2.90585611, + 3.77705872, + 4.47466328, + 5.79615295, + ], + ) win32_expected_params: NDArray = np.array( [ @@ -98,22 +100,24 @@ class TestCalibrate: ], ) - win32_expected_losses = [ - 0.43742215, - 0.99224516, - 1.15590624, - 1.25892187, - 1.76330622, - 1.88165325, - 2.30766018, - 2.55676207, - 2.86482802, - 2.88057794, - 2.90585611, - 3.77705872, - 4.47466328, - 5.67663570, - ] + win32_expected_losses: NDArray = np.array( + [ + 0.43742215, + 0.99224516, + 1.15590624, + 1.25892187, + 1.76330622, + 1.88165325, + 2.30766018, + 2.55676207, + 2.86482802, + 2.88057794, + 2.90585611, + 3.77705872, + 4.47466328, + 5.67663570, + ], + ) def setup(self) -> None: """Set up the tests.""" diff --git a/tests/test_examples/test_main.py b/tests/test_examples/test_main.py index 639287e5..3450032f 100644 --- a/tests/test_examples/test_main.py +++ b/tests/test_examples/test_main.py @@ -15,6 +15,7 @@ # along with this program. If not, see . """Test the main example.""" +from __future__ import annotations from tests.conftest import DEFAULT_SUBPROCESS_TIMEOUT, ROOT_DIR from tests.test_examples.base import BaseMainExampleTestClass @@ -32,7 +33,7 @@ class TestMainExample(BaseMainExampleTestClass): script_path = EXAMPLE_MAIN_SCRIPT_PATH timeout = DEFAULT_SUBPROCESS_TIMEOUT nb_batches = 5 - expected_lines = [ + expected_lines: tuple[str, ...] = ( "PARAMS SAMPLED: 0", "METHOD: HaltonSampler", "METHOD: RandomForestSampler", @@ -40,4 +41,4 @@ class TestMainExample(BaseMainExampleTestClass): *[f"BATCH NUMBER: {i}" for i in range(1, nb_batches + 1)], TRUE_PARAMETERS_STR, BEST_PARAMETERS_STR, - ] + ) diff --git a/tests/test_plot/base.py b/tests/test_plot/base.py index b776db78..3a62c91b 100644 --- a/tests/test_plot/base.py +++ b/tests/test_plot/base.py @@ -20,7 +20,7 @@ import logging from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Callable +from typing import Any, Callable, Sequence import matplotlib.pyplot as plt from matplotlib.testing.compare import compare_images @@ -32,7 +32,7 @@ class BasePlotTest: plotting_function: Callable expected_image: Path tolerance: float | None = 0.000 - args: list[Any] = [] + args: Sequence[Any] = () def test_run(self) -> None: """Run the test.""" diff --git a/tests/test_plot/test_plot_descriptive_statistics.py b/tests/test_plot/test_plot_descriptive_statistics.py index ff3c1839..6c9b4bc0 100644 --- a/tests/test_plot/test_plot_descriptive_statistics.py +++ b/tests/test_plot/test_plot_descriptive_statistics.py @@ -17,7 +17,7 @@ """This test module contains tests for the plot_descriptive_statistics.py module.""" from __future__ import annotations -from typing import Any +from typing import Any, Sequence import numpy as np @@ -32,11 +32,11 @@ class TestTsStats(BasePlotTest): """Test 'ts_stats' plotting function.""" plotting_function = ts_stats - args: list[Any] = [] + args: Sequence[Any] = () expected_image = PLOT_DIR / "ts_stats-expected.png" def setup(self) -> None: """Set up the test.""" rng = np.random.default_rng(42) data = rng.random(100) - self.args = [data] + self.args = (data,) From 72f85cbff38929bfc09123bebb45dcf35d367f9b Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 12:46:55 +0200 Subject: [PATCH 54/56] ci: add ruff-check step in lint workflow --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 79f13af4..f0056542 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,7 @@ jobs: run: pip install tox - name: Code style check run: | + tox -e ruff-check tox -e black-check tox -e vulture tox -e darglint From 0f2b6b0ae3326be44f04c569aae8effac7b95a24 Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 1 Sep 2023 12:47:42 +0200 Subject: [PATCH 55/56] chore: add --show-fixes in tox.ini ruff-check testenv command --- Makefile | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 962ce277..bfc36cae 100644 --- a/Makefile +++ b/Makefile @@ -97,12 +97,12 @@ black-check-files: ## check black formatting for specific files (specified with PHONY.: ruff ruff: ## run ruff linter - ruff check --fix . + ruff check --fix --show-fixes . PHONY.: ruff-files ruff-files: ## run ruff linter for specific files (specified with files="file1 file2 somedir ...") $(call check_defined, files) - ruff check --fix $(files) + ruff check --fix --show-fixes $(files) PHONY.: ruff-check ruff-check: ## check ruff linter rules diff --git a/tox.ini b/tox.ini index 110efe60..964bed92 100644 --- a/tox.ini +++ b/tox.ini @@ -61,7 +61,7 @@ commands = black black_it tests scripts examples --check --verbose [testenv:ruff] skip_install = True deps = ruff==0.0.290 -commands = ruff check --fix . +commands = ruff check --fix --show-fixes . [testenv:ruff-check] skip_install = True From 628399d70367284164c4c1db90c64795fbafee1a Mon Sep 17 00:00:00 2001 From: Marco Favorito Date: Fri, 22 Sep 2023 09:40:40 +0200 Subject: [PATCH 56/56] ci: run all tox testenv in a single command --- .github/workflows/lint.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f0056542..71514836 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,15 +27,11 @@ jobs: run: pip install tox - name: Code style check run: | - tox -e ruff-check - tox -e black-check - tox -e vulture - tox -e darglint + tox -e black-check,ruff-check,vulture,darglint - name: Static type check run: tox -e mypy - name: Check copyright run: tox -e check-copyright - name: Misc checks run: | - tox -e bandit - tox -e safety \ No newline at end of file + tox -e bandit,safety \ No newline at end of file