diff --git a/README.md b/README.md index 260536b..22798c9 100644 --- a/README.md +++ b/README.md @@ -257,9 +257,6 @@ You can remove this file or selected lines from inside this file. This will trig ## TODO -- Use pydantic instead of schema - Add cli options to force re-download versions and document link lists -- Add download size to stats -- Add `--validate` option to `download` command that will trigger validation after download - Add some example terraform and/or ansible to use for deploy to VM in cloud - Use proper logging diff --git a/poetry.lock b/poetry.lock index 151bb60..87eecc6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + [[package]] name = "attrs" version = "23.2.0" @@ -218,17 +229,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "contextlib2" -version = "21.6.0" -description = "Backports and enhancements for the contextlib module" -optional = false -python-versions = ">=3.6" -files = [ - {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, - {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, -] - [[package]] name = "cryptography" version = "42.0.4" @@ -674,6 +674,116 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +[[package]] +name = "pydantic" +version = "2.7.0" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.7.0-py3-none-any.whl", hash = "sha256:9dee74a271705f14f9a1567671d144a851c675b072736f0a7b2608fd9e495352"}, + {file = "pydantic-2.7.0.tar.gz", hash = "sha256:b5ecdd42262ca2462e2624793551e80911a1e989f462910bb81aef974b4bb383"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.18.1" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.18.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ee9cf33e7fe14243f5ca6977658eb7d1042caaa66847daacbd2117adb258b226"}, + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b7bbb97d82659ac8b37450c60ff2e9f97e4eb0f8a8a3645a5568b9334b08b50"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df4249b579e75094f7e9bb4bd28231acf55e308bf686b952f43100a5a0be394c"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0491006a6ad20507aec2be72e7831a42efc93193d2402018007ff827dc62926"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ae80f72bb7a3e397ab37b53a2b49c62cc5496412e71bc4f1277620a7ce3f52b"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58aca931bef83217fca7a390e0486ae327c4af9c3e941adb75f8772f8eeb03a1"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1be91ad664fc9245404a789d60cba1e91c26b1454ba136d2a1bf0c2ac0c0505a"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:667880321e916a8920ef49f5d50e7983792cf59f3b6079f3c9dac2b88a311d17"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f7054fdc556f5421f01e39cbb767d5ec5c1139ea98c3e5b350e02e62201740c7"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:030e4f9516f9947f38179249778709a460a3adb516bf39b5eb9066fcfe43d0e6"}, + {file = "pydantic_core-2.18.1-cp310-none-win32.whl", hash = "sha256:2e91711e36e229978d92642bfc3546333a9127ecebb3f2761372e096395fc649"}, + {file = "pydantic_core-2.18.1-cp310-none-win_amd64.whl", hash = "sha256:9a29726f91c6cb390b3c2338f0df5cd3e216ad7a938762d11c994bb37552edb0"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9ece8a49696669d483d206b4474c367852c44815fca23ac4e48b72b339807f80"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a5d83efc109ceddb99abd2c1316298ced2adb4570410defe766851a804fcd5b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7973c381283783cd1043a8c8f61ea5ce7a3a58b0369f0ee0ee975eaf2f2a1b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54c7375c62190a7845091f521add19b0f026bcf6ae674bdb89f296972272e86d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd63cec4e26e790b70544ae5cc48d11b515b09e05fdd5eff12e3195f54b8a586"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:561cf62c8a3498406495cfc49eee086ed2bb186d08bcc65812b75fda42c38294"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68717c38a68e37af87c4da20e08f3e27d7e4212e99e96c3d875fbf3f4812abfc"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d5728e93d28a3c63ee513d9ffbac9c5989de8c76e049dbcb5bfe4b923a9739d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0f17814c505f07806e22b28856c59ac80cee7dd0fbb152aed273e116378f519"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d816f44a51ba5175394bc6c7879ca0bd2be560b2c9e9f3411ef3a4cbe644c2e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win32.whl", hash = "sha256:09f03dfc0ef8c22622eaa8608caa4a1e189cfb83ce847045eca34f690895eccb"}, + {file = "pydantic_core-2.18.1-cp311-none-win_amd64.whl", hash = "sha256:27f1009dc292f3b7ca77feb3571c537276b9aad5dd4efb471ac88a8bd09024e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win_arm64.whl", hash = "sha256:48dd883db92e92519201f2b01cafa881e5f7125666141a49ffba8b9facc072b0"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b6b0e4912030c6f28bcb72b9ebe4989d6dc2eebcd2a9cdc35fefc38052dd4fe8"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3202a429fe825b699c57892d4371c74cc3456d8d71b7f35d6028c96dfecad31"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3982b0a32d0a88b3907e4b0dc36809fda477f0757c59a505d4e9b455f384b8b"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25595ac311f20e5324d1941909b0d12933f1fd2171075fcff763e90f43e92a0d"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14fe73881cf8e4cbdaded8ca0aa671635b597e42447fec7060d0868b52d074e6"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca976884ce34070799e4dfc6fbd68cb1d181db1eefe4a3a94798ddfb34b8867f"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684d840d2c9ec5de9cb397fcb3f36d5ebb6fa0d94734f9886032dd796c1ead06"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:54764c083bbe0264f0f746cefcded6cb08fbbaaf1ad1d78fb8a4c30cff999a90"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:201713f2f462e5c015b343e86e68bd8a530a4f76609b33d8f0ec65d2b921712a"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd1a9edb9dd9d79fbeac1ea1f9a8dd527a6113b18d2e9bcc0d541d308dae639b"}, + {file = "pydantic_core-2.18.1-cp312-none-win32.whl", hash = "sha256:d5e6b7155b8197b329dc787356cfd2684c9d6a6b1a197f6bbf45f5555a98d411"}, + {file = "pydantic_core-2.18.1-cp312-none-win_amd64.whl", hash = "sha256:9376d83d686ec62e8b19c0ac3bf8d28d8a5981d0df290196fb6ef24d8a26f0d6"}, + {file = "pydantic_core-2.18.1-cp312-none-win_arm64.whl", hash = "sha256:c562b49c96906b4029b5685075fe1ebd3b5cc2601dfa0b9e16c2c09d6cbce048"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3e352f0191d99fe617371096845070dee295444979efb8f27ad941227de6ad09"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0295d52b012cbe0d3059b1dba99159c3be55e632aae1999ab74ae2bd86a33d7"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56823a92075780582d1ffd4489a2e61d56fd3ebb4b40b713d63f96dd92d28144"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd3f79e17b56741b5177bcc36307750d50ea0698df6aa82f69c7db32d968c1c2"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38a5024de321d672a132b1834a66eeb7931959c59964b777e8f32dbe9523f6b1"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ce426ee691319d4767748c8e0895cfc56593d725594e415f274059bcf3cb76"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2adaeea59849ec0939af5c5d476935f2bab4b7f0335b0110f0f069a41024278e"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9b6431559676a1079eac0f52d6d0721fb8e3c5ba43c37bc537c8c83724031feb"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:85233abb44bc18d16e72dc05bf13848a36f363f83757541f1a97db2f8d58cfd9"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:641a018af4fe48be57a2b3d7a1f0f5dbca07c1d00951d3d7463f0ac9dac66622"}, + {file = "pydantic_core-2.18.1-cp38-none-win32.whl", hash = "sha256:63d7523cd95d2fde0d28dc42968ac731b5bb1e516cc56b93a50ab293f4daeaad"}, + {file = "pydantic_core-2.18.1-cp38-none-win_amd64.whl", hash = "sha256:907a4d7720abfcb1c81619863efd47c8a85d26a257a2dbebdb87c3b847df0278"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:aad17e462f42ddbef5984d70c40bfc4146c322a2da79715932cd8976317054de"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:94b9769ba435b598b547c762184bcfc4783d0d4c7771b04a3b45775c3589ca44"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e0e57cc704a52fb1b48f16d5b2c8818da087dbee6f98d9bf19546930dc64b5"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76b86e24039c35280ceee6dce7e62945eb93a5175d43689ba98360ab31eebc4a"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12a05db5013ec0ca4a32cc6433f53faa2a014ec364031408540ba858c2172bb0"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:250ae39445cb5475e483a36b1061af1bc233de3e9ad0f4f76a71b66231b07f88"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a32204489259786a923e02990249c65b0f17235073149d0033efcebe80095570"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6395a4435fa26519fd96fdccb77e9d00ddae9dd6c742309bd0b5610609ad7fb2"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2533ad2883f001efa72f3d0e733fb846710c3af6dcdd544fe5bf14fa5fe2d7db"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b560b72ed4816aee52783c66854d96157fd8175631f01ef58e894cc57c84f0f6"}, + {file = "pydantic_core-2.18.1-cp39-none-win32.whl", hash = "sha256:582cf2cead97c9e382a7f4d3b744cf0ef1a6e815e44d3aa81af3ad98762f5a9b"}, + {file = "pydantic_core-2.18.1-cp39-none-win_amd64.whl", hash = "sha256:ca71d501629d1fa50ea7fa3b08ba884fe10cefc559f5c6c8dfe9036c16e8ae89"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e178e5b66a06ec5bf51668ec0d4ac8cfb2bdcb553b2c207d58148340efd00143"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:72722ce529a76a4637a60be18bd789d8fb871e84472490ed7ddff62d5fed620d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fe0c1ce5b129455e43f941f7a46f61f3d3861e571f2905d55cdbb8b5c6f5e2c"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4284c621f06a72ce2cb55f74ea3150113d926a6eb78ab38340c08f770eb9b4d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0c3e718f4e064efde68092d9d974e39572c14e56726ecfaeebbe6544521f47"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2027493cc44c23b598cfaf200936110433d9caa84e2c6cf487a83999638a96ac"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76909849d1a6bffa5a07742294f3fa1d357dc917cb1fe7b470afbc3a7579d539"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ee7ccc7fb7e921d767f853b47814c3048c7de536663e82fbc37f5eb0d532224b"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ee2794111c188548a4547eccc73a6a8527fe2af6cf25e1a4ebda2fd01cdd2e60"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a139fe9f298dc097349fb4f28c8b81cc7a202dbfba66af0e14be5cfca4ef7ce5"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d074b07a10c391fc5bbdcb37b2f16f20fcd9e51e10d01652ab298c0d07908ee2"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c69567ddbac186e8c0aadc1f324a60a564cfe25e43ef2ce81bcc4b8c3abffbae"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf1c7b78cddb5af00971ad5294a4583188bda1495b13760d9f03c9483bb6203"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2684a94fdfd1b146ff10689c6e4e815f6a01141781c493b97342cdc5b06f4d5d"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:73c1bc8a86a5c9e8721a088df234265317692d0b5cd9e86e975ce3bc3db62a59"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e60defc3c15defb70bb38dd605ff7e0fae5f6c9c7cbfe0ad7868582cb7e844a6"}, + {file = "pydantic_core-2.18.1.tar.gz", hash = "sha256:de9d3e8717560eb05e28739d1b35e4eac2e458553a52a301e51352a7ffc86a35"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pyjwt" version = "2.8.0" @@ -778,7 +888,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -842,7 +951,6 @@ optional = false python-versions = "*" files = [ {file = "requests-file-2.0.0.tar.gz", hash = "sha256:20c5931629c558fda566cacc10cfe2cd502433e628f568c34c80d96a0cc95972"}, - {file = "requests_file-2.0.0-py2.py3-none-any.whl", hash = "sha256:3e493d390adb44aa102ebea827a48717336d5268968c370eaf19abaf5cae13bf"}, ] [package.dependencies] @@ -888,20 +996,6 @@ files = [ {file = "ruff-0.1.15.tar.gz", hash = "sha256:f6dfa8c1b21c913c326919056c390966648b680966febcb796cc9d1aaab8564e"}, ] -[[package]] -name = "schema" -version = "0.7.5" -description = "Simple data validation library" -optional = false -python-versions = "*" -files = [ - {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, - {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, -] - -[package.dependencies] -contextlib2 = ">=0.5.5" - [[package]] name = "simple-salesforce" version = "1.12.5" @@ -1037,4 +1131,4 @@ xmlsec = ["xmlsec (>=0.6.1)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "8de875892b9a510c861ffddb6796262f6ee74e503028c798256b3acf9410e31e" +content-hash = "7e17f8684e431cdd630fac74598edc057905157d1ae56d93da01967af0d4732c" diff --git a/pyproject.toml b/pyproject.toml index c1b4519..ffe08ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ python = "^3.11" simple-salesforce = "^1.12.5" PyYAML = "^6.0.1" click = "^8.1.7" -schema = "^0.7.5" python-dateutil = "^2.8.2" types-PyYAML = "^6.0.12.12" humanize = "^4.9.0" +pydantic = "^2.7.0" [tool.poetry.group.dev.dependencies] pytest-mock = "^3.12.0" diff --git a/src/salesforce_archivist/archivist.py b/src/salesforce_archivist/archivist.py index 561f054..77a6641 100644 --- a/src/salesforce_archivist/archivist.py +++ b/src/salesforce_archivist/archivist.py @@ -1,11 +1,12 @@ import datetime -import os.path -from typing import Any +import os +from typing import Any, Dict import click import humanize -import yaml -from schema import And, Optional, Or, Schema, Use +from pydantic import BaseModel, Field, field_validator, ValidationInfo, computed_field +from typing import Optional +from typing_extensions import Annotated from simple_salesforce import Salesforce as SalesforceClient from salesforce_archivist.salesforce.api import SalesforceApiClient @@ -14,40 +15,12 @@ from salesforce_archivist.salesforce.validation import ValidatedContentVersionList -class ArchivistObject: - def __init__( - self, - data_dir: str, - obj_type: str, - modified_date_lt: datetime.datetime | None = None, - modified_date_gt: datetime.datetime | None = None, - dir_name_field: str | None = None, - ): - self._data_dir: str = os.path.join(data_dir, obj_type) - self._obj_type: str = obj_type - self._modified_date_lt: datetime.datetime | None = modified_date_lt - self._modified_date_gt: datetime.datetime | None = modified_date_gt - self._dir_name_field: str | None = dir_name_field - - @property - def data_dir(self) -> str: - return self._data_dir - - @property - def obj_type(self) -> str: - return self._obj_type - - @property - def modified_date_lt(self) -> datetime.datetime | None: - return self._modified_date_lt - - @property - def modified_date_gt(self) -> datetime.datetime | None: - return self._modified_date_gt - - @property - def dir_name_field(self) -> str | None: - return self._dir_name_field +class ArchivistObject(BaseModel): + data_dir: Annotated[str, Field(min_length=1)] + obj_type: Annotated[str, Field(min_length=1)] + modified_date_lt: Optional[datetime.datetime] = None + modified_date_gt: Optional[datetime.datetime] = None + dir_name_field: Optional[str] = None def __eq__(self, other: Any) -> bool: if not isinstance(other, type(self)): @@ -60,102 +33,49 @@ def __eq__(self, other: Any) -> bool: other.modified_date_lt, ) - -class ArchivistAuth: - def __init__(self, instance_url: str, username: str, consumer_key: str, private_key: str): - self._instance_url = instance_url - self._username = username - self._consumer_key = consumer_key - self._private_key = private_key - - @property - def instance_url(self) -> str: - return self._instance_url - + # https://github.com/python/mypy/issues/14461 + @computed_field # type: ignore[misc] @property - def username(self) -> str: - return self._username - - @property - def consumer_key(self) -> str: - return self._consumer_key - - @property - def private_key(self) -> str: - return self._private_key - - -class ArchivistConfig: - _schema = Schema( - { - "data_dir": And(str, len, os.path.isdir, error="data_dir must be set and be a directory"), - "max_api_usage_percent": Or(int, float, Use(float), lambda v: 0.0 < v <= 100.0), - Optional("max_workers"): Optional(int, lambda v: 0 < v), - Optional("modified_date_gt"): lambda d: isinstance(d, datetime.datetime), - Optional("modified_date_lt"): lambda d: isinstance(d, datetime.datetime), - "auth": { - "instance_url": And(str, len), - "username": And(str, len), - "consumer_key": And(str, len), - "private_key": And(bytes, len, Use(lambda b: b.decode("UTF-8"))), - }, - "objects": { - str: { - Optional("modified_date_gt"): lambda d: isinstance(d, datetime.datetime), - Optional("modified_date_lt"): lambda d: isinstance(d, datetime.datetime), - Optional("dir_name_field"): And(str, len), + def obj_dir(self) -> str: + return os.path.join(self.data_dir, self.obj_type) + + +class ArchivistAuth(BaseModel): + instance_url: Annotated[str, Field(min_length=1)] + username: Annotated[str, Field(min_length=1)] + consumer_key: Annotated[str, Field(min_length=1)] + private_key: Annotated[str, Field(min_length=1)] + + +class ArchivistConfig(BaseModel): + auth: ArchivistAuth + data_dir: Annotated[str, Field(min_length=1)] + max_api_usage_percent: Optional[Annotated[float, Field(gt=0.0, le=100.0)]] = None + max_workers: Optional[Annotated[int, Field(gt=0)]] = None + modified_date_gt: Optional[datetime.datetime] = None + modified_date_lt: Optional[datetime.datetime] = None + objects: Dict[str, ArchivistObject] + + @field_validator("objects", mode="before") + @classmethod + def serialize_categories(cls, objects: dict, info: ValidationInfo) -> dict: + for obj_type, obj_dict in objects.items(): + obj_dict.update( + { + "obj_type": obj_type, + "data_dir": info.data["data_dir"], + "modified_date_gt": obj_dict.get("modified_date_gt", info.data["modified_date_gt"]), + "modified_date_lt": obj_dict.get("modified_date_lt", info.data["modified_date_lt"]), } - }, - } - ) - - def __init__(self, path: str): - with open(path) as file: - config = self._schema.validate(yaml.load(file, Loader=yaml.FullLoader)) - self._auth: ArchivistAuth = ArchivistAuth(**config["auth"]) - self._data_dir: str = config["data_dir"] - self._max_api_usage_percent: float = config["max_api_usage_percent"] - self.modified_date_gt: datetime.datetime | None = config.get("modified_date_gt") - self.modified_date_lt: datetime.datetime | None = config.get("modified_date_lt") - self._max_workers: int = config.get("max_workers") - self._objects = [] - for obj_type, obj_config in config["objects"].items(): - self._objects.append( - ArchivistObject( - data_dir=self._data_dir, - obj_type=obj_type, - modified_date_lt=obj_config.get("modified_date_lt", self.modified_date_lt), - modified_date_gt=obj_config.get("modified_date_gt", self.modified_date_gt), - dir_name_field=obj_config.get("dir_name_field"), - ) ) - - @property - def data_dir(self) -> str: - return self._data_dir - - @property - def max_workers(self) -> int: - return self._max_workers - - @property - def max_api_usage_percent(self) -> float: - return self._max_api_usage_percent - - @property - def auth(self) -> ArchivistAuth: - return self._auth - - @property - def objects(self) -> list[ArchivistObject]: - return self._objects + return objects class Archivist: def __init__( self, data_dir: str, - objects: list[ArchivistObject], + objects: dict[str, ArchivistObject], sf_client: SalesforceClient, max_api_usage_percent: float | None = None, max_workers: int | None = None, @@ -177,7 +97,7 @@ def download(self) -> bool: "errors": 0, "size": 0, } - for archivist_obj in self._objects: + for archivist_obj in self._objects.values(): obj_type = archivist_obj.obj_type salesforce = Salesforce( archivist_obj=archivist_obj, @@ -194,7 +114,7 @@ def download(self) -> bool: download_list = DownloadContentVersionList( document_link_list=document_link_list, content_version_list=content_version_list, - data_dir=archivist_obj.data_dir, + data_dir=archivist_obj.obj_dir, ) stats = salesforce.download_files( download_content_version_list=download_list, @@ -225,7 +145,7 @@ def validate(self) -> bool: "processed": 0, "invalid": 0, } - for archivist_obj in self._objects: + for archivist_obj in self._objects.values(): salesforce = Salesforce( archivist_obj=archivist_obj, client=SalesforceApiClient(self._sf_client), @@ -238,7 +158,7 @@ def validate(self) -> bool: download_list = DownloadContentVersionList( document_link_list=document_link_list, content_version_list=content_version_list, - data_dir=archivist_obj.data_dir, + data_dir=archivist_obj.obj_dir, ) stats = salesforce.validate_download( download_content_version_list=download_list, diff --git a/src/salesforce_archivist/cli.py b/src/salesforce_archivist/cli.py index d15064b..6a41601 100644 --- a/src/salesforce_archivist/cli.py +++ b/src/salesforce_archivist/cli.py @@ -2,6 +2,7 @@ from types import FrameType import click +import yaml from click import Context from salesforce_archivist.archivist import Archivist, ArchivistConfig @@ -20,7 +21,9 @@ def signal_handler(signum: int, frame: FrameType | None) -> None: @click.pass_context def cli(ctx: Context) -> None: ctx.ensure_object(dict) - ctx.obj["config"] = ArchivistConfig("config.yaml") + with open("config.yaml") as file: + config = yaml.load(file, Loader=yaml.FullLoader) + ctx.obj["config"] = ArchivistConfig(**config) @cli.command() diff --git a/src/salesforce_archivist/salesforce/salesforce.py b/src/salesforce_archivist/salesforce/salesforce.py index e5d9615..2d9cd8e 100644 --- a/src/salesforce_archivist/salesforce/salesforce.py +++ b/src/salesforce_archivist/salesforce/salesforce.py @@ -39,7 +39,7 @@ def __init__( self._dir_name_field = dir_name_field def _init_tmp_dir(self) -> str: - tmp_dir = os.path.join(self._archivist_obj.data_dir, "tmp") + tmp_dir = os.path.join(self._archivist_obj.obj_dir, "tmp") os.makedirs(tmp_dir, exist_ok=True) for entry in os.scandir(tmp_dir): if entry.is_file(): @@ -90,7 +90,7 @@ def download_content_document_link_list( def load_content_document_link_list(self) -> ContentDocumentLinkList: document_link_list = ContentDocumentLinkList( - data_dir=self._archivist_obj.data_dir, dir_name_field=self._archivist_obj.dir_name_field + data_dir=self._archivist_obj.obj_dir, dir_name_field=self._archivist_obj.dir_name_field ) if not document_link_list.data_file_exist(): self.download_content_document_link_list(document_link_list=document_link_list) @@ -105,7 +105,7 @@ def load_content_version_list( document_link_list: ContentDocumentLinkList, batch_size: int = 3000, ) -> ContentVersionList: - content_version_list = ContentVersionList(data_dir=self._archivist_obj.data_dir) + content_version_list = ContentVersionList(data_dir=self._archivist_obj.obj_dir) if not content_version_list.data_file_exist(): doc_id_list = [link.content_document_id for link in document_link_list] list_size = len(doc_id_list) diff --git a/test/salesforce/test_download.py b/test/salesforce/test_download.py index d26c68a..38ed667 100644 --- a/test/salesforce/test_download.py +++ b/test/salesforce/test_download.py @@ -156,10 +156,10 @@ def test_downloaded_content_version_list_is_downloaded(): def test_download_content_version_list(): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version = ContentVersion( id="VID", document_id=link.content_document_id, @@ -171,12 +171,12 @@ def test_download_content_version_list(): ) version_list.add_version(version=version) download = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) generator = download.__iter__() assert next(generator) == ( version, - os.path.join(archivist_obj.data_dir, "files", link.download_dir_name, version.filename), + os.path.join(archivist_obj.obj_dir, "files", link.download_dir_name, version.filename), ) with pytest.raises(StopIteration): next(generator) @@ -185,10 +185,10 @@ def test_download_content_version_list(): @patch.object(concurrent.futures.ThreadPoolExecutor, "submit") def test_content_version_downloader_download_will_download_in_parallel(submit_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version_list.add_version( version=ContentVersion( id="VID1", @@ -212,9 +212,9 @@ def test_content_version_downloader_download_will_download_in_parallel(submit_mo ) ) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = Mock() downloader = ContentVersionDownloader( sf_client=sf_client, @@ -227,12 +227,12 @@ def test_content_version_downloader_download_will_download_in_parallel(submit_mo @patch("concurrent.futures.ThreadPoolExecutor") def test_content_version_downloader_download_will_use_defined_workers(thread_pool_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = Mock() max_workers = 3 downloader = ContentVersionDownloader( @@ -246,10 +246,10 @@ def test_content_version_downloader_download_will_use_defined_workers(thread_poo @patch.object(concurrent.futures.ThreadPoolExecutor, "shutdown", return_value=None) def test_content_version_downloader_download_will_gracefully_shutdown(shutdown_mock, submit_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version_list.add_version( version=ContentVersion( id="VID1", @@ -262,9 +262,9 @@ def test_content_version_downloader_download_will_gracefully_shutdown(shutdown_m ) ) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = Mock() downloader = ContentVersionDownloader( sf_client=sf_client, @@ -278,10 +278,10 @@ def test_content_version_downloader_download_will_gracefully_shutdown(shutdown_m @patch.object(ContentVersionDownloader, "download_content_version_from_sf", side_effect=RuntimeError) def test_content_version_downloader_download_will_return_download_stats(download_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version_list.add_version( version=ContentVersion( id="VID1", @@ -294,9 +294,9 @@ def test_content_version_downloader_download_will_return_download_stats(download ) ) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = Mock() downloader = ContentVersionDownloader( @@ -325,7 +325,7 @@ def test_content_version_downloader_download_content_version_from_sf_will_add_al version_number=1, content_size=10, ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = Mock() downloader = ContentVersionDownloader( sf_client=sf_client, @@ -341,8 +341,8 @@ def test_content_version_downloader_download_content_version_from_sf_will_copy_e with tempfile.TemporaryDirectory() as tmp_dir: archivist_obj = ArchivistObject(data_dir=tmp_dir, obj_type="User") - already_downloaded_path = os.path.join(archivist_obj.data_dir, "files", "file1.txt") - to_download_path = os.path.join(archivist_obj.data_dir, "files", "file2.txt") + already_downloaded_path = os.path.join(archivist_obj.obj_dir, "files", "file1.txt") + to_download_path = os.path.join(archivist_obj.obj_dir, "files", "file2.txt") download_list_mock = MagicMock() version1 = ContentVersion( id="CID", document_id="DOC1", checksum="c", extension="e", title="title", version_number=1, content_size=10 @@ -359,7 +359,7 @@ def test_content_version_downloader_download_content_version_from_sf_will_copy_e with open(already_downloaded_path, "wb") as downloaded_file: downloaded_file.write(file_contents) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) downloaded_version = DownloadedContentVersion( id=version1.id, document_id=version1.document_id, @@ -389,7 +389,7 @@ def test_content_version_downloader_download_content_version_from_sf_will_downlo version_number=1, content_size=10, ) - downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.obj_dir) sf_client = MagicMock() sf_client.download_content_version.return_value.iter_content.return_value = [b"test"] diff --git a/test/salesforce/test_salesforce.py b/test/salesforce/test_salesforce.py index 0001d3e..e9f32a9 100644 --- a/test/salesforce/test_salesforce.py +++ b/test/salesforce/test_salesforce.py @@ -129,7 +129,7 @@ def test_download_content_document_link_list_queries( ) client.bulk2.assert_called_with( query=expected_query, - path=os.path.join(archivist_obj.data_dir, "tmp"), + path=os.path.join(archivist_obj.obj_dir, "tmp"), max_records=50000, ) @@ -186,7 +186,7 @@ def test_download_content_document_link_list_csv_reading( ) client.bulk2 = Mock( side_effect=lambda *args, **kwargs: gen_temp_csv_files( - data=csv_files_data, dir_name=os.path.join(archivist_obj.data_dir, "tmp") + data=csv_files_data, dir_name=os.path.join(archivist_obj.obj_dir, "tmp") ) ) document_link_list = Mock() @@ -253,7 +253,7 @@ def test_download_content_version_list_queries( salesforce.download_content_version_list(**call_args) client.bulk2.assert_called_with( query=expected_query, - path=os.path.join(archivist_obj.data_dir, "tmp"), + path=os.path.join(archivist_obj.obj_dir, "tmp"), max_records=expected_max_records, ) @@ -298,7 +298,7 @@ def test_download_content_version_list_csv_reading( archivist_obj = ArchivistObject(data_dir=tmp_dir, obj_type="User") client.bulk2 = Mock( side_effect=lambda *args, **kwargs: gen_temp_csv_files( - data=csv_files_data, dir_name=os.path.join(archivist_obj.data_dir, "tmp") + data=csv_files_data, dir_name=os.path.join(archivist_obj.obj_dir, "tmp") ) ) content_version_list = Mock() diff --git a/test/salesforce/test_validation.py b/test/salesforce/test_validation.py index fdd19e5..d505cec 100644 --- a/test/salesforce/test_validation.py +++ b/test/salesforce/test_validation.py @@ -136,10 +136,10 @@ def test_download_stats_add_processed(): @patch.object(concurrent.futures.ThreadPoolExecutor, "submit") def test_content_version_download_validator_validate_will_validate_in_parallel(submit_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version_list.add_version( version=ContentVersion( id="VID1", @@ -163,9 +163,9 @@ def test_content_version_download_validator_validate_will_validate_in_parallel(s ) ) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) validator.validate(download_list=download_content_version_list) assert submit_mock.call_count == 2 @@ -174,12 +174,12 @@ def test_content_version_download_validator_validate_will_validate_in_parallel(s @patch("concurrent.futures.ThreadPoolExecutor") def test_content_version_download_validator_validate_will_use_defined_workers(thread_pool_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) max_workers = 3 validator = ContentVersionDownloadValidator( validated_content_version_list=validated_version_list, max_workers=max_workers @@ -192,10 +192,10 @@ def test_content_version_download_validator_validate_will_use_defined_workers(th @patch.object(concurrent.futures.ThreadPoolExecutor, "shutdown", return_value=None) def test_content_version_download_validator_validate_will_gracefully_shutdown(shutdown_mock, submit_mock): archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") - link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link_list = ContentDocumentLinkList(data_dir=archivist_obj.obj_dir) link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") link_list.add_link(doc_link=link) - version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list = ContentVersionList(data_dir=archivist_obj.obj_dir) version_list.add_version( version=ContentVersion( id="VID1", @@ -208,9 +208,9 @@ def test_content_version_download_validator_validate_will_gracefully_shutdown(sh ) ) download_content_version_list = DownloadContentVersionList( - document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.obj_dir ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) with pytest.raises(KeyboardInterrupt): validator.validate(download_list=download_content_version_list) @@ -228,7 +228,7 @@ def test_content_version_download_validator_validate_version_will_find_missing_f version_number=1, content_size=10, ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) assert not validator.validate_version(version=version, download_path="/non/existing/path") @@ -254,7 +254,7 @@ def test_content_version_download_validator_validate_version_will_check_validate version_number=2, content_size=10, ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validated_path = "/path/to/file" validated_version_list.add_version(ValidatedContentVersion(path=validated_path, checksum="abc")) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) @@ -287,7 +287,7 @@ def test_content_version_download_validator_validate_version_will_calculate_chec version_number=1, content_size=10, ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) assert validator.validate_version(version=version, download_path=download_path) == should_match assert len(validated_version_list) == 1 @@ -310,7 +310,7 @@ def test_content_version_download_validator_validate_version_will_update_validat version_number=1, content_size=10, ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) validator.validate_version(version=version, download_path=download_path) assert len(validated_version_list) == 1 @@ -329,6 +329,6 @@ def test_content_version_download_validator_validate_version_will_return_invalid version_number=1, content_size=10, ) - validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.data_dir) + validated_version_list = ValidatedContentVersionList(data_dir=archivist_obj.obj_dir) validator = ContentVersionDownloadValidator(validated_content_version_list=validated_version_list) assert not validator.validate_version(version=version, download_path="/fake/path/download") diff --git a/test/test_archivist.py b/test/test_archivist.py index 7e5f355..ae4a269 100644 --- a/test/test_archivist.py +++ b/test/test_archivist.py @@ -5,7 +5,8 @@ from unittest.mock import patch, MagicMock, call, ANY import pytest -import schema +import yaml +from pydantic import ValidationError from salesforce_archivist.archivist import ArchivistObject, ArchivistAuth, ArchivistConfig, Archivist from salesforce_archivist.salesforce.download import DownloadedContentVersionList, DownloadStats @@ -46,7 +47,8 @@ def test_archivist_object_props(data_dir, obj_type, modified_date_lt, modified_d archivist_obj.modified_date_lt, archivist_obj.modified_date_gt, archivist_obj.dir_name_field, - ) == (os.path.join(data_dir, obj_type), obj_type, modified_date_lt, modified_date_gt, dir_name_field) + archivist_obj.obj_dir, + ) == (data_dir, obj_type, modified_date_lt, modified_date_gt, dir_name_field, os.path.join(data_dir, obj_type)) def test_archivist_object_quality(): @@ -94,7 +96,7 @@ def test_archivist_auth_props(): @pytest.mark.parametrize( - "yaml_data, expect_exception", + "yaml_str, expect_exception", [ ( """\ @@ -148,22 +150,19 @@ def test_archivist_auth_props(): ), ], ) -def test_archivist_config_validation(yaml_data, expect_exception): +def test_archivist_config_validation(yaml_str, expect_exception): with tempfile.TemporaryDirectory() as tmp_dir: - path = os.path.join(tmp_dir, "config.yaml") - with open(path, "wb") as config: - data = yaml_data.replace("{data_dir}", tmp_dir) - config.write(data.encode("utf-8")) + data = yaml_str.replace("{data_dir}", tmp_dir) if expect_exception: - with pytest.raises(schema.SchemaError): - ArchivistConfig(path) + with pytest.raises(ValidationError): + ArchivistConfig(**yaml.safe_load(data)) else: - ArchivistConfig(path) + ArchivistConfig(**yaml.safe_load(data)) def test_archivist_config_props(): with tempfile.TemporaryDirectory() as tmp_dir: - yaml = textwrap.dedent( + yaml_str = textwrap.dedent( """\ data_dir: {data_dir} max_api_usage_percent: 40 @@ -184,11 +183,8 @@ def test_archivist_config_props(): Booking__c: {{}} """ ).format(data_dir=tmp_dir) - path = os.path.join(tmp_dir, "config.yaml") - with open(path, "wb") as config: - config.write(yaml.encode("utf-8")) - config = ArchivistConfig(path) + config = ArchivistConfig(**yaml.safe_load(yaml_str)) assert config.data_dir == tmp_dir assert config.max_api_usage_percent == 40.0 @@ -199,9 +195,10 @@ def test_archivist_config_props(): assert config.auth.instance_url == "https://login.salesforce.com/" assert config.auth.consumer_key == "abc" assert config.auth.private_key == "test\n" - archivist_object = config.objects[0] + archivist_object = config.objects["User"] assert archivist_object.obj_type == "User" - assert archivist_object.data_dir == os.path.join(config.data_dir, archivist_object.obj_type) + assert archivist_object.data_dir == config.data_dir + assert archivist_object.obj_dir == os.path.join(config.data_dir, archivist_object.obj_type) assert archivist_object.dir_name_field == "LinkedEntity.Username" assert archivist_object.modified_date_gt == datetime.datetime( year=2017, month=1, day=1, tzinfo=datetime.timezone.utc @@ -209,7 +206,7 @@ def test_archivist_config_props(): assert archivist_object.modified_date_lt == datetime.datetime( year=2023, month=8, day=1, tzinfo=datetime.timezone.utc ) - archivist_object_with_defaults = config.objects[1] + archivist_object_with_defaults = config.objects["Booking__c"] assert archivist_object_with_defaults.modified_date_gt == datetime.datetime( year=2011, month=1, day=1, tzinfo=datetime.timezone.utc ) @@ -221,7 +218,7 @@ def test_archivist_config_props(): @patch.object(DownloadedContentVersionList, "data_file_exist", side_effect=[False, True]) @patch.object(DownloadedContentVersionList, "load_data_from_file") def test_archivist_download_will_load_downloaded_list_if_possible(load_mock, exist_mock): - archivist = Archivist(data_dir="/fake/dir", objects=[], sf_client=MagicMock()) + archivist = Archivist(data_dir="/fake/dir", objects={}, sf_client=MagicMock()) archivist.download() exist_mock.assert_called_once() load_mock.assert_not_called() @@ -237,10 +234,10 @@ def test_archivist_download_will_load_lists_and_call_download_method( download_mock, load_version_list_mock, load_doc_link_list_mock ): download_mock.return_value = DownloadStats() - objects = [ - ArchivistObject(data_dir="/fakse/dir", obj_type="User"), - ArchivistObject(data_dir="/fakse/dir", obj_type="Email"), - ] + objects = { + "User": ArchivistObject(data_dir="/fakse/dir", obj_type="User"), + "Email": ArchivistObject(data_dir="/fakse/dir", obj_type="Email"), + } archivist = Archivist(data_dir="/fake/dir", objects=objects, sf_client=MagicMock()) archivist.download() assert load_doc_link_list_mock.call_count == 2 @@ -262,9 +259,9 @@ def test_archivist_download_will_return_correct_bool_value( download_mock.return_value = stats archivist = Archivist( data_dir="/fake/dir", - objects=[ - ArchivistObject(data_dir="/fakse/dir", obj_type="User"), - ], + objects={ + "User": ArchivistObject(data_dir="/fakse/dir", obj_type="User"), + }, sf_client=MagicMock(), ) assert archivist.download() == expected_return @@ -273,7 +270,7 @@ def test_archivist_download_will_return_correct_bool_value( @patch.object(ValidatedContentVersionList, "data_file_exist", side_effect=[False, True]) @patch.object(ValidatedContentVersionList, "load_data_from_file") def test_archivist_validate_will_load_validated_list_if_possible(load_mock, exist_mock): - archivist = Archivist(data_dir="/fake/dir", objects=[], sf_client=MagicMock()) + archivist = Archivist(data_dir="/fake/dir", objects={}, sf_client=MagicMock()) archivist.validate() exist_mock.assert_called_once() load_mock.assert_not_called() @@ -289,10 +286,10 @@ def test_archivist_validate_will_load_lists_and_call_validate_method( validate_mock, load_version_list_mock, load_doc_link_list_mock ): validate_mock.return_value = ValidationStats() - objects = [ - ArchivistObject(data_dir="/fakse/dir", obj_type="User"), - ArchivistObject(data_dir="/fakse/dir", obj_type="Email"), - ] + objects = { + "User": ArchivistObject(data_dir="/fakse/dir", obj_type="User"), + "Email": ArchivistObject(data_dir="/fakse/dir", obj_type="Email"), + } max_workers = 6 archivist = Archivist(data_dir="/fake/dir", objects=objects, sf_client=MagicMock(), max_workers=max_workers) archivist.validate()