From 1d71e98198a084c25068f057fe3fc01a3c9ab2da Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 8 Apr 2024 15:36:24 -0700 Subject: [PATCH 01/15] Initial commit to push semantic model generator --- .github/workflows/lint.yaml | 59 + .github/workflows/release.yaml | 42 + .github/workflows/test.yaml | 46 + .gitignore | 37 + .python-version | 1 + CHANGELOG.md | 7 + Makefile | 56 + README.md | 50 +- mypy.ini | 10 + poetry.lock | 1393 +++++++++++++++++ pyproject.toml | 33 + semantic_model_generator/__init__.py | 0 .../data_processing/__init__.py | 0 .../data_processing/data_types.py | 47 + .../data_processing/proto_utils.py | 37 + semantic_model_generator/main.py | 288 ++++ .../protos/semantic_model.proto | 192 +++ .../protos/semantic_model_pb2.py | 44 + .../protos/semantic_model_pb2.pyi | 167 ++ .../snowflake_utils/env_vars.py | 8 + .../snowflake_utils/snowflake_connector.py | 374 +++++ .../snowflake_utils/utils.py | 73 + .../tests/generate_semantic_model_test.py | 280 ++++ .../tests/snowflake_connector_test.py | 164 ++ 24 files changed, 3407 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lint.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 CHANGELOG.md create mode 100644 Makefile create mode 100644 mypy.ini create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 semantic_model_generator/__init__.py create mode 100644 semantic_model_generator/data_processing/__init__.py create mode 100644 semantic_model_generator/data_processing/data_types.py create mode 100644 semantic_model_generator/data_processing/proto_utils.py create mode 100644 semantic_model_generator/main.py create mode 100644 semantic_model_generator/protos/semantic_model.proto create mode 100644 semantic_model_generator/protos/semantic_model_pb2.py create mode 100644 semantic_model_generator/protos/semantic_model_pb2.pyi create mode 100644 semantic_model_generator/snowflake_utils/env_vars.py create mode 100644 semantic_model_generator/snowflake_utils/snowflake_connector.py create mode 100644 semantic_model_generator/snowflake_utils/utils.py create mode 100644 semantic_model_generator/tests/generate_semantic_model_test.py create mode 100644 semantic_model_generator/tests/snowflake_connector_test.py diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..5551f4cf --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,59 @@ +name: Semantic Model Format & Lint + +on: + pull_request: + paths: + - "semantic_model_generator/**" + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # Caching dependencies using Poetry + - name: Cache Poetry virtualenv + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + + - name: Configure Poetry + run: | + $HOME/.local/bin/poetry config virtualenvs.create false + + - name: Install dependencies using Poetry + run: | + $HOME/.local/bin/poetry install --no-interaction + + - name: Run mypy + run: | + cd semantic_model_generator && make run_mypy + + - name: Check with black + run: | + cd semantic_model_generator && make check_black + + - name: Check with isort + run: | + cd semantic_model_generator && make check_isort + + - name: Run flake8 + run: | + cd semantic_model_generator && make run_flake8 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..f012fcbf --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,42 @@ +name: Build and Attach Wheel to GitHub Release + +on: + push: + tags: + - 'v*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install Poetry + run: pip install poetry + + - name: Get the version from pyproject.toml + run: echo "VERSION=$(poetry version -s)" >> $GITHUB_ENV + + - name: Build Wheel + run: poetry build -f wheel + + - name: Extract Changelog for the Version + run: | + VERSION=${{ env.VERSION }} + CHANGELOG=$(awk '/^## \['"${VERSION//./\\.}"'\]/ {flag=1; next} /^## \[/ {flag=0} flag' CHANGELOG.md) + echo "CHANGELOG<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Upload Wheel to Release + uses: softprops/action-gh-release@v1 + with: + files: dist/*.whl + body: ${{ env.CHANGELOG }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..3ffb40d9 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,46 @@ +name: Semantic Model Generator Test + +on: + pull_request: + paths: + - "semantic_model_generator/**" + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # Caching dependencies using Poetry + - name: Cache Poetry virtualenv + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + + - name: Configure Poetry + run: | + $HOME/.local/bin/poetry config virtualenvs.create false + + - name: Install dependencies using Poetry + run: | + $HOME/.local/bin/poetry install --no-interaction + + - name: Test + run: | + make test diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2d06207f --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Local python environment +pyvenv + +# Notebook intermediate state +*.ipynb_checkpoints + +# Mypy +.mypy_cache + +# Pytest +.pytest_cache + +# pycache +**/__pycache__ + +# Python package builds +*.egg-info + +# VSCode +.vscode/settings.json +.vscode/.ropeproject +.vscode/*.log + +# Envs +.env +.venv +.direnv +.envrc + +# Output semantic models +semantic_model_generator/output_models/* + +# Whls +dist/ + +# test coverage +.coverage \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..c8cfe395 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..8f334a17 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +You must follow the format of `## [VERSION-NUMBER]` for the GitHub workflow to pick up the text. + +## [1.0.0] - 2024-04-08 +### Released +- Initial release of the project. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..4f2e15f3 --- /dev/null +++ b/Makefile @@ -0,0 +1,56 @@ + + + +check-deps: ## Check if poetry is installed on your system. + @command -v poetry >/dev/null 2>&1 || { echo >&2 "Poetry is required but it's not installed. Please install Poetry by following the instructions at: https://python-poetry.org/docs/#installation"; exit 1; } + @command -v pyenv >/dev/null 2>&1 || { echo >&2 "pyenv is recommended for managing Python versions but it's not installed. Install via `brew install pyenv`"; exit 1; } + @echo "Setting Python version to 3.10 using pyenv." + @pyenv local 3.10 + +shell: check-deps ## Get into a poetry shell + poetry shell + +setup: check-deps shell ## Install dependencies into your poetry environment. + poetry install + +# Linting and formatting below. +run_mypy: ## Run mypy + mypy --config-file=mypy.ini . + +run_flake8: ## Run flake8 + flake8 --ignore=E203,E501,W503 --exclude=pyvenv,tmp,*_pb2.py,*_pb2.pyi,images/*/src . + +check_black: ## Check to see if files would be updated with black. + # Exclude pyvenv and all generated protobuf code. + black --check --exclude="pyvenv|.*_pb2.py|.*_pb2.pyi" . + +run_black: ## Run black to format files. + # Exclude pyvenv, tmp, and all generated protobuf code. + black --exclude="pyvenv|tmp|.*_pb2.py|.*_pb2.pyi" . + +check_isort: ## Check if files would be updated with isort. + isort --profile black --check --skip=pyvenv --skip-glob='*_pb2.py*' . + +run_isort: ## Run isort to update imports. + isort --profile black --skip=pyvenv --skip=tmp --skip-glob='*_pb2.py*' . + + +fmt_lint: shell ## lint/fmt in current python environment + make run_mypy run_black run_isort run_flake8 + +# Test below +test: shell ## Run tests. + python -m pytest -vvs semantic_model_generator + +# Release +update-version: ## Bump poetry and github version. TYPE should be `patch` `minor` or `major` + @echo "Updating Poetry version ($(TYPE)) and creating a Git tag..." + @poetry version $(TYPE) + @VERSION=$$(poetry version -s) && git add pyproject.toml && git commit -m "Bump version to $$VERSION" && git tag v$$VERSION + @echo "Version updated to $$VERSION. Merge your branch then run `make release`" + +release: ## Runs the release workflow. + git push && git push --tags + +help: ## Show this help. + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's \ No newline at end of file diff --git a/README.md b/README.md index 76c650e1..cbfb8d05 100644 --- a/README.md +++ b/README.md @@ -1 +1,49 @@ -# copilot-semantic-model-generator \ No newline at end of file +# semantic-model-generator + +The `Semantic Model Generator` is used to generate a semantic model for use in your Snowflake account. + +## Setup + +This project uses `poetry` to manage dependencies and we recommend `pyenv` for python version management. + +1. `make setup` + +Next, export your credentials as environment variables. Note, `host` is optional depending on your Snowflake deployment. + +```bash +export SNOWFLAKE_ROLE = "" +export SNOWFLAKE_WAREHOUSE = "" +export SNOWFLAKE_USER = "" +export SNOWFLAKE_PASSWORD = "" +export SNOWFLAKE_HOST = "" +``` + +## Usage + +You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. + +All generated models be default are saved under `semantic_model_generator/output_models`. + +**Important**: After generation, your yamls will have a series lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. + + +```bash +python -m semantic_model_generator.main \ + --fqn_tables "['..','..']" \ + --snowflake_account="" +``` + +## Release + +In order to push a new build and release, follow the below steps. + +1. Checkout a new branch from main. Please name this branch `release-YYYY-MM-DD`. +2. Bump the poetry and github tags depending on if this is a patch, minor, or major version update: + * `export TYPE=patch make update-version` + * `export TYPE=minor make update-version` + * `export TYPE=major make update-version` +3. Update the `CHANGELOG.md` adding a relevant header for your version number along with a description of the changes made. +4. Commit the updated `pyproject.toml` and `CHANGELOG.md` and push. +5. Merge your branch. +6. Push the updated tags to trigger the release workflow with `make release`. + diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..298f7a5c --- /dev/null +++ b/mypy.ini @@ -0,0 +1,10 @@ +[mypy] +plugins = pydantic.mypy + +ignore_missing_imports = True +strict = True +disallow_untyped_defs = True +warn_unused_ignores = False +disallow_any_generics = True + +exclude = pyvenv|(_test\.py|test_.*\.py)|_pb2\.py|_pb2\.pyi diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..4cd043f9 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1393 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +optional = false +python-versions = "*" +files = [ + {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, + {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, +] + +[[package]] +name = "black" +version = "24.3.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395"}, + {file = "black-24.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995"}, + {file = "black-24.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"}, + {file = "black-24.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0"}, + {file = "black-24.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9"}, + {file = "black-24.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597"}, + {file = "black-24.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d"}, + {file = "black-24.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5"}, + {file = "black-24.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f"}, + {file = "black-24.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11"}, + {file = "black-24.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4"}, + {file = "black-24.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5"}, + {file = "black-24.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837"}, + {file = "black-24.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd"}, + {file = "black-24.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213"}, + {file = "black-24.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959"}, + {file = "black-24.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb"}, + {file = "black-24.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7"}, + {file = "black-24.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7"}, + {file = "black-24.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f"}, + {file = "black-24.3.0-py3-none-any.whl", hash = "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93"}, + {file = "black-24.3.0.tar.gz", hash = "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.5" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da"}, + {file = "cryptography-42.0.5-cp37-abi3-win32.whl", hash = "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74"}, + {file = "cryptography-42.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940"}, + {file = "cryptography-42.0.5-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30"}, + {file = "cryptography-42.0.5-cp39-abi3-win32.whl", hash = "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413"}, + {file = "cryptography-42.0.5-cp39-abi3-win_amd64.whl", hash = "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd"}, + {file = "cryptography-42.0.5.tar.gz", hash = "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.13.3" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.3-py3-none-any.whl", hash = "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb"}, + {file = "filelock-3.13.3.tar.gz", hash = "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "flake8" +version = "7.0.0" +description = "the modular source code checker: pep8 pyflakes and co" +optional = false +python-versions = ">=3.8.1" +files = [ + {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, + {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, +] + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.11.0,<2.12.0" +pyflakes = ">=3.2.0,<3.3.0" + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + +[[package]] +name = "jsonargparse" +version = "4.27.7" +description = "Implement minimal boilerplate CLIs derived from type hints and parse from command line, config files and environment variables." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonargparse-4.27.7-py3-none-any.whl", hash = "sha256:9042339d1fc9f39b2ef61c0f9c4accd144db14a3aab9359e9b2e883d3af3740a"}, + {file = "jsonargparse-4.27.7.tar.gz", hash = "sha256:480fa4b2f4c75f267cbaa13fbf4dc86553f27252041d1d9066f2d0e25bf6389b"}, +] + +[package.dependencies] +PyYAML = ">=3.13" + +[package.extras] +all = ["jsonargparse[argcomplete]", "jsonargparse[fsspec]", "jsonargparse[jsonnet]", "jsonargparse[jsonschema]", "jsonargparse[omegaconf]", "jsonargparse[reconplogger]", "jsonargparse[ruyaml]", "jsonargparse[signatures]", "jsonargparse[typing-extensions]", "jsonargparse[urls]"] +argcomplete = ["argcomplete (>=2.0.0)"] +coverage = ["jsonargparse[test-no-urls]", "pytest-cov (>=4.0.0)"] +dev = ["build (>=0.10.0)", "jsonargparse[coverage]", "jsonargparse[doc]", "jsonargparse[mypy]", "jsonargparse[test]", "pre-commit (>=2.19.0)", "tox (>=3.25.0)"] +doc = ["Sphinx (>=1.7.9)", "autodocsumm (>=0.1.10)", "sphinx-autodoc-typehints (>=1.19.5)", "sphinx-rtd-theme (>=1.2.2)"] +fsspec = ["fsspec (>=0.8.4)"] +jsonnet = ["jsonnet (>=0.13.0)", "jsonnet-binary (>=0.17.0)"] +jsonschema = ["jsonschema (>=3.2.0)"] +maintainer = ["bump2version (>=0.5.11)", "twine (>=4.0.2)"] +omegaconf = ["omegaconf (>=2.1.1)"] +reconplogger = ["reconplogger (>=4.4.0)"] +ruyaml = ["ruyaml (>=0.20.0)"] +signatures = ["docstring-parser (>=0.15)", "jsonargparse[typing-extensions]", "typeshed-client (>=2.1.0)"] +test = ["attrs (>=22.2.0)", "jsonargparse[test-no-urls]", "pydantic (>=2.3.0)", "responses (>=0.12.0)", "types-PyYAML (>=6.0.11)", "types-requests (>=2.28.9)"] +test-no-urls = ["pytest (>=6.2.5)", "pytest-subtests (>=0.8.0)"] +typing-extensions = ["typing-extensions (>=3.10.0.0)"] +urls = ["requests (>=2.18.4)"] + +[[package]] +name = "loguru" +version = "0.7.2" +description = "Python logging made (stupidly) simple" +optional = false +python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb"}, + {file = "loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mypy" +version = "1.9.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"}, + {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"}, + {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"}, + {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"}, + {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"}, + {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"}, + {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"}, + {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"}, + {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"}, + {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"}, + {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"}, + {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"}, + {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"}, + {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"}, + {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"}, + {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"}, + {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"}, + {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pandas" +version = "2.2.1" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88"}, + {file = "pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944"}, + {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359"}, + {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51"}, + {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06"}, + {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9"}, + {file = "pandas-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0"}, + {file = "pandas-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b"}, + {file = "pandas-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a"}, + {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02"}, + {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403"}, + {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd"}, + {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7"}, + {file = "pandas-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e"}, + {file = "pandas-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c"}, + {file = "pandas-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee"}, + {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2"}, + {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0"}, + {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc"}, + {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89"}, + {file = "pandas-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb"}, + {file = "pandas-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397"}, + {file = "pandas-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16"}, + {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019"}, + {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"}, + {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6"}, + {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be"}, + {file = "pandas-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab"}, + {file = "pandas-2.2.1.tar.gz", hash = "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "pip-licenses" +version = "4.4.0" +description = "Dump the software license list of Python packages installed with pip." +optional = false +python-versions = "~=3.8" +files = [ + {file = "pip-licenses-4.4.0.tar.gz", hash = "sha256:996817118375445243a34faafe23c06f6b2d250247c4046571b5a6722d45be69"}, + {file = "pip_licenses-4.4.0-py3-none-any.whl", hash = "sha256:dbad2ac5a25f574cabe2716f2f031a0c5fa359bed9b3ef615301f4e546893b46"}, +] + +[package.dependencies] +prettytable = ">=2.3.0" + +[package.extras] +test = ["docutils", "mypy", "pytest-cov", "pytest-pycodestyle", "pytest-runner"] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "prettytable" +version = "3.10.0" +description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" +optional = false +python-versions = ">=3.8" +files = [ + {file = "prettytable-3.10.0-py3-none-any.whl", hash = "sha256:6536efaf0757fdaa7d22e78b3aac3b69ea1b7200538c2c6995d649365bddab92"}, + {file = "prettytable-3.10.0.tar.gz", hash = "sha256:9665594d137fb08a1117518c25551e0ede1687197cf353a4fdc78d27e1073568"}, +] + +[package.dependencies] +wcwidth = "*" + +[package.extras] +tests = ["pytest", "pytest-cov", "pytest-lazy-fixtures"] + +[[package]] +name = "protobuf" +version = "5.26.1" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-5.26.1-cp310-abi3-win32.whl", hash = "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23"}, + {file = "protobuf-5.26.1-cp310-abi3-win_amd64.whl", hash = "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33"}, + {file = "protobuf-5.26.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d"}, + {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca"}, + {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7"}, + {file = "protobuf-5.26.1-cp38-cp38-win32.whl", hash = "sha256:d693d2504ca96750d92d9de8a103102dd648fda04540495535f0fec7577ed8fc"}, + {file = "protobuf-5.26.1-cp38-cp38-win_amd64.whl", hash = "sha256:9b557c317ebe6836835ec4ef74ec3e994ad0894ea424314ad3552bc6e8835b4e"}, + {file = "protobuf-5.26.1-cp39-cp39-win32.whl", hash = "sha256:b9ba3ca83c2e31219ffbeb9d76b63aad35a3eb1544170c55336993d7a18ae72c"}, + {file = "protobuf-5.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ee014c2c87582e101d6b54260af03b6596728505c79f17c8586e7523aaa8f8c"}, + {file = "protobuf-5.26.1-py3-none-any.whl", hash = "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932"}, + {file = "protobuf-5.26.1.tar.gz", hash = "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51"}, +] + +[[package]] +name = "pyarrow" +version = "15.0.2" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-15.0.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:88b340f0a1d05b5ccc3d2d986279045655b1fe8e41aba6ca44ea28da0d1455d8"}, + {file = "pyarrow-15.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eaa8f96cecf32da508e6c7f69bb8401f03745c050c1dd42ec2596f2e98deecac"}, + {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23c6753ed4f6adb8461e7c383e418391b8d8453c5d67e17f416c3a5d5709afbd"}, + {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f639c059035011db8c0497e541a8a45d98a58dbe34dc8fadd0ef128f2cee46e5"}, + {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:290e36a59a0993e9a5224ed2fb3e53375770f07379a0ea03ee2fce2e6d30b423"}, + {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06c2bb2a98bc792f040bef31ad3e9be6a63d0cb39189227c08a7d955db96816e"}, + {file = "pyarrow-15.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:f7a197f3670606a960ddc12adbe8075cea5f707ad7bf0dffa09637fdbb89f76c"}, + {file = "pyarrow-15.0.2-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:5f8bc839ea36b1f99984c78e06e7a06054693dc2af8920f6fb416b5bca9944e4"}, + {file = "pyarrow-15.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5e81dfb4e519baa6b4c80410421528c214427e77ca0ea9461eb4097c328fa33"}, + {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a4f240852b302a7af4646c8bfe9950c4691a419847001178662a98915fd7ee7"}, + {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e7d9cfb5a1e648e172428c7a42b744610956f3b70f524aa3a6c02a448ba853e"}, + {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2d4f905209de70c0eb5b2de6763104d5a9a37430f137678edfb9a675bac9cd98"}, + {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90adb99e8ce5f36fbecbbc422e7dcbcbed07d985eed6062e459e23f9e71fd197"}, + {file = "pyarrow-15.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:b116e7fd7889294cbd24eb90cd9bdd3850be3738d61297855a71ac3b8124ee38"}, + {file = "pyarrow-15.0.2-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:25335e6f1f07fdaa026a61c758ee7d19ce824a866b27bba744348fa73bb5a440"}, + {file = "pyarrow-15.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:90f19e976d9c3d8e73c80be84ddbe2f830b6304e4c576349d9360e335cd627fc"}, + {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a22366249bf5fd40ddacc4f03cd3160f2d7c247692945afb1899bab8a140ddfb"}, + {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2a335198f886b07e4b5ea16d08ee06557e07db54a8400cc0d03c7f6a22f785f"}, + {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e6d459c0c22f0b9c810a3917a1de3ee704b021a5fb8b3bacf968eece6df098f"}, + {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:033b7cad32198754d93465dcfb71d0ba7cb7cd5c9afd7052cab7214676eec38b"}, + {file = "pyarrow-15.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:29850d050379d6e8b5a693098f4de7fd6a2bea4365bfd073d7c57c57b95041ee"}, + {file = "pyarrow-15.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7167107d7fb6dcadb375b4b691b7e316f4368f39f6f45405a05535d7ad5e5058"}, + {file = "pyarrow-15.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e85241b44cc3d365ef950432a1b3bd44ac54626f37b2e3a0cc89c20e45dfd8bf"}, + {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:248723e4ed3255fcd73edcecc209744d58a9ca852e4cf3d2577811b6d4b59818"}, + {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ff3bdfe6f1b81ca5b73b70a8d482d37a766433823e0c21e22d1d7dde76ca33f"}, + {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f3d77463dee7e9f284ef42d341689b459a63ff2e75cee2b9302058d0d98fe142"}, + {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:8c1faf2482fb89766e79745670cbca04e7018497d85be9242d5350cba21357e1"}, + {file = "pyarrow-15.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:28f3016958a8e45a1069303a4a4f6a7d4910643fc08adb1e2e4a7ff056272ad3"}, + {file = "pyarrow-15.0.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:89722cb64286ab3d4daf168386f6968c126057b8c7ec3ef96302e81d8cdb8ae4"}, + {file = "pyarrow-15.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd0ba387705044b3ac77b1b317165c0498299b08261d8122c96051024f953cd5"}, + {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad2459bf1f22b6a5cdcc27ebfd99307d5526b62d217b984b9f5c974651398832"}, + {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58922e4bfece8b02abf7159f1f53a8f4d9f8e08f2d988109126c17c3bb261f22"}, + {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:adccc81d3dc0478ea0b498807b39a8d41628fa9210729b2f718b78cb997c7c91"}, + {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8bd2baa5fe531571847983f36a30ddbf65261ef23e496862ece83bdceb70420d"}, + {file = "pyarrow-15.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6669799a1d4ca9da9c7e06ef48368320f5856f36f9a4dd31a11839dda3f6cc8c"}, + {file = "pyarrow-15.0.2.tar.gz", hash = "sha256:9c9bc803cb3b7bfacc1e96ffbfd923601065d9d3f911179d81e72d99fd74a3d9"}, +] + +[package.dependencies] +numpy = ">=1.16.6,<2" + +[[package]] +name = "pycodestyle" +version = "2.11.1" +description = "Python style guide checker" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, +] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pydantic" +version = "2.6.4" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.6.4-py3-none-any.whl", hash = "sha256:cc46fce86607580867bdc3361ad462bab9c222ef042d3da86f2fb333e1d916c5"}, + {file = "pydantic-2.6.4.tar.gz", hash = "sha256:b1704e0847db01817624a6b86766967f552dd9dbf3afba4004409f908dcc84e6"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.16.3" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.16.3" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"}, + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"}, + {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"}, + {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"}, + {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"}, + {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"}, + {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"}, + {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"}, + {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"}, + {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"}, + {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"}, + {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"}, + {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"}, + {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"}, + {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyflakes" +version = "3.2.0" +description = "passive checker of Python programs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, +] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyopenssl" +version = "24.1.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyOpenSSL-24.1.0-py3-none-any.whl", hash = "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad"}, + {file = "pyOpenSSL-24.1.0.tar.gz", hash = "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"}, +] + +[package.dependencies] +cryptography = ">=41.0.5,<43" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + +[[package]] +name = "pytest" +version = "8.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, + {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.4,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "ruamel-yaml" +version = "0.18.6" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, + {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.6" +files = [ + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "snowflake-connector-python" +version = "3.7.1" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake-connector-python-3.7.1.tar.gz", hash = "sha256:dc6982f653860edef1614a3047a16bcb07e22775e6e090808bbf4f7629c4191d"}, + {file = "snowflake_connector_python-3.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c5c60093a47ead3bf9dafe6194626f433a1872ddb521f1635492584ee65885c3"}, + {file = "snowflake_connector_python-3.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:959b1c93f9aff684627c3ce56e9193e23f28d867d932da83a18d8aff613cc182"}, + {file = "snowflake_connector_python-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6966a50dd981ce8f553867e8e4216ba966ae8d0625b2ef63de383f3184d7351"}, + {file = "snowflake_connector_python-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8f1c1c95388b3bb18fb17f185e21860cf4f8feae4167a432183dfadbda3f7ae"}, + {file = "snowflake_connector_python-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:d771ff5e632a088b9aa1c663f956cc89714cc1c5732cbdeb39217d749c8c447b"}, + {file = "snowflake_connector_python-3.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:796be35e6b4842fc4eaf7d82628f61a9bff946c89a7c47a97c2ac442e7c69b45"}, + {file = "snowflake_connector_python-3.7.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:d26b509185333a7681191fed6d4e345d3206bb630e545f575d8cce0149f2f7cf"}, + {file = "snowflake_connector_python-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a972f4cabb3c5fd23f6b310ba1c2216f0d5afa0dbb95d05ad935ddeba700a718"}, + {file = "snowflake_connector_python-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:266506d4001f3260a80425d1d9b61e1f4d1183dcb4623e3799b7dc0757327469"}, + {file = "snowflake_connector_python-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:cb6d90cf992af2a795613ba54a9b0824dc1b73b4f4377b4f7479f40b5135d4eb"}, + {file = "snowflake_connector_python-3.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:09b62cb4fd447d99756aa251d07fdec726b4c0173978450d6172e900ede2da7f"}, + {file = "snowflake_connector_python-3.7.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:3f466f459942520fc2bc4ae2a0ffbdc2535cb2eac73e180c7ec8fa11a75fe993"}, + {file = "snowflake_connector_python-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0a770fe783868b1559ba5e72c436b2781cd0a06ca0c9127776a3f927a91249f"}, + {file = "snowflake_connector_python-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5f178d71892d54f246681e601a628cab7086fbd80828430d795b60a5e4991f"}, + {file = "snowflake_connector_python-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:c6822d6b21d22a237d90b0b45b7d28d12d9b9ff392445fc1aaf3fe60aa200e81"}, + {file = "snowflake_connector_python-3.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2e2fdd6c65f03f0fd649477e35828e7ebd0bd2717852542e720f2a4ab3443bc8"}, + {file = "snowflake_connector_python-3.7.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:0ec3a7d17643167e1224d0f992fd34abf3ddca64099c3c377bbeb5ec54b58fc6"}, + {file = "snowflake_connector_python-3.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcce35becef53b80d8addbd0d248242c14f74e842eb989baabceb42016800d66"}, + {file = "snowflake_connector_python-3.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:093732fb559d255e6687eb1940e6677ae98cfa3713c548696ad99f345c3c43d7"}, + {file = "snowflake_connector_python-3.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:cf1021752e11b9e23688440b81e116dd54ada6335a9745db8ce3f223fa5e942c"}, + {file = "snowflake_connector_python-3.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2ab47de37dcf66464f45eb365c66ec07c3af19a4011ef1d575116af308fcb025"}, + {file = "snowflake_connector_python-3.7.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:c4e824c6226a8c01ccc0af258517d707b614254f58cc7b0b6005cf641e34d2c2"}, + {file = "snowflake_connector_python-3.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97f13203ec29790d1c7e6e7d37a16ddb5182c93820a328a7f6d3678733baadd8"}, + {file = "snowflake_connector_python-3.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d73d2af0435cb29bda23609a94220dd0e4765e997fe669a7ef117620b731c772"}, + {file = "snowflake_connector_python-3.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:1b833b4165e7c3b0e911fb33aba936bee46b320a367b3f9451987039bef85be9"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset-normalizer = ">=2,<4" +cryptography = ">=3.1.0,<43.0.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +packaging = "*" +pandas = {version = ">=1.0.0,<3.0.0", optional = true, markers = "extra == \"pandas\""} +platformdirs = ">=2.6.0,<4.0.0" +pyarrow = {version = "*", optional = true, markers = "extra == \"pandas\""} +pyjwt = "<3.0.0" +pyOpenSSL = ">=16.2.0,<25.0.0" +pytz = "*" +requests = "<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing-extensions = ">=4.3,<5" + +[package.extras] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.27.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pyarrow"] +secure-local-storage = ["keyring (>=23.1.0,<25.0.0)"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.12.4" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.4-py3-none-any.whl", hash = "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b"}, + {file = "tomlkit-0.12.4.tar.gz", hash = "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3"}, +] + +[[package]] +name = "tqdm" +version = "4.66.2" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"}, + {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "types-protobuf" +version = "4.24.0.20240311" +description = "Typing stubs for protobuf" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-protobuf-4.24.0.20240311.tar.gz", hash = "sha256:c80426f9fb9b21aee514691e96ab32a5cd694a82e2ac07964b352c3e7e0182bc"}, + {file = "types_protobuf-4.24.0.20240311-py3-none-any.whl", hash = "sha256:8e039486df058141cb221ab99f88c5878c08cca4376db1d84f63279860aa09cd"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20240311" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-PyYAML-6.0.12.20240311.tar.gz", hash = "sha256:a9e0f0f88dc835739b0c1ca51ee90d04ca2a897a71af79de9aec5f38cb0a5342"}, + {file = "types_PyYAML-6.0.12.20240311-py3-none-any.whl", hash = "sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6"}, +] + +[[package]] +name = "typing-extensions" +version = "4.10.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +optional = false +python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "c526d278605794b7506e43292eb0d58b0bbc27383e7d59fdd19ae7fca4669019" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6dc97c43 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "semantic-model-generator" +version = "0.1.0" +description = "Generate a Semantic Model from your Snowflake tables" +authors = ["Jonathan Hilgart ", "Nipun Sehrawar ", "Renee Huang ", "Nicole Limtiaco "] +license = "Apache Software License; BSD License" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +pandas = "^2.2.1" +loguru = "^0.7.2" +snowflake-connector-python = {extras = ["pandas"], version = "^3.7.1"} +protobuf = "^5.26.1" +pydantic = "^2.6.4" +PyYAML = "^6.0.1" +"ruamel.yaml" = "^0.18.6" +jsonargparse = "^4.27.7" +tqdm = "^4.66.2" + +[tool.poetry.group.dev.dependencies] +mypy = "^1.9.0" +black = "^24.3.0" +isort = "^5.13.2" +flake8 = "^7.0.0" +pytest = "^8.1.1" +types-pyyaml = "^6.0.12.20240311" +types-protobuf = "^4.24.0.20240311" +pip-licenses = "^4.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/semantic_model_generator/__init__.py b/semantic_model_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic_model_generator/data_processing/__init__.py b/semantic_model_generator/data_processing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic_model_generator/data_processing/data_types.py b/semantic_model_generator/data_processing/data_types.py new file mode 100644 index 00000000..111e96d5 --- /dev/null +++ b/semantic_model_generator/data_processing/data_types.py @@ -0,0 +1,47 @@ +from typing import Any, List, Optional + +from pydantic.dataclasses import dataclass + + +@dataclass +class FQNParts: + database: str + schema: str + table: str + + def __post_init__(self: Any) -> None: + """Uppercase table name""" + self.table = self.table.upper() + + +@dataclass +class Column: + id_: int + column_name: str + column_type: str + values: list[str] | None = None + comment: Optional[str] = ( + None # comment field's to save the column comment user specified on the column + ) + + def __post_init__(self: Any) -> None: + """ + Update column_type to cleaned up version, eg. NUMBER(38,0) -> NUMBER + """ + + self.column_type = self.column_type.split("(")[0].strip().upper() + + +@dataclass +class Table: + id_: int + name: str + columns: List[Column] + comment: Optional[str] = ( + None # comment field's to save the table comment user specified on the table + ) + + def __post_init__(self: Any) -> None: + for col in self.columns: + if col.column_name == "": + raise ValueError("column name in table must be nonempty") diff --git a/semantic_model_generator/data_processing/proto_utils.py b/semantic_model_generator/data_processing/proto_utils.py new file mode 100644 index 00000000..f33a8993 --- /dev/null +++ b/semantic_model_generator/data_processing/proto_utils.py @@ -0,0 +1,37 @@ +import io +import json +from typing import TypeVar + +import ruamel.yaml +from google.protobuf import json_format +from google.protobuf.message import Message + +ProtoMsg = TypeVar("ProtoMsg", bound=Message) + + +def proto_to_yaml(message: ProtoMsg) -> str: + """Serializes the input proto into a yaml message. + + Args: + message: Protobuf message to be serialized. + + Returns: + The serialized yaml string, or None if an error occurs. + """ + try: + json_data = json.loads( + json_format.MessageToJson(message, preserving_proto_field_name=True) + ) + + # Using ruamel.yaml package to preserve message order. + yaml = ruamel.yaml.YAML() + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.preserve_quotes = True + + with io.StringIO() as stream: + yaml.dump(json_data, stream) + yaml_str = stream.getvalue() + assert isinstance(yaml_str, str) + return yaml_str + except Exception as e: + raise ValueError(f"Failed to convert protobuf message to YAML: {e}") diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py new file mode 100644 index 00000000..7d1fe78a --- /dev/null +++ b/semantic_model_generator/main.py @@ -0,0 +1,288 @@ +import string +from datetime import datetime + +import jsonargparse +from loguru import logger + +from semantic_model_generator.data_processing import data_types, proto_utils +from semantic_model_generator.protos import semantic_model_pb2 +from semantic_model_generator.snowflake_utils.snowflake_connector import ( + DIMENSION_DATATYPE_COMMON_NAME, + MEASURE_DATATYPE_COMMON_NAME, + TIME_MEASURE_DATATYPE_COMMON_NAME, + SnowflakeConnector, + get_table_representation, + get_valid_schemas_tables_columns_df, +) +from semantic_model_generator.snowflake_utils.utils import create_fqn_table + +_PLACEHOLDER_COMMENT = " " +_FILL_OUT_TOKEN = " # " + + +def _expr_to_name(expr: str) -> str: + return expr.translate( + expr.maketrans(string.punctuation, " " * len(string.punctuation)) + ).title() + + +def _get_placeholder_filter() -> list[semantic_model_pb2.NamedFilter]: + return [ + semantic_model_pb2.NamedFilter( + name=_PLACEHOLDER_COMMENT, + synonyms=[_PLACEHOLDER_COMMENT], + description=_PLACEHOLDER_COMMENT, + expr=_PLACEHOLDER_COMMENT, + ) + ] + + +def _raw_table_to_semantic_context_table( + database: str, schema: str, raw_table: data_types.Table +) -> semantic_model_pb2.Table: + """ + Converts a raw table representation to a semantic model table in protobuf format. + + Args: + database (str): The name of the database containing the table. + schema (str): The name of the schema containing the table. + raw_table (data_types.Table): The raw table object to be transformed. + + Returns: + semantic_model_pb2.Table: A protobuf representation of the semantic table. + + This function categorizes table columns into TimeDimensions, Dimensions, or Measures based on their data type, + populates them with sample values, and sets placeholders for descriptions and filters. + """ + + # For each columns, decide if it is a TimeDimension, Measure, or Dimension column. + # For now, we decide this based on datatype. + # Any time datatype, is TimeDimension. + # Any varchar/text is Dimension. + # Any numerical column is Measure. + + time_dimensions = [] + dimensions = [] + measures = [] + + for col in raw_table.columns: + + if col.column_type == TIME_MEASURE_DATATYPE_COMMON_NAME: + time_dimensions.append( + semantic_model_pb2.TimeDimension( + name=_expr_to_name(col.column_name), + expr=col.column_name, + data_type=col.column_type, + sample_values=col.values, + synonyms=[_PLACEHOLDER_COMMENT], + description=_PLACEHOLDER_COMMENT, + ) + ) + + elif col.column_type == DIMENSION_DATATYPE_COMMON_NAME: + dimensions.append( + semantic_model_pb2.Dimension( + name=_expr_to_name(col.column_name), + expr=col.column_name, + data_type=col.column_type, + sample_values=col.values, + synonyms=[_PLACEHOLDER_COMMENT], + description=_PLACEHOLDER_COMMENT, + ) + ) + + elif col.column_type == MEASURE_DATATYPE_COMMON_NAME: + measures.append( + semantic_model_pb2.Measure( + name=_expr_to_name(col.column_name), + expr=col.column_name, + data_type=col.column_type, + sample_values=col.values, + synonyms=[_PLACEHOLDER_COMMENT], + description=_PLACEHOLDER_COMMENT, + ) + ) + + return semantic_model_pb2.Table( + name=_expr_to_name(raw_table.name), + base_table=semantic_model_pb2.FullyQualifiedTable( + database=database, schema=schema, table=raw_table.name + ), + # For fields we can not automatically infer, leave a comment for the user to fill out. + description=_PLACEHOLDER_COMMENT, + filters=_get_placeholder_filter(), + dimensions=dimensions, + time_dimensions=time_dimensions, + measures=measures, + ) + + +def raw_schema_to_semantic_context( + fqn_tables: list[str], snowflake_account: str +) -> tuple[semantic_model_pb2.SemanticModel, str]: + """ + Converts a list of fully qualified Snowflake table names into a semantic model. + + Parameters: + - fqn_tables (list[str]): Fully qualified table names to include in the semantic model. + - snowflake_account (str): Snowflake account identifier. + + Returns: + - tuple: A tuple containing the semantic model (semantic_model_pb2.SemanticModel) and the model name (str). + + This function fetches metadata for the specified tables, performs schema validation, extracts key information, + enriches metadata from the Snowflake database, and constructs a semantic model in protobuf format. + It handles different databases and schemas within the same account by creating unique Snowflake connections as needed. + + Raises: + - AssertionError: If no valid tables are found in the specified schema. + """ + connector = SnowflakeConnector( + account_name=snowflake_account, + ndv_per_column=3, # number of sample values to pull per column. + max_workers=1, + ) + # For FQN tables, create a new snowflake connection per table in case the db/schema is different. + table_objects = [] + unique_database_schema: set[str] = set() + for table in fqn_tables: + # Verify this is a valid FQN table. For now, we check that the table follows the following format. + # {database}.{schema}.{table} + fqn_table = create_fqn_table(table) + fqn_databse_schema = f"{fqn_table.database}_{fqn_table.schema}" + if fqn_databse_schema not in unique_database_schema: + unique_database_schema.add(fqn_databse_schema) + + with connector.connect( + db_name=fqn_table.database, schema_name=fqn_table.schema + ) as conn: + logger.info(f"Pulling column information from {fqn_table}") + valid_schemas_tables_columns_df = get_valid_schemas_tables_columns_df( + conn=conn, table_schema=fqn_table.schema, table_names=[fqn_table.table] + ) + assert not valid_schemas_tables_columns_df.empty + + # get the valid columns for this table. + valid_columns_df_this_table = valid_schemas_tables_columns_df[ + valid_schemas_tables_columns_df["TABLE_NAME"] == fqn_table.table + ] + + raw_table = get_table_representation( + conn=conn, + schema_name=fqn_table.schema, + table_name=fqn_table.table, + table_index=0, + ndv_per_column=3, + columns_df=valid_columns_df_this_table, + max_workers=1, + ) + + table_object = _raw_table_to_semantic_context_table( + database=fqn_table.database, + schema=fqn_table.schema, + raw_table=raw_table, + ) + table_objects.append(table_object) + semantic_model_name = "_".join(unique_database_schema) + context = semantic_model_pb2.SemanticModel( + name=_expr_to_name(semantic_model_name), tables=table_objects + ) + return context, semantic_model_name + + +def append_comment_to_placeholders(yaml_str: str) -> str: + """ + Finds all instances of a specified placeholder in a YAML string and appends a given text to these placeholders. + This is the homework to fill out after your yaml is generated. + + Args: + - yaml_str (str): The YAML string to process. + + Returns: + - str: The modified YAML string with appended text to placeholders. + """ + updated_yaml = [] + # Split the string into lines to process each line individually + lines = yaml_str.split("\n") + + for line in lines: + # Check if the placeholder is in the current line. + # Strip the last quote to match. + if line.rstrip("'").endswith(_PLACEHOLDER_COMMENT): + # Replace the _PLACEHOLDER_COMMENT with itself plus the append_text + updated_line = line + _FILL_OUT_TOKEN + updated_yaml.append(updated_line) + else: + updated_yaml.append(line) + + # Join the lines back together into a single string + return "\n".join(updated_yaml) + + +def generate_base_semantic_context_from_snowflake( + fqn_tables: list[str], + snowflake_account: str, + output_yaml_path: str | None = None, +) -> None: + """ + Generates a base semantic context from specified Snowflake tables and exports it to a YAML file. + + Args: + fqn_tables: Fully qualified names of Snowflake tables to include in the semantic context. + snowflake_account: Identifier of the Snowflake account. + output_yaml_path: Path for the output YAML file. If None, defaults to 'semantic_model_generator/output_models/YYYYMMDDHHMMSS_.yaml'. + + Returns: + None. Writes the semantic context to a YAML file. + """ + context, semantic_model_name = raw_schema_to_semantic_context( + fqn_tables=fqn_tables, + snowflake_account=snowflake_account, + ) + yaml_str = proto_utils.proto_to_yaml(context) + # Once we have the yaml, update to include to # tokens. + yaml_str = append_comment_to_placeholders(yaml_str) + if output_yaml_path: + write_path = output_yaml_path + else: + current_datetime = datetime.now() + + # Format the current date and time as "YYYY-MM-DD" + formatted_datetime = current_datetime.strftime("%Y%m%d%H%M%S") + write_path = f"semantic_model_generator/output_models/{formatted_datetime}_{semantic_model_name}.yaml" + with open(write_path, "w") as f: + f.write(yaml_str) + return None + + +if __name__ == "__main__": + parser = jsonargparse.ArgumentParser( + description="CLI tool to generate semantic context models from Snowflake schemas." + ) + + parser.add_argument( + "--fqn_tables", + type=list, + required=True, + help="The list of fully qualified table names all following the format {database_name}.{schema_name}{table_name}", + ) + parser.add_argument( + "--snowflake_account", + type=str, + required=True, + help="Your Snowflake account ID.", + ) + parser.add_argument( + "--output_yaml_path", + type=str, + required=False, + help="Custom path to save the YAML. Optional.", + ) + + args = parser.parse_args() + + generate_base_semantic_context_from_snowflake( + fqn_tables=args.fqn_tables, + snowflake_account=args.snowflake_account, + output_yaml_path=args.output_yaml_path, + ) diff --git a/semantic_model_generator/protos/semantic_model.proto b/semantic_model_generator/protos/semantic_model.proto new file mode 100644 index 00000000..3449d5f3 --- /dev/null +++ b/semantic_model_generator/protos/semantic_model.proto @@ -0,0 +1,192 @@ +// If you make changes to this file, you'll need to run protoc to updated the +// generated files by running the following command: +// +// protoc -I=semantic_model_generator/protos/ --python_out=semantic_model_generator/protos/ --pyi_out=semantic_model_generator/protos/ semantic_model_generator/protos/semantic_model.proto + +syntax = "proto3"; + +package semantic_model_generator; + +// AggregationType defines a list of various aggregations. +enum AggregationType { + aggregation_type_unknown = 0; + sum = 1; + avg = 2; + median = 7; + min = 3; + max = 4; + count = 5; + count_distinct = 6; +} + +// ColumnKind defines various kinds of columns, mainly categorized into +// dimensions and measures. +enum ColumnKind { + column_kind_unknown = 0; + // A column containing categorical values such as names, countries, dates. + dimension = 1; + // A column containing numerical values such as revenue, impressions, salary. + measure = 2; + // A column containing date/time data. + time_dimension = 3; +} + +// Column is analogous to a database column and defines various semantic properties +// of a column. A column can either simply be a column in the base database schema +// or it can be an arbitrary expression over the base schema, e.g. +// `base_column1 + base_column2`. +message Column { + // A descriptive name for this column. + string name = 1; + // A list of other terms/phrases used to refer to this column. + repeated string synonyms = 2; + // A brief description about this column, including things like what data this + // column has. + string description = 3; + // The SQL expression for this column. Could simply be a base table column name + // or an arbitrary SQL expression over one or more columns of the base table. + string expr = 4; + // The data type of this column. + string data_type = 5; + // The kind of this column - dimension or measure. + ColumnKind kind = 6; + // If true, assume that this column has unique values. + bool unique = 7; + // If no aggregation is specified, then this is the default aggregation + // applied to this column in context of a grouping. + AggregationType default_aggregation = 8; + // Sample values of this column. + repeated string sample_values = 9; +} + +// Dimension columns contain categorical values (e.g. state, user_type, platform). +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() +// of snowpilot/semantic_context/utils/utils.py. +message Dimension { + // A descriptive name for this dimension. + string name = 1; + // A list of other terms/phrases used to refer to this dimension. + repeated string synonyms = 2; + // A brief description about this dimension, including things like + // what data this dimension has. + string description = 3; + // The SQL expression defining this dimension. Could simply be a physical column name + // or an arbitrary SQL expression over one or more columns of the physical table. + string expr = 4; + // The data type of this dimension. + string data_type = 5; + // If true, assume that this dimension has unique values. + bool unique = 6; + // Sample values of this column. + repeated string sample_values = 7; +} + +// Time dimension columns contain time values (e.g. sale_date, created_at, year). +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() +// of snowpilot/semantic_context/utils/utils.py. +message TimeDimension { + // A descriptive name for this time dimension. + string name = 1; + // A list of other terms/phrases used to refer to this time dimension. + repeated string synonyms = 2; + // A brief description about this time dimension, including things like + // what data it has, the timezone of values, etc. + string description = 3; + // The SQL expression defining this time dimension. Could simply be a physical + // column name or an arbitrary SQL expression over one or more columns of the + // physical table. + string expr = 4; + // The data type of this time dimension. + string data_type = 5; + // If true, assume that this time dimension has unique values. + bool unique = 6; + // Sample values of this time dimension. + repeated string sample_values = 7; +} + +// Measure columns contain numerical values (e.g. revenue, impressions, salary). +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() +// of snowpilot/semantic_context/utils/utils.py. +message Measure { + // A descriptive name for this measure. + string name = 1; + // A list of other terms/phrases used to refer to this measure. + repeated string synonyms = 2; + // A brief description about this measure, including things like what data + // it has. + string description = 3; + // The SQL expression defining this measure. Could simply be a physical column + // name or an arbitrary SQL expression over one or more physical columns of + // the underlying physical table. + string expr = 4; + // The data type of this measure. + string data_type = 5; + // If no aggregation is specified, then this is the default aggregation + // applied to this measure in contxt of a grouping. + AggregationType default_aggregation = 6; + // Sample values of this measure. + repeated string sample_values = 7; +} + +// Filter represents a named SQL expression that's used for filtering. +message NamedFilter { + // A descriptive name for this filter. + string name = 1; + // A list of other term/phrases used to refer to this column. + repeated string synonyms = 2; + // A brief description about this column, including details of what this filter + // is typically used for. + string description = 3; + // The SQL expression of this filter. + string expr = 4; +} + +// FullyQualifiedTable is used to represent three part table names - +// (database, schema, table). +message FullyQualifiedTable { + string database = 1; + string schema = 2; + string table = 3; +} + +// Table is analogous to a database table and provides a simple view over an +// existing database table. A table can leave out some columns from the base +// table and/or introduce new derived columns. +message Table { + // A descriptive name for this table. + string name = 1; + // A list of other term/phrases used to refer to this table. + repeated string synonyms = 2; + // A brief description of this table, including details of what kinds of + // analysis is it typically used for. + string description = 3; + // Fully qualified name of the underlying base table. + FullyQualifiedTable base_table = 4; + + // We allow two formats for specifying logical columns of a table: + // 1. As a list of columns. + // 2. As three separate list of dimensions, time dimensions, and measures. + // For the external facing yaml specification, we have chosen to go with (2). + // However, for the time being we'll support both (1) and (2) and continue + // using (1) as the internal representation. + repeated Column columns = 5; + repeated Dimension dimensions = 9; + repeated TimeDimension time_dimensions = 10; + repeated Measure measures = 11; + + + // Predefined filters on this table, if any. + repeated NamedFilter filters = 8; + // NEXT_TAG: 12. +} + +// The semantic context relevant to generating SQL for answering a data question. +message SemanticModel { + // A descriptive name of the project. + string name = 1; + // A brief description of this project, including details of what kind of + // analysis does this project enable. + string description = 2; + // List of tables in this project. + repeated Table tables = 3; +} \ No newline at end of file diff --git a/semantic_model_generator/protos/semantic_model_pb2.py b/semantic_model_generator/protos/semantic_model_pb2.py new file mode 100644 index 00000000..61f92bf8 --- /dev/null +++ b/semantic_model_generator/protos/semantic_model_pb2.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: semantic_model.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14semantic_model.proto\x12\x18semantic_model_generator\"\x81\x02\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x32\n\x04kind\x18\x06 \x01(\x0e\x32$.semantic_model_generator.ColumnKind\x12\x0e\n\x06unique\x18\x07 \x01(\x08\x12\x46\n\x13\x64\x65\x66\x61ult_aggregation\x18\x08 \x01(\x0e\x32).semantic_model_generator.AggregationType\x12\x15\n\rsample_values\x18\t \x03(\t\"\x88\x01\n\tDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\x8c\x01\n\rTimeDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\xbe\x01\n\x07Measure\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x46\n\x13\x64\x65\x66\x61ult_aggregation\x18\x06 \x01(\x0e\x32).semantic_model_generator.AggregationType\x12\x15\n\rsample_values\x18\x07 \x03(\t\"P\n\x0bNamedFilter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\"F\n\x13\x46ullyQualifiedTable\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\x9a\x03\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x41\n\nbase_table\x18\x04 \x01(\x0b\x32-.semantic_model_generator.FullyQualifiedTable\x12\x31\n\x07\x63olumns\x18\x05 \x03(\x0b\x32 .semantic_model_generator.Column\x12\x37\n\ndimensions\x18\t \x03(\x0b\x32#.semantic_model_generator.Dimension\x12@\n\x0ftime_dimensions\x18\n \x03(\x0b\x32\'.semantic_model_generator.TimeDimension\x12\x33\n\x08measures\x18\x0b \x03(\x0b\x32!.semantic_model_generator.Measure\x12\x36\n\x07\x66ilters\x18\x08 \x03(\x0b\x32%.semantic_model_generator.NamedFilter\"c\n\rSemanticModel\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.semantic_model_generator.Table*~\n\x0f\x41ggregationType\x12\x1c\n\x18\x61ggregation_type_unknown\x10\x00\x12\x07\n\x03sum\x10\x01\x12\x07\n\x03\x61vg\x10\x02\x12\n\n\x06median\x10\x07\x12\x07\n\x03min\x10\x03\x12\x07\n\x03max\x10\x04\x12\t\n\x05\x63ount\x10\x05\x12\x12\n\x0e\x63ount_distinct\x10\x06*U\n\nColumnKind\x12\x17\n\x13\x63olumn_kind_unknown\x10\x00\x12\r\n\tdimension\x10\x01\x12\x0b\n\x07measure\x10\x02\x12\x12\n\x0etime_dimension\x10\x03\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'semantic_model_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_AGGREGATIONTYPE']._serialized_start=1453 + _globals['_AGGREGATIONTYPE']._serialized_end=1579 + _globals['_COLUMNKIND']._serialized_start=1581 + _globals['_COLUMNKIND']._serialized_end=1666 + _globals['_COLUMN']._serialized_start=51 + _globals['_COLUMN']._serialized_end=308 + _globals['_DIMENSION']._serialized_start=311 + _globals['_DIMENSION']._serialized_end=447 + _globals['_TIMEDIMENSION']._serialized_start=450 + _globals['_TIMEDIMENSION']._serialized_end=590 + _globals['_MEASURE']._serialized_start=593 + _globals['_MEASURE']._serialized_end=783 + _globals['_NAMEDFILTER']._serialized_start=785 + _globals['_NAMEDFILTER']._serialized_end=865 + _globals['_FULLYQUALIFIEDTABLE']._serialized_start=867 + _globals['_FULLYQUALIFIEDTABLE']._serialized_end=937 + _globals['_TABLE']._serialized_start=940 + _globals['_TABLE']._serialized_end=1350 + _globals['_SEMANTICMODEL']._serialized_start=1352 + _globals['_SEMANTICMODEL']._serialized_end=1451 +# @@protoc_insertion_point(module_scope) diff --git a/semantic_model_generator/protos/semantic_model_pb2.pyi b/semantic_model_generator/protos/semantic_model_pb2.pyi new file mode 100644 index 00000000..c0382875 --- /dev/null +++ b/semantic_model_generator/protos/semantic_model_pb2.pyi @@ -0,0 +1,167 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class AggregationType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + aggregation_type_unknown: _ClassVar[AggregationType] + sum: _ClassVar[AggregationType] + avg: _ClassVar[AggregationType] + median: _ClassVar[AggregationType] + min: _ClassVar[AggregationType] + max: _ClassVar[AggregationType] + count: _ClassVar[AggregationType] + count_distinct: _ClassVar[AggregationType] + +class ColumnKind(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + column_kind_unknown: _ClassVar[ColumnKind] + dimension: _ClassVar[ColumnKind] + measure: _ClassVar[ColumnKind] + time_dimension: _ClassVar[ColumnKind] +aggregation_type_unknown: AggregationType +sum: AggregationType +avg: AggregationType +median: AggregationType +min: AggregationType +max: AggregationType +count: AggregationType +count_distinct: AggregationType +column_kind_unknown: ColumnKind +dimension: ColumnKind +measure: ColumnKind +time_dimension: ColumnKind + +class Column(_message.Message): + __slots__ = ("name", "synonyms", "description", "expr", "data_type", "kind", "unique", "default_aggregation", "sample_values") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + EXPR_FIELD_NUMBER: _ClassVar[int] + DATA_TYPE_FIELD_NUMBER: _ClassVar[int] + KIND_FIELD_NUMBER: _ClassVar[int] + UNIQUE_FIELD_NUMBER: _ClassVar[int] + DEFAULT_AGGREGATION_FIELD_NUMBER: _ClassVar[int] + SAMPLE_VALUES_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + expr: str + data_type: str + kind: ColumnKind + unique: bool + default_aggregation: AggregationType + sample_values: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ..., data_type: _Optional[str] = ..., kind: _Optional[_Union[ColumnKind, str]] = ..., unique: bool = ..., default_aggregation: _Optional[_Union[AggregationType, str]] = ..., sample_values: _Optional[_Iterable[str]] = ...) -> None: ... + +class Dimension(_message.Message): + __slots__ = ("name", "synonyms", "description", "expr", "data_type", "unique", "sample_values") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + EXPR_FIELD_NUMBER: _ClassVar[int] + DATA_TYPE_FIELD_NUMBER: _ClassVar[int] + UNIQUE_FIELD_NUMBER: _ClassVar[int] + SAMPLE_VALUES_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + expr: str + data_type: str + unique: bool + sample_values: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ..., data_type: _Optional[str] = ..., unique: bool = ..., sample_values: _Optional[_Iterable[str]] = ...) -> None: ... + +class TimeDimension(_message.Message): + __slots__ = ("name", "synonyms", "description", "expr", "data_type", "unique", "sample_values") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + EXPR_FIELD_NUMBER: _ClassVar[int] + DATA_TYPE_FIELD_NUMBER: _ClassVar[int] + UNIQUE_FIELD_NUMBER: _ClassVar[int] + SAMPLE_VALUES_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + expr: str + data_type: str + unique: bool + sample_values: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ..., data_type: _Optional[str] = ..., unique: bool = ..., sample_values: _Optional[_Iterable[str]] = ...) -> None: ... + +class Measure(_message.Message): + __slots__ = ("name", "synonyms", "description", "expr", "data_type", "default_aggregation", "sample_values") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + EXPR_FIELD_NUMBER: _ClassVar[int] + DATA_TYPE_FIELD_NUMBER: _ClassVar[int] + DEFAULT_AGGREGATION_FIELD_NUMBER: _ClassVar[int] + SAMPLE_VALUES_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + expr: str + data_type: str + default_aggregation: AggregationType + sample_values: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ..., data_type: _Optional[str] = ..., default_aggregation: _Optional[_Union[AggregationType, str]] = ..., sample_values: _Optional[_Iterable[str]] = ...) -> None: ... + +class NamedFilter(_message.Message): + __slots__ = ("name", "synonyms", "description", "expr") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + EXPR_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + expr: str + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ...) -> None: ... + +class FullyQualifiedTable(_message.Message): + __slots__ = ("database", "schema", "table") + DATABASE_FIELD_NUMBER: _ClassVar[int] + SCHEMA_FIELD_NUMBER: _ClassVar[int] + TABLE_FIELD_NUMBER: _ClassVar[int] + database: str + schema: str + table: str + def __init__(self, database: _Optional[str] = ..., schema: _Optional[str] = ..., table: _Optional[str] = ...) -> None: ... + +class Table(_message.Message): + __slots__ = ("name", "synonyms", "description", "base_table", "columns", "dimensions", "time_dimensions", "measures", "filters") + NAME_FIELD_NUMBER: _ClassVar[int] + SYNONYMS_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + BASE_TABLE_FIELD_NUMBER: _ClassVar[int] + COLUMNS_FIELD_NUMBER: _ClassVar[int] + DIMENSIONS_FIELD_NUMBER: _ClassVar[int] + TIME_DIMENSIONS_FIELD_NUMBER: _ClassVar[int] + MEASURES_FIELD_NUMBER: _ClassVar[int] + FILTERS_FIELD_NUMBER: _ClassVar[int] + name: str + synonyms: _containers.RepeatedScalarFieldContainer[str] + description: str + base_table: FullyQualifiedTable + columns: _containers.RepeatedCompositeFieldContainer[Column] + dimensions: _containers.RepeatedCompositeFieldContainer[Dimension] + time_dimensions: _containers.RepeatedCompositeFieldContainer[TimeDimension] + measures: _containers.RepeatedCompositeFieldContainer[Measure] + filters: _containers.RepeatedCompositeFieldContainer[NamedFilter] + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., base_table: _Optional[_Union[FullyQualifiedTable, _Mapping]] = ..., columns: _Optional[_Iterable[_Union[Column, _Mapping]]] = ..., dimensions: _Optional[_Iterable[_Union[Dimension, _Mapping]]] = ..., time_dimensions: _Optional[_Iterable[_Union[TimeDimension, _Mapping]]] = ..., measures: _Optional[_Iterable[_Union[Measure, _Mapping]]] = ..., filters: _Optional[_Iterable[_Union[NamedFilter, _Mapping]]] = ...) -> None: ... + +class SemanticModel(_message.Message): + __slots__ = ("name", "description", "tables") + NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + TABLES_FIELD_NUMBER: _ClassVar[int] + name: str + description: str + tables: _containers.RepeatedCompositeFieldContainer[Table] + def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., tables: _Optional[_Iterable[_Union[Table, _Mapping]]] = ...) -> None: ... diff --git a/semantic_model_generator/snowflake_utils/env_vars.py b/semantic_model_generator/snowflake_utils/env_vars.py new file mode 100644 index 00000000..a71339bd --- /dev/null +++ b/semantic_model_generator/snowflake_utils/env_vars.py @@ -0,0 +1,8 @@ +import os + +DEFAULT_SESSION_TIMEOUT_SEC = int(os.environ.get("SNOWFLAKE_SESSION_TIMEOUT_SEC", 120)) +SNOWFLAKE_ROLE = os.getenv("SNOWFLAKE_ROLE") +SNOWFLAKE_WAREHOUSE = os.getenv("SNOWFLAKE_WAREHOUSE") +SNOWFLAKE_USER = os.getenv("SNOWFLAKE_USER") +SNOWFLAKE_PASSWORD = os.getenv("SNOWFLAKE_PASSWORD") +SNOWFLAKE_HOST = os.getenv("SNOWFLAKE_HOST") diff --git a/semantic_model_generator/snowflake_utils/snowflake_connector.py b/semantic_model_generator/snowflake_utils/snowflake_connector.py new file mode 100644 index 00000000..57d5653d --- /dev/null +++ b/semantic_model_generator/snowflake_utils/snowflake_connector.py @@ -0,0 +1,374 @@ +import concurrent.futures +from collections import defaultdict +from contextlib import contextmanager +from typing import Any, Generator, Optional, TypeVar + +import pandas as pd +from loguru import logger +from snowflake.connector import DictCursor +from snowflake.connector.connection import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from semantic_model_generator.data_processing.data_types import Column, Table +from semantic_model_generator.snowflake_utils import env_vars +from semantic_model_generator.snowflake_utils.utils import snowflake_connection + +ConnectionType = TypeVar("ConnectionType") +# This is the raw column name from snowflake information schema or desc table +_COMMENT_COL = "COMMENT" +_COLUMN_NAME_COL = "COLUMN_NAME" +_DATATYPE_COL = "DATA_TYPE" +_TABLE_SCHEMA_COL = "TABLE_SCHEMA" +_TABLE_NAME_COL = "TABLE_NAME" +# Below are the renamed column names when we fetch into dataframe, to differentiate between table/column comments +_COLUMN_COMMENT_ALIAS = "COLUMN_COMMENT" +_TABLE_COMMENT_COL = "TABLE_COMMENT" + +# https://docs.snowflake.com/en/sql-reference/data-types-datetime +_TIME_MEASURE_DATATYPES = [ + "DATE", + "DATETIME", + "TIMESTAMP_LTZ", + "TIMESTAMP_NTZ", + "TIMESTAMP_TZ", + "TIMESTAMP", + "TIME", +] +TIME_MEASURE_DATATYPE_COMMON_NAME = "TIMESTAMP" +# https://docs.snowflake.com/en/sql-reference/data-types-text +_DIMENSION_DATATYPES = [ + "VARCHAR", + "CHAR", + "CHARACTER", + "NCHAR", + "STRING", + "TEXT", + "NVARCHAR", + "NVARCHAR2", + "CHAR VARYING", + " NCHAR VARYING", + "BINARY", + "VARBINARY", +] +DIMENSION_DATATYPE_COMMON_NAME = "TEXT" +# https://docs.snowflake.com/en/sql-reference/data-types-numeric +_MEASURE_DATATYPES = [ + "NUMBER", + "DECIMAL", + "DEC", + "NUMERIC", + "INT", + "INTEGER", + "BIGINT", + "SMALLINT", + "TINYINT", + "BYTEINT", + "FLOAT", + "FLOAT4", + "FLOAT8", + "DOUBLE", + "DOUBLE PRECISION", + "REAL", +] +MEASURE_DATATYPE_COMMON_NAME = "NUMBER" + + +_QUERY_TAG = "SEMANTIC_MODEL_GENERATOR" + + +def determine_col_datatype(snowflake_datatype: str) -> str: + # Takes in a snowflake datatype and returns the common datatype + if snowflake_datatype in _TIME_MEASURE_DATATYPES: + return TIME_MEASURE_DATATYPE_COMMON_NAME + elif snowflake_datatype in _DIMENSION_DATATYPES: + return DIMENSION_DATATYPE_COMMON_NAME + elif snowflake_datatype in _MEASURE_DATATYPES: + return MEASURE_DATATYPE_COMMON_NAME + else: + raise ValueError( + f"Input snowflake datatype {snowflake_datatype} not supported yet. Please add to either _TIME_MEASURE_DATATYPES, _DIMENSION_DATATYPES, or _MEASURE_DATATYPES" + ) + + +def get_table_representation( + conn: SnowflakeConnection, + schema_name: str, + table_name: str, + table_index: int, + ndv_per_column: int, + columns_df: pd.DataFrame, + max_workers: int, +) -> Table: + table_comment = columns_df[_TABLE_COMMENT_COL].iloc[0] + + def _get_col(col_index: int, column_row: pd.Series) -> Column: + return _get_column_representation( + conn=conn, + schema_name=schema_name, + table_name=table_name, + column_name=column_row[_COLUMN_NAME_COL], + column_comment=column_row[_COLUMN_COMMENT_ALIAS], + column_index=col_index, + column_datatype=column_row[_DATATYPE_COL], + ndv=ndv_per_column, + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_col_index = { + executor.submit(_get_col, col_index, column_row): col_index + for col_index, (_, column_row) in enumerate(columns_df.iterrows()) + } + index_and_column = [] + for future in concurrent.futures.as_completed(future_to_col_index): + col_index = future_to_col_index[future] + column = future.result() + index_and_column.append((col_index, column)) + columns = [c for _, c in sorted(index_and_column, key=lambda x: x[0])] + + return Table( + id_=table_index, name=table_name, comment=table_comment, columns=columns + ) + + +def _get_column_representation( + conn: SnowflakeConnection, + schema_name: str, + table_name: str, + column_name: str, + column_comment: str, + column_index: int, + column_datatype: str, + ndv: int, +) -> Column: + + column_values = None + if ndv > 0: + # Pull sample values. + try: + cursor = conn.cursor(DictCursor) + assert cursor is not None, "Cursor is unexpectedly None" + cursor_execute = cursor.execute( + f'select distinct "{column_name}" from "{schema_name}"."{table_name}" limit {ndv}' + ) + assert cursor_execute is not None, "cursor_execute should not be none " + res = cursor_execute.fetchall() + # Cast all values to string to ensure the list is json serializable. + # A better solution would be to identify the possible types that are not + # json serializable (e.g. datetime objects) and apply the appropriate casting + # in just those cases. + if len(res) > 0: + if isinstance(res[0], dict): + col_key = [k for k in res[0].keys()][0] + column_values = [str(r[col_key]) for r in res] + else: + raise ValueError( + f"Expected the first item of res to be a dict. Instead passed {res}" + ) + except Exception as e: + logger.error(f"unable to get values: {e}") + + column = Column( + id_=column_index, + column_name=column_name, + comment=column_comment, + column_type=determine_col_datatype(column_datatype), + values=column_values, + ) + return column + + +def _fetch_valid_tables_and_views(conn: SnowflakeConnection) -> pd.DataFrame: + def _get_df(query: str) -> pd.DataFrame: + cursor = conn.cursor().execute(query) + assert cursor is not None, "cursor should not be none here." + df = pd.DataFrame( + cursor.fetchall(), columns=[c.name for c in cursor.description] + ) + return df[["name", "schema_name", "comment"]].rename( + columns=dict( + name=_TABLE_NAME_COL, + schema_name=_TABLE_SCHEMA_COL, + comment=_TABLE_COMMENT_COL, + ) + ) + + tables = _get_df("show tables in database") + views = _get_df("show views in database") + return pd.concat([tables, views], axis=0) + + +def get_valid_schemas_tables_columns_df( + conn: SnowflakeConnection, + table_schema: str | None = None, + table_names: list[str] | None = None, +) -> pd.DataFrame: + if table_names and not table_schema: + logger.warning( + "Provided table_name without table_schema, cannot filter to fetch the specific table" + ) + + where_clause = "" + if table_schema: + where_clause += f" where t.table_schema ilike '{table_schema}' " + if table_names: + table_names_str = ", ".join([f"'{t.lower()}'" for t in table_names]) + where_clause += f"AND LOWER(t.table_name) in ({table_names_str}) " + query = f"""select t.{_TABLE_SCHEMA_COL}, t.{_TABLE_NAME_COL}, c.{_COLUMN_NAME_COL}, c.{_DATATYPE_COL}, c.{_COMMENT_COL} as {_COLUMN_COMMENT_ALIAS} +from information_schema.tables as t +join information_schema.columns as c on t.table_schema = c.table_schema and t.table_name = c.table_name{where_clause} +order by 1, 2, c.ordinal_position""" + cursor_execute = conn.cursor().execute(query) + assert cursor_execute, "cursor_execute should not be None here" + schemas_tables_columns_df = cursor_execute.fetch_pandas_all() + + valid_tables_and_views_df = _fetch_valid_tables_and_views(conn=conn) + + valid_schemas_tables_columns_df = valid_tables_and_views_df.merge( + schemas_tables_columns_df, how="inner", on=(_TABLE_SCHEMA_COL, _TABLE_NAME_COL) + ) + return valid_schemas_tables_columns_df + + +class SnowflakeConnector: + def __init__( + self, + account_name: str, + ndv_per_column: int = 0, + max_workers: int = 1, + ): + self.account_name: str = account_name + self._ndv_per_column = ndv_per_column + self._max_workers = max_workers + + # Required env vars below + def _get_role(self) -> str: + role = env_vars.SNOWFLAKE_ROLE + if not role: + raise ValueError( + "You need to set an env var for the snowflake role. export SNOWFLAKE_ROLE=" + ) + return role + + def _get_user(self) -> str: + user = env_vars.SNOWFLAKE_USER + if not user: + raise ValueError( + "You need to set an env var for the snowflake user. export SNOWFLAKE_USER=" + ) + return user + + def _get_password(self) -> str: + password = env_vars.SNOWFLAKE_PASSWORD + if not password: + raise ValueError( + "You need to set an env var for the snowflake password. export SNOWFLAKE_PASSWORD=" + ) + return password + + def _get_warehouse(self) -> str: + warehouse = env_vars.SNOWFLAKE_WAREHOUSE + if not warehouse: + raise ValueError( + "You need to set an env var for the snowflake warehouse. export SNOWFLAKE_WAREHOUSE=" + ) + return warehouse + + def _get_host(self) -> str | None: + host = env_vars.SNOWFLAKE_HOST + if not host: + logger.info( + "No host set. Attempting to connect without. To set export SNOWFLAKE_HOST=" + ) + return host + + @contextmanager + def connect( + self, db_name: str, schema_name: Optional[str] = None + ) -> Generator[SnowflakeConnection, None, None]: + """Opens a connection to the database and optional schema. + + This function is a context manager for a connection that can be used to execute queries. + Example usage: + + with connector.connect(db_name="my_db", schema_name="my_schema") as conn: + connector.execute(conn=conn, query="select * from table") + + Args: + db_name: The name of the database to connect to. + schema_name: The name of the schema to connect to. Primarily needed for Snowflake databases. + """ + conn = None + try: + conn = self._open_connection(db_name, schema_name=schema_name) + yield conn + finally: + if conn is not None: + self._close_connection(connection=conn) + + def _open_connection( + self, db_name: str, schema_name: Optional[str] = None + ) -> SnowflakeConnection: + connection = snowflake_connection( + user=self._get_user(), + password=self._get_password(), + account=str(self.account_name), + role=self._get_role(), + warehouse=self._get_warehouse(), + host=self._get_host(), + ) + if db_name: + try: + connection.cursor().execute(f"USE DATABASE {db_name}") + except Exception as e: + raise ValueError( + f"Could not connect to database {db_name}. Does the database exist in {self.account_name}?" + ) from e + if schema_name: + try: + connection.cursor().execute(f"USE SCHEMA {schema_name}") + except Exception as e: + raise ValueError( + f"Could not connect to schema {schema_name}. Does the schema exist in the {db_name} database?" + ) from e + if _QUERY_TAG: + connection.cursor().execute(f"ALTER SESSION SET QUERY_TAG = '{_QUERY_TAG}'") + connection.cursor().execute( + f"ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = {env_vars.DEFAULT_SESSION_TIMEOUT_SEC}" + ) + return connection + + def _close_connection(self, connection: SnowflakeConnection) -> None: + connection.close() + + def execute( + self, + connection: SnowflakeConnection, + query: str, + ) -> dict[str, list[Any]]: + try: + if connection.warehouse is None: + warehouse = self._get_warehouse() + logger.debug( + f"There is no Warehouse assigned to Connection, setting it to config default ({warehouse})" + ) + connection.cursor().execute( + f'use warehouse {warehouse.replace("-", "_")}' + ) + cursor = connection.cursor(DictCursor) + logger.info(f"Executing query = {query}") + cursor_execute = cursor.execute(query) + assert cursor_execute, "cursor_execute should not be None here" + result = cursor_execute.fetchall() + except ProgrammingError as e: + raise ValueError(f"Query Error: {e}") + + out_dict = defaultdict(list) + for row in result: + if isinstance(row, dict): + for k, v in row.items(): + out_dict[k].append(v) + else: + raise ValueError( + f"Expected a dict for row object. Instead passed {row}" + ) + return out_dict diff --git a/semantic_model_generator/snowflake_utils/utils.py b/semantic_model_generator/snowflake_utils/utils.py new file mode 100644 index 00000000..a760c272 --- /dev/null +++ b/semantic_model_generator/snowflake_utils/utils.py @@ -0,0 +1,73 @@ +from typing import Dict + +from snowflake.connector import connect +from snowflake.connector.connection import SnowflakeConnection + +from semantic_model_generator.data_processing.data_types import FQNParts + + +def create_fqn_table(fqn_str: str) -> FQNParts: + if fqn_str.count(".") != 2: + raise ValueError( + "Expected to have a table fully qualified name following the {database}.{schema}.{table} format." + + f"Instead found {fqn_str}" + ) + database, schema, table = fqn_str.split(".") + return FQNParts(database=database, schema=schema, table=table) + + +def create_connection_parameters( + user: str, + password: str, + account: str, + host: str | None = None, + role: str | None = None, + warehouse: str | None = None, + database: str | None = None, + schema: str | None = None, + authenticator: str | None = None, +) -> Dict[str, str]: + connection_parameters: dict[str, str] = dict( + user=user, password=password, account=account + ) + if role: + connection_parameters["role"] = role + if warehouse: + connection_parameters["warehouse"] = warehouse + if database: + connection_parameters["database"] = database + if schema: + connection_parameters["schema"] = schema + if authenticator: + connection_parameters["authenticator"] = authenticator + if host: + connection_parameters["host"] = host + return connection_parameters + + +def _connection(connection_parameters: dict[str, str]) -> SnowflakeConnection: + # https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect + return connect(**connection_parameters) + + +def snowflake_connection( + user: str, + password: str, + account: str, + role: str, + warehouse: str, + host: str | None = None, +) -> SnowflakeConnection: + """ + Returns a Snowflake Connection to the specified account. + """ + return _connection( + create_connection_parameters( + user=user, + password=password, + host=host, + account=account, + role=role, + warehouse=warehouse, + ) + ) diff --git a/semantic_model_generator/tests/generate_semantic_model_test.py b/semantic_model_generator/tests/generate_semantic_model_test.py new file mode 100644 index 00000000..2a5a2d70 --- /dev/null +++ b/semantic_model_generator/tests/generate_semantic_model_test.py @@ -0,0 +1,280 @@ +from unittest.mock import MagicMock, mock_open, patch + +import pandas as pd +import pytest +import yaml + +from semantic_model_generator.data_processing import proto_utils +from semantic_model_generator.data_processing.data_types import Column, Table +from semantic_model_generator.main import ( + generate_base_semantic_context_from_snowflake, + raw_schema_to_semantic_context, +) +from semantic_model_generator.protos import semantic_model_pb2 +from semantic_model_generator.snowflake_utils.snowflake_connector import ( + SnowflakeConnector, +) + + +@pytest.fixture +def mock_snowflake_connection(): + """Fixture to mock the snowflake_connection function.""" + with patch( + "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" + ) as mock: + mock.return_value = MagicMock() + yield mock + + +_CONVERTED_TABLE_ALIAS = Table( + id_=0, + name="ALIAS", + columns=[ + Column( + id_=0, + column_name="ZIP_CODE", + column_type="TEXT", + values=None, + comment=None, + ), + Column( + id_=1, + column_name="AREA_CODE", + column_type="NUMBER", + values=None, + comment=None, + ), + Column( + id_=2, + column_name="BAD_ALIAS", + column_type="TIMESTAMP", + values=None, + comment=None, + ), + Column( + id_=3, + column_name="CBSA", + column_type="NUMBER", + values=None, + comment=None, + ), + ], + comment=None, +) + +_CONVERTED_TABLE_ZIP_CODE = Table( + id_=0, + name="PRODUCTS", + columns=[ + Column( + id_=0, + column_name="SKU", + column_type="NUMBER", + values=["1", "2", "3"], + comment=None, + ), + ], + comment=None, +) + + +@pytest.fixture +def mock_snowflake_connection_env(monkeypatch): + # Mock environment variable + monkeypatch.setenv("SNOWFLAKE_HOST", "test_host") + + # Use this fixture to also patch instance methods if needed + with patch.object( + SnowflakeConnector, "_get_user", return_value="test_user" + ), patch.object( + SnowflakeConnector, "_get_password", return_value="test_password" + ), patch.object( + SnowflakeConnector, "_get_role", return_value="test_role" + ), patch.object( + SnowflakeConnector, "_get_warehouse", return_value="test_warehouse" + ), patch.object( + SnowflakeConnector, "_get_host", return_value="test_host" + ): + yield + + +@pytest.fixture +def mock_dependencies(mock_snowflake_connection): + valid_schemas_tables_columns_df_alias = pd.DataFrame( + { + "TABLE_NAME": ["ALIAS"] * 4, + "COLUMN_NAME": ["ZIP_CODE", "AREA_CODE", "BAD_ALIAS", "CBSA"], + "DATA_TYPE": ["VARCHAR", "INTEGER", "DATETIME", "DECIMAL"], + } + ) + valid_schemas_tables_columns_df_zip_code = pd.DataFrame( + { + "TABLE_NAME": ["PRODUCTS"], + "COLUMN_NAME": ["SKU"], + "DATA_TYPE": ["NUMBER"], + } + ) + valid_schemas_tables_representations = [ + valid_schemas_tables_columns_df_alias, + valid_schemas_tables_columns_df_zip_code, + ] + table_representations = [ + _CONVERTED_TABLE_ALIAS, # Value returned on the first call. + _CONVERTED_TABLE_ZIP_CODE, # Value returned on the second call. + ] + + with patch( + "semantic_model_generator.main.get_valid_schemas_tables_columns_df", + side_effect=valid_schemas_tables_representations, + ), patch( + "semantic_model_generator.main.get_table_representation", + side_effect=table_representations, + ): + yield + + +def test_raw_schema_to_semantic_context( + mock_dependencies, mock_snowflake_connection, mock_snowflake_connection_env +): + want_yaml = "name: Test Db Schema Test\ntables:\n - name: Alias\n description: ' '\n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' '\n synonyms:\n - ' '\n description: ' '\n expr: ' '\n dimensions:\n - name: Zip Code\n synonyms:\n - ' '\n description: ' '\n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' '\n description: ' '\n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' '\n description: ' '\n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' '\n description: ' '\n expr: CBSA\n data_type: NUMBER\n" + + snowflake_account = "test_account" + fqn_tables = ["test_db.schema_test.ALIAS"] + + semantic_model, unique_database_schemas = raw_schema_to_semantic_context( + fqn_tables=fqn_tables, snowflake_account=snowflake_account + ) + + # Assert the result as expected + assert isinstance(semantic_model, semantic_model_pb2.SemanticModel) + assert isinstance(unique_database_schemas, str) + assert len(semantic_model.tables) > 0 + assert unique_database_schemas == "test_db_schema_test" + + result_yaml = proto_utils.proto_to_yaml(semantic_model) + assert result_yaml == want_yaml + + mock_snowflake_connection.assert_called_once_with( + user="test_user", + password="test_password", + account="test_account", + role="test_role", + warehouse="test_warehouse", + host="test_host", + ) + + +@patch("builtins.open", new_callable=mock_open) +def test_generate_base_context_with_placeholder_comments( + mock_file, + mock_dependencies, + mock_snowflake_connection, + mock_snowflake_connection_env, +): + + fqn_tables = ["test_db.schema_test.ALIAS"] + snowflake_account = "test_account" + output_path = "output_model_path.yaml" + + generate_base_semantic_context_from_snowflake( + fqn_tables=fqn_tables, + snowflake_account=snowflake_account, + output_yaml_path=output_path, + ) + + mock_file.assert_called_once_with(output_path, "w") + # Assert file save called with placeholder comments added. + mock_file().write.assert_called_once_with( + "name: Test Db Schema Test\ntables:\n - name: Alias\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: Zip Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n" + ) + + +@patch("builtins.open", new_callable=mock_open) +def test_generate_base_context_with_placeholder_comments_cross_database_cross_schema( + mock_file, + mock_dependencies, + mock_snowflake_connection, + mock_snowflake_connection_env, +): + + fqn_tables = [ + "test_db.schema_test.ALIAS", + "a_different_database.a_different_schema.PRODUCTS", + ] + snowflake_account = "test_account" + output_path = "output_model_path.yaml" + + generate_base_semantic_context_from_snowflake( + fqn_tables=fqn_tables, + snowflake_account=snowflake_account, + output_yaml_path=output_path, + ) + + mock_file.assert_called_once_with(output_path, "w") + # Assert file save called with placeholder comments added along with sample values and cross-database + mock_file().write.assert_called_once_with( + "name: Test Db Schema Test A Different Database A Different Schema\ntables:\n - name: Alias\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: Zip Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n - name: Products\n description: ' ' # \n base_table:\n database: a_different_database\n schema: a_different_schema\n table: PRODUCTS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n measures:\n - name: Sku\n synonyms:\n - ' ' # \n description: ' ' # \n expr: SKU\n data_type: NUMBER\n sample_values:\n - '1'\n - '2'\n - '3'\n" + ) + + +def test_semantic_model_to_yaml() -> None: + want_yaml = "name: transaction_ctx\ntables:\n - name: transactions\n description: A table containing data about financial transactions. Each row contains\n details of a financial transaction.\n base_table:\n database: my_database\n schema: my_schema\n table: transactions\n columns:\n - name: transaction_id\n description: A unique id for this transaction.\n expr: transaction_id\n data_type: BIGINT\n kind: dimension\n unique: true\n - name: account_id\n description: The account id that initialized this transaction.\n expr: account_id\n data_type: BIGINT\n kind: dimension\n - name: initiation_date\n description: Timestamp when the transaction was initiated. In UTC.\n expr: initiation_date\n data_type: DATETIME\n kind: time_dimension\n - name: amount\n description: The amount of this transaction.\n expr: amount\n data_type: DECIMAL\n kind: measure\n default_aggregation: sum\n" + got = semantic_model_pb2.SemanticModel( + name="transaction_ctx", + tables=[ + semantic_model_pb2.Table( + name="transactions", + description="A table containing data about financial transactions. Each row contains details of a financial transaction.", + base_table=semantic_model_pb2.FullyQualifiedTable( + database="my_database", + schema="my_schema", + table="transactions", + ), + columns=[ + semantic_model_pb2.Column( + name="transaction_id", + kind=semantic_model_pb2.ColumnKind.dimension, + description="A unique id for this transaction.", + expr="transaction_id", + data_type="BIGINT", + unique=True, + ), + semantic_model_pb2.Column( + name="account_id", + kind=semantic_model_pb2.ColumnKind.dimension, + description="The account id that initialized this transaction.", + expr="account_id", + data_type="BIGINT", + unique=False, + ), + semantic_model_pb2.Column( + name="initiation_date", + kind=semantic_model_pb2.ColumnKind.time_dimension, + description="Timestamp when the transaction was initiated. In UTC.", + expr="initiation_date", + data_type="DATETIME", + unique=False, + ), + semantic_model_pb2.Column( + name="amount", + kind=semantic_model_pb2.ColumnKind.measure, + description="The amount of this transaction.", + expr="amount", + data_type="DECIMAL", + default_aggregation=semantic_model_pb2.AggregationType.sum, + ), + ], + ) + ], + ) + got_yaml = proto_utils.proto_to_yaml(got) + assert got_yaml == want_yaml + + # Parse the YAML strings into Python data structures + want_data = yaml.safe_load(want_yaml) + got_data = yaml.safe_load(got_yaml) + + # Now compare the data structures + assert ( + want_data == got_data + ), "The generated YAML does not match the expected structure." diff --git a/semantic_model_generator/tests/snowflake_connector_test.py b/semantic_model_generator/tests/snowflake_connector_test.py new file mode 100644 index 00000000..9615f6d7 --- /dev/null +++ b/semantic_model_generator/tests/snowflake_connector_test.py @@ -0,0 +1,164 @@ +from unittest import mock +from unittest.mock import call + +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from semantic_model_generator.data_processing.data_types import Column, Table +from semantic_model_generator.snowflake_utils import snowflake_connector + + +@pytest.fixture +def schemas_tables_columns() -> pd.DataFrame: + return pd.DataFrame( + columns=[ + "TABLE_SCHEMA", + "TABLE_NAME", + "COLUMN_NAME", + "DATA_TYPE", + "COLUMN_COMMENT", + ], + data=[ + ["TEST_SCHEMA_1", "table_1", "col_1", "VARCHAR", None], + ["TEST_SCHEMA_1", "table_1", "col_2", "NUMBER", None], + ["TEST_SCHEMA_1", "table_2", "col_1", "NUMBER", "table_2_col_1_comment"], + [ + "TEST_SCHEMA_1", + "table_2", + "col_2", + "TIMESTAMP_NTZ", + "table_2_col_2_comment", + ], + ["TEST_SCHEMA_2", "table_3", "col_1", "VARIANT", None], + [ + "TEST_SCHEMA_2", + "invalid_table", + "col_1", + "VARIANT", + "invalid_table_col_1_comment", + ], + ], + ) + + +@pytest.fixture +def valid_tables() -> pd.DataFrame: + return pd.DataFrame( + columns=["TABLE_SCHEMA", "TABLE_NAME", "TABLE_COMMENT"], + data=[ + ["TEST_SCHEMA_1", "table_1", None], + ["TEST_SCHEMA_1", "table_2", "table_2_comment"], + ["TEST_SCHEMA_2", "table_3", "table_3_comment"], + ], + ) + + +_TEST_TABLE_ONE = Table( + id_=0, + name="table_1", + columns=[ + Column( + id_=0, + column_name="col_1", + column_type="text", + is_primary_key=True, + is_foreign_key=False, + ), + Column( + id_=1, + column_name="col_2", + column_type="number", + is_primary_key=False, + is_foreign_key=False, + ), + ], +) + + +@mock.patch( + "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" +) +def test_connect(mock_snowflake_connection: mock.MagicMock): + mock_snowflake_connection.return_value = mock.MagicMock() + + connector = snowflake_connector.SnowflakeConnector(account_name="test_account") + with connector.connect(db_name="test") as conn: + pass + + conn.cursor().execute.assert_has_calls( + [ + call("USE DATABASE test"), + call("ALTER SESSION SET QUERY_TAG = 'SEMANTIC_MODEL_GENERATOR'"), + call("ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = 120"), + ] + ) + conn.close.assert_called_with() + + +@mock.patch( + "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" +) +def test_connect_with_schema(mock_snowflake_connection: mock.MagicMock): + mock_snowflake_connection.return_value = mock.MagicMock() + + connector = snowflake_connector.SnowflakeConnector( + account_name="test_account", + ) + with connector.connect(db_name="test_db", schema_name="test_schema") as conn: + pass + + conn.cursor().execute.assert_has_calls( + [ + call("USE DATABASE test_db"), + call("USE SCHEMA test_schema"), + call("ALTER SESSION SET QUERY_TAG = 'SEMANTIC_MODEL_GENERATOR'"), + call("ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = 120"), + ] + ) + conn.close.assert_called_with() + + +@mock.patch( + "semantic_model_generator.snowflake_utils.snowflake_connector._fetch_valid_tables_and_views" +) +@mock.patch( + "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" +) +def test_get_valid_schema_table_columns_df( + mock_snowflake_connection: mock.MagicMock, + mock_valid_tables: mock.MagicMock, + valid_tables: pd.DataFrame, + schemas_tables_columns: pd.DataFrame, +): + mock_conn = mock.MagicMock() + # We expect get_database_representation() to execute queries in this order: + # - select from information_schema.tables + # - select from information_schema.columns for each table. + mock_conn.cursor().execute().fetch_pandas_all.side_effect = [ + schemas_tables_columns[schemas_tables_columns["TABLE_NAME"] == "table_1"] + ] + mock_snowflake_connection.return_value = mock_conn + mock_valid_tables.return_value = valid_tables + + got = snowflake_connector.get_valid_schemas_tables_columns_df( + mock_conn, "TEST_SCHEMA_1", ["table_1"] + ) + + want_data = { + "TABLE_SCHEMA": ["TEST_SCHEMA_1", "TEST_SCHEMA_1"], + "TABLE_NAME": ["table_1", "table_1"], + "TABLE_COMMENT": [None, None], + "COLUMN_NAME": ["col_1", "col_2"], + "DATA_TYPE": ["VARCHAR", "NUMBER"], + "COLUMN_COMMENT": [None, None], + } + + # Create a DataFrame + want = pd.DataFrame(want_data) + + assert_frame_equal(want, got) + + # Assert that the connection executed the expected queries. + query = "select t.TABLE_SCHEMA, t.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE, c.COMMENT as COLUMN_COMMENT\nfrom information_schema.tables as t\njoin information_schema.columns as c on t.table_schema = c.table_schema and t.table_name = c.table_name where t.table_schema ilike 'TEST_SCHEMA_1' AND LOWER(t.table_name) in ('table_1') \norder by 1, 2, c.ordinal_position" + mock_conn.cursor().execute.assert_any_call(query) From ca637728a180c9b0cdd0e499d230ca366fcef35d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 8 Apr 2024 15:39:31 -0700 Subject: [PATCH 02/15] update lint workflow --- .github/workflows/lint.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 5551f4cf..1422a106 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -44,16 +44,16 @@ jobs: - name: Run mypy run: | - cd semantic_model_generator && make run_mypy + make run_mypy - name: Check with black run: | - cd semantic_model_generator && make check_black + make check_black - name: Check with isort run: | - cd semantic_model_generator && make check_isort + make check_isort - name: Run flake8 run: | - cd semantic_model_generator && make run_flake8 + make run_flake8 From 862edd96635657c6dec3a61df661e04e49d3cc24 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 8 Apr 2024 15:41:02 -0700 Subject: [PATCH 03/15] update test workflow --- .github/workflows/test.yaml | 2 +- Makefile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3ffb40d9..9d0807b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,4 +43,4 @@ jobs: - name: Test run: | - make test + make test_github_workflow diff --git a/Makefile b/Makefile index 4f2e15f3..b9b44cbc 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ fmt_lint: shell ## lint/fmt in current python environment test: shell ## Run tests. python -m pytest -vvs semantic_model_generator +test_github_workflow: + python -m pytest -vvs semantic_model_generator + # Release update-version: ## Bump poetry and github version. TYPE should be `patch` `minor` or `major` @echo "Updating Poetry version ($(TYPE)) and creating a Git tag..." From 7f7bd1aa9998dc144d7f3a6d9051bb5d0cb79052 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 8 Apr 2024 15:44:50 -0700 Subject: [PATCH 04/15] update tests --- semantic_model_generator/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py index 7d1fe78a..9a5a2f2c 100644 --- a/semantic_model_generator/main.py +++ b/semantic_model_generator/main.py @@ -144,14 +144,14 @@ def raw_schema_to_semantic_context( ) # For FQN tables, create a new snowflake connection per table in case the db/schema is different. table_objects = [] - unique_database_schema: set[str] = set() + unique_database_schema: list[str] = [] for table in fqn_tables: # Verify this is a valid FQN table. For now, we check that the table follows the following format. # {database}.{schema}.{table} fqn_table = create_fqn_table(table) fqn_databse_schema = f"{fqn_table.database}_{fqn_table.schema}" if fqn_databse_schema not in unique_database_schema: - unique_database_schema.add(fqn_databse_schema) + unique_database_schema.append(fqn_databse_schema) with connector.connect( db_name=fqn_table.database, schema_name=fqn_table.schema From fe5e879f65785d32441362b0f74349294681c434 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 8 Apr 2024 15:47:57 -0700 Subject: [PATCH 05/15] additional test updates --- .../tests/snowflake_connector_test.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/semantic_model_generator/tests/snowflake_connector_test.py b/semantic_model_generator/tests/snowflake_connector_test.py index 9615f6d7..167f1a5e 100644 --- a/semantic_model_generator/tests/snowflake_connector_test.py +++ b/semantic_model_generator/tests/snowflake_connector_test.py @@ -7,6 +7,26 @@ from semantic_model_generator.data_processing.data_types import Column, Table from semantic_model_generator.snowflake_utils import snowflake_connector +from unittest.mock import patch + +@pytest.fixture +def mock_snowflake_connection_env(monkeypatch): + # Mock environment variable + monkeypatch.setenv("SNOWFLAKE_HOST", "test_host") + + # Use this fixture to also patch instance methods if needed + with patch.object( + snowflake_connector.SnowflakeConnector, "_get_user", return_value="test_user" + ), patch.object( + snowflake_connector.SnowflakeConnector, "_get_password", return_value="test_password" + ), patch.object( + snowflake_connector.SnowflakeConnector, "_get_role", return_value="test_role" + ), patch.object( + snowflake_connector.SnowflakeConnector, "_get_warehouse", return_value="test_warehouse" + ), patch.object( + snowflake_connector.SnowflakeConnector, "_get_host", return_value="test_host" + ): + yield @pytest.fixture @@ -79,7 +99,7 @@ def valid_tables() -> pd.DataFrame: @mock.patch( "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" ) -def test_connect(mock_snowflake_connection: mock.MagicMock): +def test_connect(mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env): mock_snowflake_connection.return_value = mock.MagicMock() connector = snowflake_connector.SnowflakeConnector(account_name="test_account") @@ -99,7 +119,7 @@ def test_connect(mock_snowflake_connection: mock.MagicMock): @mock.patch( "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" ) -def test_connect_with_schema(mock_snowflake_connection: mock.MagicMock): +def test_connect_with_schema(mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env): mock_snowflake_connection.return_value = mock.MagicMock() connector = snowflake_connector.SnowflakeConnector( From ac2ddc45c321d61115a0f6f6a1ebbe0ed21b307e Mon Sep 17 00:00:00 2001 From: Jonathan Hilgart Date: Tue, 9 Apr 2024 09:42:14 -0700 Subject: [PATCH 06/15] Update README.md Co-authored-by: Pieter Verhoeven <122306749+sfc-gh-pverhoeven@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cbfb8d05..657e4bc6 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ export SNOWFLAKE_HOST = "" You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. -All generated models be default are saved under `semantic_model_generator/output_models`. +All generated semantic models by default are saved under `semantic_model_generator/output_models`. **Important**: After generation, your yamls will have a series lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. From 185951c9fc01c37c0177e9f27d6949dff31afaf3 Mon Sep 17 00:00:00 2001 From: Jonathan Hilgart Date: Tue, 9 Apr 2024 09:42:24 -0700 Subject: [PATCH 07/15] Update README.md Co-authored-by: Pieter Verhoeven <122306749+sfc-gh-pverhoeven@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 657e4bc6..eadd138d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You may generate a semantic model for a given list of fully qualified tables fol All generated semantic models by default are saved under `semantic_model_generator/output_models`. -**Important**: After generation, your yamls will have a series lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. +**Important**: After generation, your YAML files will have a series of lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. ```bash From e91aa9eb9dbcb8554bb695b8edfc3ffa67f0c240 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 09:49:30 -0700 Subject: [PATCH 08/15] Bump version to 0.1.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6dc97c43..8c17a8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-model-generator" -version = "0.1.0" +version = "0.1.1" description = "Generate a Semantic Model from your Snowflake tables" authors = ["Jonathan Hilgart ", "Nipun Sehrawar ", "Renee Huang ", "Nicole Limtiaco "] license = "Apache Software License; BSD License" From 9f75e84d3421577ead06a53daeaf09ee0681a7e8 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 10:59:29 -0700 Subject: [PATCH 09/15] First round of feedback --- .github/workflows/release.yaml | 9 +- Makefile | 8 +- README.md | 33 +++++--- pyproject.toml | 2 +- semantic_model_generator/main.py | 84 +++++++++++++------ .../protos/semantic_model.proto | 58 ++----------- .../protos/semantic_model_pb2.py | 38 ++++----- .../protos/semantic_model_pb2.pyi | 39 +-------- .../snowflake_utils/snowflake_connector.py | 31 ++----- ...te_semantic_model_test.py => main_test.py} | 65 +++++++------- .../tests/snowflake_connector_test.py | 20 +++-- semantic_model_generator/tests/utils_test.py | 18 ++++ 12 files changed, 191 insertions(+), 214 deletions(-) rename semantic_model_generator/tests/{generate_semantic_model_test.py => main_test.py} (60%) create mode 100644 semantic_model_generator/tests/utils_test.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f012fcbf..747e09e5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -2,11 +2,16 @@ name: Build and Attach Wheel to GitHub Release on: push: - tags: - - 'v*' + branches: + - 'main' + paths: + - '**.py' + - 'pyproject.toml' + - 'CHANGELOG.md' jobs: build: + if: contains(github.ref, 'refs/heads/release/v') runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index b9b44cbc..ecec4d7d 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,11 @@ test_github_workflow: update-version: ## Bump poetry and github version. TYPE should be `patch` `minor` or `major` @echo "Updating Poetry version ($(TYPE)) and creating a Git tag..." @poetry version $(TYPE) - @VERSION=$$(poetry version -s) && git add pyproject.toml && git commit -m "Bump version to $$VERSION" && git tag v$$VERSION - @echo "Version updated to $$VERSION. Merge your branch then run `make release`" + @VERSION=$$(poetry version -s) + @echo "Version updated to $$VERSION. Push your branch." + + -release: ## Runs the release workflow. - git push && git push --tags help: ## Show this help. @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's \ No newline at end of file diff --git a/README.md b/README.md index eadd138d..2e6c50d6 100644 --- a/README.md +++ b/README.md @@ -20,30 +20,41 @@ export SNOWFLAKE_HOST = "" ## Usage -You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. +### Generation -All generated semantic models by default are saved under `semantic_model_generator/output_models`. +You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. Every table should be a physical table present in your database.schema. -**Important**: After generation, your YAML files will have a series of lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. +All generated semantic models by default are saved under `semantic_model_generator/output_models`. ```bash python -m semantic_model_generator.main \ - --fqn_tables "['..','..']" \ + --fqn_tables "['..','..']" \ + --semantic_model_name "" \ --snowflake_account="" ``` +### Post-Generation + +**Important**: After generation, your YAML files will have a series of lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. + +In addition, consider adding the following elements to your semantic model: + +1. Logical columns for a given table. + * Example: `col1 - col2` could be the `expr` for a logical col +2. Synonyms. Any additional synonyms for column names. +3. Metrics. Additional metrics with their relevant `expr`. + ## Release In order to push a new build and release, follow the below steps. -1. Checkout a new branch from main. Please name this branch `release-YYYY-MM-DD`. -2. Bump the poetry and github tags depending on if this is a patch, minor, or major version update: - * `export TYPE=patch make update-version` - * `export TYPE=minor make update-version` - * `export TYPE=major make update-version` +1. Checkout a new branch from main. You must name this branch `release/vYYYY-MM-DD`. The `release/v` prefix is used to trigger a github workflow post-merge. +2. Bump the poetry: + * `poetry version patch` + * `poetry version minor` + * `poetry version major` 3. Update the `CHANGELOG.md` adding a relevant header for your version number along with a description of the changes made. 4. Commit the updated `pyproject.toml` and `CHANGELOG.md` and push. -5. Merge your branch. -6. Push the updated tags to trigger the release workflow with `make release`. +5. Merge your branch which will trigger the release workflow. diff --git a/pyproject.toml b/pyproject.toml index 8c17a8a0..6dc97c43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-model-generator" -version = "0.1.1" +version = "0.1.0" description = "Generate a Semantic Model from your Snowflake tables" authors = ["Jonathan Hilgart ", "Nipun Sehrawar ", "Renee Huang ", "Nicole Limtiaco "] license = "Apache Software License; BSD License" diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py index 9a5a2f2c..337ca046 100644 --- a/semantic_model_generator/main.py +++ b/semantic_model_generator/main.py @@ -1,4 +1,3 @@ -import string from datetime import datetime import jsonargparse @@ -7,9 +6,9 @@ from semantic_model_generator.data_processing import data_types, proto_utils from semantic_model_generator.protos import semantic_model_pb2 from semantic_model_generator.snowflake_utils.snowflake_connector import ( - DIMENSION_DATATYPE_COMMON_NAME, - MEASURE_DATATYPE_COMMON_NAME, - TIME_MEASURE_DATATYPE_COMMON_NAME, + DIMENSION_DATATYPES, + MEASURE_DATATYPES, + TIME_MEASURE_DATATYPES, SnowflakeConnector, get_table_representation, get_valid_schemas_tables_columns_df, @@ -20,12 +19,6 @@ _FILL_OUT_TOKEN = " # " -def _expr_to_name(expr: str) -> str: - return expr.translate( - expr.maketrans(string.punctuation, " " * len(string.punctuation)) - ).title() - - def _get_placeholder_filter() -> list[semantic_model_pb2.NamedFilter]: return [ semantic_model_pb2.NamedFilter( @@ -55,7 +48,7 @@ def _raw_table_to_semantic_context_table( populates them with sample values, and sets placeholders for descriptions and filters. """ - # For each columns, decide if it is a TimeDimension, Measure, or Dimension column. + # For each column, decide if it is a TimeDimension, Measure, or Dimension column. # For now, we decide this based on datatype. # Any time datatype, is TimeDimension. # Any varchar/text is Dimension. @@ -67,10 +60,10 @@ def _raw_table_to_semantic_context_table( for col in raw_table.columns: - if col.column_type == TIME_MEASURE_DATATYPE_COMMON_NAME: + if col.column_type in TIME_MEASURE_DATATYPES: time_dimensions.append( semantic_model_pb2.TimeDimension( - name=_expr_to_name(col.column_name), + name=col.column_name, expr=col.column_name, data_type=col.column_type, sample_values=col.values, @@ -79,10 +72,10 @@ def _raw_table_to_semantic_context_table( ) ) - elif col.column_type == DIMENSION_DATATYPE_COMMON_NAME: + elif col.column_type in DIMENSION_DATATYPES: dimensions.append( semantic_model_pb2.Dimension( - name=_expr_to_name(col.column_name), + name=col.column_name, expr=col.column_name, data_type=col.column_type, sample_values=col.values, @@ -91,10 +84,10 @@ def _raw_table_to_semantic_context_table( ) ) - elif col.column_type == MEASURE_DATATYPE_COMMON_NAME: + elif col.column_type in MEASURE_DATATYPES: measures.append( semantic_model_pb2.Measure( - name=_expr_to_name(col.column_name), + name=col.column_name, expr=col.column_name, data_type=col.column_type, sample_values=col.values, @@ -102,9 +95,13 @@ def _raw_table_to_semantic_context_table( description=_PLACEHOLDER_COMMENT, ) ) + else: + raise ValueError( + f"Column datatype does not map to a known datatype. Input was = {col.column_type}. If this is a new datatype, please update the constants in snwoflake_connector.py." + ) return semantic_model_pb2.Table( - name=_expr_to_name(raw_table.name), + name=raw_table.name, base_table=semantic_model_pb2.FullyQualifiedTable( database=database, schema=schema, table=raw_table.name ), @@ -118,17 +115,18 @@ def _raw_table_to_semantic_context_table( def raw_schema_to_semantic_context( - fqn_tables: list[str], snowflake_account: str -) -> tuple[semantic_model_pb2.SemanticModel, str]: + fqn_tables: list[str], snowflake_account: str, semantic_model_name: str +) -> semantic_model_pb2.SemanticModel: """ Converts a list of fully qualified Snowflake table names into a semantic model. Parameters: - fqn_tables (list[str]): Fully qualified table names to include in the semantic model. - snowflake_account (str): Snowflake account identifier. + - semantic_model_name (str): A meaningful semantic model name. Returns: - - tuple: A tuple containing the semantic model (semantic_model_pb2.SemanticModel) and the model name (str). + - The semantic model (semantic_model_pb2.SemanticModel). This function fetches metadata for the specified tables, performs schema validation, extracts key information, enriches metadata from the Snowflake database, and constructs a semantic model in protobuf format. @@ -139,7 +137,7 @@ def raw_schema_to_semantic_context( """ connector = SnowflakeConnector( account_name=snowflake_account, - ndv_per_column=3, # number of sample values to pull per column. + ndv_per_column=10, # number of sample values to pull per column. max_workers=1, ) # For FQN tables, create a new snowflake connection per table in case the db/schema is different. @@ -183,11 +181,11 @@ def raw_schema_to_semantic_context( raw_table=raw_table, ) table_objects.append(table_object) - semantic_model_name = "_".join(unique_database_schema) + # TODO(jhilgart): Call cortex model to generate a semantically friendly name here. context = semantic_model_pb2.SemanticModel( - name=_expr_to_name(semantic_model_name), tables=table_objects + name=semantic_model_name, tables=table_objects ) - return context, semantic_model_name + return context def append_comment_to_placeholders(yaml_str: str) -> str: @@ -219,9 +217,31 @@ def append_comment_to_placeholders(yaml_str: str) -> str: return "\n".join(updated_yaml) +def _to_snake_case(s: str) -> str: + """ + Convert a string into snake case. + + Parameters: + s (str): The string to convert. + + Returns: + str: The snake case version of the string. + """ + # Replace common delimiters with spaces + s = s.replace("-", " ").replace("_", " ") + + words = s.split(" ") + + # Convert each word to lowercase and join with underscores + snake_case_str = "_".join([word.lower() for word in words if word]).strip() + + return snake_case_str + + def generate_base_semantic_context_from_snowflake( fqn_tables: list[str], snowflake_account: str, + semantic_model_name: str, output_yaml_path: str | None = None, ) -> None: """ @@ -230,15 +250,18 @@ def generate_base_semantic_context_from_snowflake( Args: fqn_tables: Fully qualified names of Snowflake tables to include in the semantic context. snowflake_account: Identifier of the Snowflake account. + semantic_model_name: The human readable model name. This should be semantically meaningful to an organization. output_yaml_path: Path for the output YAML file. If None, defaults to 'semantic_model_generator/output_models/YYYYMMDDHHMMSS_.yaml'. Returns: None. Writes the semantic context to a YAML file. """ - context, semantic_model_name = raw_schema_to_semantic_context( + context = raw_schema_to_semantic_context( fqn_tables=fqn_tables, snowflake_account=snowflake_account, + semantic_model_name=semantic_model_name, ) + yaml_str = proto_utils.proto_to_yaml(context) # Once we have the yaml, update to include to # tokens. yaml_str = append_comment_to_placeholders(yaml_str) @@ -249,7 +272,7 @@ def generate_base_semantic_context_from_snowflake( # Format the current date and time as "YYYY-MM-DD" formatted_datetime = current_datetime.strftime("%Y%m%d%H%M%S") - write_path = f"semantic_model_generator/output_models/{formatted_datetime}_{semantic_model_name}.yaml" + write_path = f"semantic_model_generator/output_models/{formatted_datetime}_{_to_snake_case(semantic_model_name)}.yaml" with open(write_path, "w") as f: f.write(yaml_str) return None @@ -272,6 +295,12 @@ def generate_base_semantic_context_from_snowflake( required=True, help="Your Snowflake account ID.", ) + parser.add_argument( + "--semantic_model_name", + type=str, + required=True, + help="What is the name of this semantic model? Examples could be (Churn Analysis, Marketing, Sales Prospects ...etc)", + ) parser.add_argument( "--output_yaml_path", type=str, @@ -284,5 +313,6 @@ def generate_base_semantic_context_from_snowflake( generate_base_semantic_context_from_snowflake( fqn_tables=args.fqn_tables, snowflake_account=args.snowflake_account, + semantic_model_name=args.semantic_model_name, output_yaml_path=args.output_yaml_path, ) diff --git a/semantic_model_generator/protos/semantic_model.proto b/semantic_model_generator/protos/semantic_model.proto index 3449d5f3..520bc0d4 100644 --- a/semantic_model_generator/protos/semantic_model.proto +++ b/semantic_model_generator/protos/semantic_model.proto @@ -19,49 +19,8 @@ enum AggregationType { count_distinct = 6; } -// ColumnKind defines various kinds of columns, mainly categorized into -// dimensions and measures. -enum ColumnKind { - column_kind_unknown = 0; - // A column containing categorical values such as names, countries, dates. - dimension = 1; - // A column containing numerical values such as revenue, impressions, salary. - measure = 2; - // A column containing date/time data. - time_dimension = 3; -} - -// Column is analogous to a database column and defines various semantic properties -// of a column. A column can either simply be a column in the base database schema -// or it can be an arbitrary expression over the base schema, e.g. -// `base_column1 + base_column2`. -message Column { - // A descriptive name for this column. - string name = 1; - // A list of other terms/phrases used to refer to this column. - repeated string synonyms = 2; - // A brief description about this column, including things like what data this - // column has. - string description = 3; - // The SQL expression for this column. Could simply be a base table column name - // or an arbitrary SQL expression over one or more columns of the base table. - string expr = 4; - // The data type of this column. - string data_type = 5; - // The kind of this column - dimension or measure. - ColumnKind kind = 6; - // If true, assume that this column has unique values. - bool unique = 7; - // If no aggregation is specified, then this is the default aggregation - // applied to this column in context of a grouping. - AggregationType default_aggregation = 8; - // Sample values of this column. - repeated string sample_values = 9; -} - // Dimension columns contain categorical values (e.g. state, user_type, platform). -// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() -// of snowpilot/semantic_context/utils/utils.py. +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format(). message Dimension { // A descriptive name for this dimension. string name = 1; @@ -82,8 +41,7 @@ message Dimension { } // Time dimension columns contain time values (e.g. sale_date, created_at, year). -// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() -// of snowpilot/semantic_context/utils/utils.py. +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format(). message TimeDimension { // A descriptive name for this time dimension. string name = 1; @@ -105,8 +63,7 @@ message TimeDimension { } // Measure columns contain numerical values (e.g. revenue, impressions, salary). -// NOTE: If modifying this protobuf, make appropriate changes in to_column_format() -// of snowpilot/semantic_context/utils/utils.py. +// NOTE: If modifying this protobuf, make appropriate changes in to_column_format(). message Measure { // A descriptive name for this measure. string name = 1; @@ -163,13 +120,8 @@ message Table { // Fully qualified name of the underlying base table. FullyQualifiedTable base_table = 4; - // We allow two formats for specifying logical columns of a table: - // 1. As a list of columns. - // 2. As three separate list of dimensions, time dimensions, and measures. - // For the external facing yaml specification, we have chosen to go with (2). - // However, for the time being we'll support both (1) and (2) and continue - // using (1) as the internal representation. - repeated Column columns = 5; + // Dimension, TimeDimension, and Measure are all types of Columns for a table. + // These may be logical or physical columns. repeated Dimension dimensions = 9; repeated TimeDimension time_dimensions = 10; repeated Measure measures = 11; diff --git a/semantic_model_generator/protos/semantic_model_pb2.py b/semantic_model_generator/protos/semantic_model_pb2.py index 61f92bf8..2677081c 100644 --- a/semantic_model_generator/protos/semantic_model_pb2.py +++ b/semantic_model_generator/protos/semantic_model_pb2.py @@ -14,31 +14,27 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14semantic_model.proto\x12\x18semantic_model_generator\"\x81\x02\n\x06\x43olumn\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x32\n\x04kind\x18\x06 \x01(\x0e\x32$.semantic_model_generator.ColumnKind\x12\x0e\n\x06unique\x18\x07 \x01(\x08\x12\x46\n\x13\x64\x65\x66\x61ult_aggregation\x18\x08 \x01(\x0e\x32).semantic_model_generator.AggregationType\x12\x15\n\rsample_values\x18\t \x03(\t\"\x88\x01\n\tDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\x8c\x01\n\rTimeDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\xbe\x01\n\x07Measure\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x46\n\x13\x64\x65\x66\x61ult_aggregation\x18\x06 \x01(\x0e\x32).semantic_model_generator.AggregationType\x12\x15\n\rsample_values\x18\x07 \x03(\t\"P\n\x0bNamedFilter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\"F\n\x13\x46ullyQualifiedTable\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\x9a\x03\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x41\n\nbase_table\x18\x04 \x01(\x0b\x32-.semantic_model_generator.FullyQualifiedTable\x12\x31\n\x07\x63olumns\x18\x05 \x03(\x0b\x32 .semantic_model_generator.Column\x12\x37\n\ndimensions\x18\t \x03(\x0b\x32#.semantic_model_generator.Dimension\x12@\n\x0ftime_dimensions\x18\n \x03(\x0b\x32\'.semantic_model_generator.TimeDimension\x12\x33\n\x08measures\x18\x0b \x03(\x0b\x32!.semantic_model_generator.Measure\x12\x36\n\x07\x66ilters\x18\x08 \x03(\x0b\x32%.semantic_model_generator.NamedFilter\"c\n\rSemanticModel\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.semantic_model_generator.Table*~\n\x0f\x41ggregationType\x12\x1c\n\x18\x61ggregation_type_unknown\x10\x00\x12\x07\n\x03sum\x10\x01\x12\x07\n\x03\x61vg\x10\x02\x12\n\n\x06median\x10\x07\x12\x07\n\x03min\x10\x03\x12\x07\n\x03max\x10\x04\x12\t\n\x05\x63ount\x10\x05\x12\x12\n\x0e\x63ount_distinct\x10\x06*U\n\nColumnKind\x12\x17\n\x13\x63olumn_kind_unknown\x10\x00\x12\r\n\tdimension\x10\x01\x12\x0b\n\x07measure\x10\x02\x12\x12\n\x0etime_dimension\x10\x03\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14semantic_model.proto\x12\x18semantic_model_generator\"\x88\x01\n\tDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\x8c\x01\n\rTimeDimension\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x0e\n\x06unique\x18\x06 \x01(\x08\x12\x15\n\rsample_values\x18\x07 \x03(\t\"\xbe\x01\n\x07Measure\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\x12\x11\n\tdata_type\x18\x05 \x01(\t\x12\x46\n\x13\x64\x65\x66\x61ult_aggregation\x18\x06 \x01(\x0e\x32).semantic_model_generator.AggregationType\x12\x15\n\rsample_values\x18\x07 \x03(\t\"P\n\x0bNamedFilter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0c\n\x04\x65xpr\x18\x04 \x01(\t\"F\n\x13\x46ullyQualifiedTable\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\r\n\x05table\x18\x03 \x01(\t\"\xe7\x02\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08synonyms\x18\x02 \x03(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x41\n\nbase_table\x18\x04 \x01(\x0b\x32-.semantic_model_generator.FullyQualifiedTable\x12\x37\n\ndimensions\x18\t \x03(\x0b\x32#.semantic_model_generator.Dimension\x12@\n\x0ftime_dimensions\x18\n \x03(\x0b\x32\'.semantic_model_generator.TimeDimension\x12\x33\n\x08measures\x18\x0b \x03(\x0b\x32!.semantic_model_generator.Measure\x12\x36\n\x07\x66ilters\x18\x08 \x03(\x0b\x32%.semantic_model_generator.NamedFilter\"c\n\rSemanticModel\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.semantic_model_generator.Table*~\n\x0f\x41ggregationType\x12\x1c\n\x18\x61ggregation_type_unknown\x10\x00\x12\x07\n\x03sum\x10\x01\x12\x07\n\x03\x61vg\x10\x02\x12\n\n\x06median\x10\x07\x12\x07\n\x03min\x10\x03\x12\x07\n\x03max\x10\x04\x12\t\n\x05\x63ount\x10\x05\x12\x12\n\x0e\x63ount_distinct\x10\x06\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'semantic_model_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None - _globals['_AGGREGATIONTYPE']._serialized_start=1453 - _globals['_AGGREGATIONTYPE']._serialized_end=1579 - _globals['_COLUMNKIND']._serialized_start=1581 - _globals['_COLUMNKIND']._serialized_end=1666 - _globals['_COLUMN']._serialized_start=51 - _globals['_COLUMN']._serialized_end=308 - _globals['_DIMENSION']._serialized_start=311 - _globals['_DIMENSION']._serialized_end=447 - _globals['_TIMEDIMENSION']._serialized_start=450 - _globals['_TIMEDIMENSION']._serialized_end=590 - _globals['_MEASURE']._serialized_start=593 - _globals['_MEASURE']._serialized_end=783 - _globals['_NAMEDFILTER']._serialized_start=785 - _globals['_NAMEDFILTER']._serialized_end=865 - _globals['_FULLYQUALIFIEDTABLE']._serialized_start=867 - _globals['_FULLYQUALIFIEDTABLE']._serialized_end=937 - _globals['_TABLE']._serialized_start=940 - _globals['_TABLE']._serialized_end=1350 - _globals['_SEMANTICMODEL']._serialized_start=1352 - _globals['_SEMANTICMODEL']._serialized_end=1451 + _globals['_AGGREGATIONTYPE']._serialized_start=1142 + _globals['_AGGREGATIONTYPE']._serialized_end=1268 + _globals['_DIMENSION']._serialized_start=51 + _globals['_DIMENSION']._serialized_end=187 + _globals['_TIMEDIMENSION']._serialized_start=190 + _globals['_TIMEDIMENSION']._serialized_end=330 + _globals['_MEASURE']._serialized_start=333 + _globals['_MEASURE']._serialized_end=523 + _globals['_NAMEDFILTER']._serialized_start=525 + _globals['_NAMEDFILTER']._serialized_end=605 + _globals['_FULLYQUALIFIEDTABLE']._serialized_start=607 + _globals['_FULLYQUALIFIEDTABLE']._serialized_end=677 + _globals['_TABLE']._serialized_start=680 + _globals['_TABLE']._serialized_end=1039 + _globals['_SEMANTICMODEL']._serialized_start=1041 + _globals['_SEMANTICMODEL']._serialized_end=1140 # @@protoc_insertion_point(module_scope) diff --git a/semantic_model_generator/protos/semantic_model_pb2.pyi b/semantic_model_generator/protos/semantic_model_pb2.pyi index c0382875..ac767331 100644 --- a/semantic_model_generator/protos/semantic_model_pb2.pyi +++ b/semantic_model_generator/protos/semantic_model_pb2.pyi @@ -16,13 +16,6 @@ class AggregationType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): max: _ClassVar[AggregationType] count: _ClassVar[AggregationType] count_distinct: _ClassVar[AggregationType] - -class ColumnKind(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - column_kind_unknown: _ClassVar[ColumnKind] - dimension: _ClassVar[ColumnKind] - measure: _ClassVar[ColumnKind] - time_dimension: _ClassVar[ColumnKind] aggregation_type_unknown: AggregationType sum: AggregationType avg: AggregationType @@ -31,32 +24,6 @@ min: AggregationType max: AggregationType count: AggregationType count_distinct: AggregationType -column_kind_unknown: ColumnKind -dimension: ColumnKind -measure: ColumnKind -time_dimension: ColumnKind - -class Column(_message.Message): - __slots__ = ("name", "synonyms", "description", "expr", "data_type", "kind", "unique", "default_aggregation", "sample_values") - NAME_FIELD_NUMBER: _ClassVar[int] - SYNONYMS_FIELD_NUMBER: _ClassVar[int] - DESCRIPTION_FIELD_NUMBER: _ClassVar[int] - EXPR_FIELD_NUMBER: _ClassVar[int] - DATA_TYPE_FIELD_NUMBER: _ClassVar[int] - KIND_FIELD_NUMBER: _ClassVar[int] - UNIQUE_FIELD_NUMBER: _ClassVar[int] - DEFAULT_AGGREGATION_FIELD_NUMBER: _ClassVar[int] - SAMPLE_VALUES_FIELD_NUMBER: _ClassVar[int] - name: str - synonyms: _containers.RepeatedScalarFieldContainer[str] - description: str - expr: str - data_type: str - kind: ColumnKind - unique: bool - default_aggregation: AggregationType - sample_values: _containers.RepeatedScalarFieldContainer[str] - def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., expr: _Optional[str] = ..., data_type: _Optional[str] = ..., kind: _Optional[_Union[ColumnKind, str]] = ..., unique: bool = ..., default_aggregation: _Optional[_Union[AggregationType, str]] = ..., sample_values: _Optional[_Iterable[str]] = ...) -> None: ... class Dimension(_message.Message): __slots__ = ("name", "synonyms", "description", "expr", "data_type", "unique", "sample_values") @@ -135,12 +102,11 @@ class FullyQualifiedTable(_message.Message): def __init__(self, database: _Optional[str] = ..., schema: _Optional[str] = ..., table: _Optional[str] = ...) -> None: ... class Table(_message.Message): - __slots__ = ("name", "synonyms", "description", "base_table", "columns", "dimensions", "time_dimensions", "measures", "filters") + __slots__ = ("name", "synonyms", "description", "base_table", "dimensions", "time_dimensions", "measures", "filters") NAME_FIELD_NUMBER: _ClassVar[int] SYNONYMS_FIELD_NUMBER: _ClassVar[int] DESCRIPTION_FIELD_NUMBER: _ClassVar[int] BASE_TABLE_FIELD_NUMBER: _ClassVar[int] - COLUMNS_FIELD_NUMBER: _ClassVar[int] DIMENSIONS_FIELD_NUMBER: _ClassVar[int] TIME_DIMENSIONS_FIELD_NUMBER: _ClassVar[int] MEASURES_FIELD_NUMBER: _ClassVar[int] @@ -149,12 +115,11 @@ class Table(_message.Message): synonyms: _containers.RepeatedScalarFieldContainer[str] description: str base_table: FullyQualifiedTable - columns: _containers.RepeatedCompositeFieldContainer[Column] dimensions: _containers.RepeatedCompositeFieldContainer[Dimension] time_dimensions: _containers.RepeatedCompositeFieldContainer[TimeDimension] measures: _containers.RepeatedCompositeFieldContainer[Measure] filters: _containers.RepeatedCompositeFieldContainer[NamedFilter] - def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., base_table: _Optional[_Union[FullyQualifiedTable, _Mapping]] = ..., columns: _Optional[_Iterable[_Union[Column, _Mapping]]] = ..., dimensions: _Optional[_Iterable[_Union[Dimension, _Mapping]]] = ..., time_dimensions: _Optional[_Iterable[_Union[TimeDimension, _Mapping]]] = ..., measures: _Optional[_Iterable[_Union[Measure, _Mapping]]] = ..., filters: _Optional[_Iterable[_Union[NamedFilter, _Mapping]]] = ...) -> None: ... + def __init__(self, name: _Optional[str] = ..., synonyms: _Optional[_Iterable[str]] = ..., description: _Optional[str] = ..., base_table: _Optional[_Union[FullyQualifiedTable, _Mapping]] = ..., dimensions: _Optional[_Iterable[_Union[Dimension, _Mapping]]] = ..., time_dimensions: _Optional[_Iterable[_Union[TimeDimension, _Mapping]]] = ..., measures: _Optional[_Iterable[_Union[Measure, _Mapping]]] = ..., filters: _Optional[_Iterable[_Union[NamedFilter, _Mapping]]] = ...) -> None: ... class SemanticModel(_message.Message): __slots__ = ("name", "description", "tables") diff --git a/semantic_model_generator/snowflake_utils/snowflake_connector.py b/semantic_model_generator/snowflake_utils/snowflake_connector.py index 57d5653d..6afb9266 100644 --- a/semantic_model_generator/snowflake_utils/snowflake_connector.py +++ b/semantic_model_generator/snowflake_utils/snowflake_connector.py @@ -25,7 +25,7 @@ _TABLE_COMMENT_COL = "TABLE_COMMENT" # https://docs.snowflake.com/en/sql-reference/data-types-datetime -_TIME_MEASURE_DATATYPES = [ +TIME_MEASURE_DATATYPES = [ "DATE", "DATETIME", "TIMESTAMP_LTZ", @@ -34,9 +34,8 @@ "TIMESTAMP", "TIME", ] -TIME_MEASURE_DATATYPE_COMMON_NAME = "TIMESTAMP" # https://docs.snowflake.com/en/sql-reference/data-types-text -_DIMENSION_DATATYPES = [ +DIMENSION_DATATYPES = [ "VARCHAR", "CHAR", "CHARACTER", @@ -46,13 +45,12 @@ "NVARCHAR", "NVARCHAR2", "CHAR VARYING", - " NCHAR VARYING", + "NCHAR VARYING", "BINARY", "VARBINARY", ] -DIMENSION_DATATYPE_COMMON_NAME = "TEXT" # https://docs.snowflake.com/en/sql-reference/data-types-numeric -_MEASURE_DATATYPES = [ +MEASURE_DATATYPES = [ "NUMBER", "DECIMAL", "DEC", @@ -70,26 +68,11 @@ "DOUBLE PRECISION", "REAL", ] -MEASURE_DATATYPE_COMMON_NAME = "NUMBER" _QUERY_TAG = "SEMANTIC_MODEL_GENERATOR" -def determine_col_datatype(snowflake_datatype: str) -> str: - # Takes in a snowflake datatype and returns the common datatype - if snowflake_datatype in _TIME_MEASURE_DATATYPES: - return TIME_MEASURE_DATATYPE_COMMON_NAME - elif snowflake_datatype in _DIMENSION_DATATYPES: - return DIMENSION_DATATYPE_COMMON_NAME - elif snowflake_datatype in _MEASURE_DATATYPES: - return MEASURE_DATATYPE_COMMON_NAME - else: - raise ValueError( - f"Input snowflake datatype {snowflake_datatype} not supported yet. Please add to either _TIME_MEASURE_DATATYPES, _DIMENSION_DATATYPES, or _MEASURE_DATATYPES" - ) - - def get_table_representation( conn: SnowflakeConnection, schema_name: str, @@ -171,7 +154,7 @@ def _get_column_representation( id_=column_index, column_name=column_name, comment=column_comment, - column_type=determine_col_datatype(column_datatype), + column_type=column_datatype, values=column_values, ) return column @@ -351,12 +334,16 @@ def execute( logger.debug( f"There is no Warehouse assigned to Connection, setting it to config default ({warehouse})" ) + # TODO(jhilgart): Do we need to replace - with _? + # Snowflake docs suggest we need identifiers with _, https://docs.snowflake.com/en/sql-reference/identifiers-syntax, + # but unclear if we need this here. connection.cursor().execute( f'use warehouse {warehouse.replace("-", "_")}' ) cursor = connection.cursor(DictCursor) logger.info(f"Executing query = {query}") cursor_execute = cursor.execute(query) + # assert below for MyPy. Should always be true. assert cursor_execute, "cursor_execute should not be None here" result = cursor_execute.fetchall() except ProgrammingError as e: diff --git a/semantic_model_generator/tests/generate_semantic_model_test.py b/semantic_model_generator/tests/main_test.py similarity index 60% rename from semantic_model_generator/tests/generate_semantic_model_test.py rename to semantic_model_generator/tests/main_test.py index 2a5a2d70..c674dfa2 100644 --- a/semantic_model_generator/tests/generate_semantic_model_test.py +++ b/semantic_model_generator/tests/main_test.py @@ -7,6 +7,7 @@ from semantic_model_generator.data_processing import proto_utils from semantic_model_generator.data_processing.data_types import Column, Table from semantic_model_generator.main import ( + _to_snake_case, generate_base_semantic_context_from_snowflake, raw_schema_to_semantic_context, ) @@ -16,6 +17,12 @@ ) +def test_to_snake_case(): + text = "Hello World-How are_you" + + assert "hello_world_how_are_you" == _to_snake_case(text) + + @pytest.fixture def mock_snowflake_connection(): """Fixture to mock the snowflake_connection function.""" @@ -136,20 +143,21 @@ def mock_dependencies(mock_snowflake_connection): def test_raw_schema_to_semantic_context( mock_dependencies, mock_snowflake_connection, mock_snowflake_connection_env ): - want_yaml = "name: Test Db Schema Test\ntables:\n - name: Alias\n description: ' '\n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' '\n synonyms:\n - ' '\n description: ' '\n expr: ' '\n dimensions:\n - name: Zip Code\n synonyms:\n - ' '\n description: ' '\n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' '\n description: ' '\n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' '\n description: ' '\n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' '\n description: ' '\n expr: CBSA\n data_type: NUMBER\n" + want_yaml = "name: this is the best semantic model ever\ntables:\n - name: ALIAS\n description: ' '\n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' '\n synonyms:\n - ' '\n description: ' '\n expr: ' '\n dimensions:\n - name: ZIP_CODE\n synonyms:\n - ' '\n description: ' '\n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: BAD_ALIAS\n synonyms:\n - ' '\n description: ' '\n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: AREA_CODE\n synonyms:\n - ' '\n description: ' '\n expr: AREA_CODE\n data_type: NUMBER\n - name: CBSA\n synonyms:\n - ' '\n description: ' '\n expr: CBSA\n data_type: NUMBER\n" snowflake_account = "test_account" fqn_tables = ["test_db.schema_test.ALIAS"] + semantic_model_name = "this is the best semantic model ever" - semantic_model, unique_database_schemas = raw_schema_to_semantic_context( - fqn_tables=fqn_tables, snowflake_account=snowflake_account + semantic_model = raw_schema_to_semantic_context( + fqn_tables=fqn_tables, + snowflake_account=snowflake_account, + semantic_model_name=semantic_model_name, ) # Assert the result as expected assert isinstance(semantic_model, semantic_model_pb2.SemanticModel) - assert isinstance(unique_database_schemas, str) assert len(semantic_model.tables) > 0 - assert unique_database_schemas == "test_db_schema_test" result_yaml = proto_utils.proto_to_yaml(semantic_model) assert result_yaml == want_yaml @@ -175,17 +183,19 @@ def test_generate_base_context_with_placeholder_comments( fqn_tables = ["test_db.schema_test.ALIAS"] snowflake_account = "test_account" output_path = "output_model_path.yaml" + semantic_model_name = "my awesome semantic model" generate_base_semantic_context_from_snowflake( fqn_tables=fqn_tables, snowflake_account=snowflake_account, output_yaml_path=output_path, + semantic_model_name=semantic_model_name, ) mock_file.assert_called_once_with(output_path, "w") # Assert file save called with placeholder comments added. mock_file().write.assert_called_once_with( - "name: Test Db Schema Test\ntables:\n - name: Alias\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: Zip Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n" + "name: my awesome semantic model\ntables:\n - name: ALIAS\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: ZIP_CODE\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: BAD_ALIAS\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: AREA_CODE\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: CBSA\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n" ) @@ -203,22 +213,24 @@ def test_generate_base_context_with_placeholder_comments_cross_database_cross_sc ] snowflake_account = "test_account" output_path = "output_model_path.yaml" + semantic_model_name = "Another Incredible Semantic Model" generate_base_semantic_context_from_snowflake( fqn_tables=fqn_tables, snowflake_account=snowflake_account, output_yaml_path=output_path, + semantic_model_name=semantic_model_name, ) mock_file.assert_called_once_with(output_path, "w") # Assert file save called with placeholder comments added along with sample values and cross-database mock_file().write.assert_called_once_with( - "name: Test Db Schema Test A Different Database A Different Schema\ntables:\n - name: Alias\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: Zip Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: Bad Alias\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: Area Code\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: Cbsa\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n - name: Products\n description: ' ' # \n base_table:\n database: a_different_database\n schema: a_different_schema\n table: PRODUCTS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n measures:\n - name: Sku\n synonyms:\n - ' ' # \n description: ' ' # \n expr: SKU\n data_type: NUMBER\n sample_values:\n - '1'\n - '2'\n - '3'\n" + "name: Another Incredible Semantic Model\ntables:\n - name: ALIAS\n description: ' ' # \n base_table:\n database: test_db\n schema: schema_test\n table: ALIAS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n dimensions:\n - name: ZIP_CODE\n synonyms:\n - ' ' # \n description: ' ' # \n expr: ZIP_CODE\n data_type: TEXT\n time_dimensions:\n - name: BAD_ALIAS\n synonyms:\n - ' ' # \n description: ' ' # \n expr: BAD_ALIAS\n data_type: TIMESTAMP\n measures:\n - name: AREA_CODE\n synonyms:\n - ' ' # \n description: ' ' # \n expr: AREA_CODE\n data_type: NUMBER\n - name: CBSA\n synonyms:\n - ' ' # \n description: ' ' # \n expr: CBSA\n data_type: NUMBER\n - name: PRODUCTS\n description: ' ' # \n base_table:\n database: a_different_database\n schema: a_different_schema\n table: PRODUCTS\n filters:\n - name: ' ' # \n synonyms:\n - ' ' # \n description: ' ' # \n expr: ' ' # \n measures:\n - name: SKU\n synonyms:\n - ' ' # \n description: ' ' # \n expr: SKU\n data_type: NUMBER\n sample_values:\n - '1'\n - '2'\n - '3'\n" ) def test_semantic_model_to_yaml() -> None: - want_yaml = "name: transaction_ctx\ntables:\n - name: transactions\n description: A table containing data about financial transactions. Each row contains\n details of a financial transaction.\n base_table:\n database: my_database\n schema: my_schema\n table: transactions\n columns:\n - name: transaction_id\n description: A unique id for this transaction.\n expr: transaction_id\n data_type: BIGINT\n kind: dimension\n unique: true\n - name: account_id\n description: The account id that initialized this transaction.\n expr: account_id\n data_type: BIGINT\n kind: dimension\n - name: initiation_date\n description: Timestamp when the transaction was initiated. In UTC.\n expr: initiation_date\n data_type: DATETIME\n kind: time_dimension\n - name: amount\n description: The amount of this transaction.\n expr: amount\n data_type: DECIMAL\n kind: measure\n default_aggregation: sum\n" + want_yaml = "name: transaction_ctx\ntables:\n - name: transactions\n description: A table containing data about financial transactions. Each row contains\n details of a financial transaction.\n base_table:\n database: my_database\n schema: my_schema\n table: transactions\n dimensions:\n - name: transaction_id\n description: A unique id for this transaction.\n expr: transaction_id\n data_type: BIGINT\n unique: true\n time_dimensions:\n - name: initiation_date\n description: Timestamp when the transaction was initiated. In UTC.\n expr: initiation_date\n data_type: DATETIME\n measures:\n - name: amount\n description: The amount of this transaction.\n expr: amount\n data_type: DECIMAL\n default_aggregation: sum\n" got = semantic_model_pb2.SemanticModel( name="transaction_ctx", tables=[ @@ -230,40 +242,33 @@ def test_semantic_model_to_yaml() -> None: schema="my_schema", table="transactions", ), - columns=[ - semantic_model_pb2.Column( - name="transaction_id", - kind=semantic_model_pb2.ColumnKind.dimension, - description="A unique id for this transaction.", - expr="transaction_id", - data_type="BIGINT", - unique=True, - ), - semantic_model_pb2.Column( - name="account_id", - kind=semantic_model_pb2.ColumnKind.dimension, - description="The account id that initialized this transaction.", - expr="account_id", - data_type="BIGINT", - unique=False, - ), - semantic_model_pb2.Column( + time_dimensions=[ + semantic_model_pb2.TimeDimension( name="initiation_date", - kind=semantic_model_pb2.ColumnKind.time_dimension, description="Timestamp when the transaction was initiated. In UTC.", expr="initiation_date", data_type="DATETIME", unique=False, - ), - semantic_model_pb2.Column( + ) + ], + measures=[ + semantic_model_pb2.Measure( name="amount", - kind=semantic_model_pb2.ColumnKind.measure, description="The amount of this transaction.", expr="amount", data_type="DECIMAL", default_aggregation=semantic_model_pb2.AggregationType.sum, ), ], + dimensions=[ + semantic_model_pb2.Dimension( + name="transaction_id", + description="A unique id for this transaction.", + expr="transaction_id", + data_type="BIGINT", + unique=True, + ) + ], ) ], ) diff --git a/semantic_model_generator/tests/snowflake_connector_test.py b/semantic_model_generator/tests/snowflake_connector_test.py index 167f1a5e..0ffd509c 100644 --- a/semantic_model_generator/tests/snowflake_connector_test.py +++ b/semantic_model_generator/tests/snowflake_connector_test.py @@ -1,5 +1,5 @@ from unittest import mock -from unittest.mock import call +from unittest.mock import call, patch import pandas as pd import pytest @@ -7,7 +7,7 @@ from semantic_model_generator.data_processing.data_types import Column, Table from semantic_model_generator.snowflake_utils import snowflake_connector -from unittest.mock import patch + @pytest.fixture def mock_snowflake_connection_env(monkeypatch): @@ -18,11 +18,15 @@ def mock_snowflake_connection_env(monkeypatch): with patch.object( snowflake_connector.SnowflakeConnector, "_get_user", return_value="test_user" ), patch.object( - snowflake_connector.SnowflakeConnector, "_get_password", return_value="test_password" + snowflake_connector.SnowflakeConnector, + "_get_password", + return_value="test_password", ), patch.object( snowflake_connector.SnowflakeConnector, "_get_role", return_value="test_role" ), patch.object( - snowflake_connector.SnowflakeConnector, "_get_warehouse", return_value="test_warehouse" + snowflake_connector.SnowflakeConnector, + "_get_warehouse", + return_value="test_warehouse", ), patch.object( snowflake_connector.SnowflakeConnector, "_get_host", return_value="test_host" ): @@ -99,7 +103,9 @@ def valid_tables() -> pd.DataFrame: @mock.patch( "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" ) -def test_connect(mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env): +def test_connect( + mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env +): mock_snowflake_connection.return_value = mock.MagicMock() connector = snowflake_connector.SnowflakeConnector(account_name="test_account") @@ -119,7 +125,9 @@ def test_connect(mock_snowflake_connection: mock.MagicMock, mock_snowflake_conne @mock.patch( "semantic_model_generator.snowflake_utils.snowflake_connector.snowflake_connection" ) -def test_connect_with_schema(mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env): +def test_connect_with_schema( + mock_snowflake_connection: mock.MagicMock, mock_snowflake_connection_env +): mock_snowflake_connection.return_value = mock.MagicMock() connector = snowflake_connector.SnowflakeConnector( diff --git a/semantic_model_generator/tests/utils_test.py b/semantic_model_generator/tests/utils_test.py new file mode 100644 index 00000000..dc826ef2 --- /dev/null +++ b/semantic_model_generator/tests/utils_test.py @@ -0,0 +1,18 @@ +import pytest + +from semantic_model_generator.data_processing.data_types import FQNParts +from semantic_model_generator.snowflake_utils.utils import create_fqn_table + + +def test_fqn_creation(): + input_name = "database.schema.table" + + fqn_parts = create_fqn_table(input_name) + + assert fqn_parts == FQNParts(database="database", schema="schema", table="table") + + +def test_fqn_creation_invalid_name(): + input_name = "database.schema table" + with pytest.raises(ValueError): + create_fqn_table(input_name) From 0963edc4f3caa7fcd0e73f8e4d70d720a5587783 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 11:00:45 -0700 Subject: [PATCH 10/15] update makefile --- Makefile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Makefile b/Makefile index ecec4d7d..95854e4f 100644 --- a/Makefile +++ b/Makefile @@ -42,18 +42,8 @@ fmt_lint: shell ## lint/fmt in current python environment test: shell ## Run tests. python -m pytest -vvs semantic_model_generator -test_github_workflow: +test_github_workflow: ## For use on github workflow. python -m pytest -vvs semantic_model_generator -# Release -update-version: ## Bump poetry and github version. TYPE should be `patch` `minor` or `major` - @echo "Updating Poetry version ($(TYPE)) and creating a Git tag..." - @poetry version $(TYPE) - @VERSION=$$(poetry version -s) - @echo "Version updated to $$VERSION. Push your branch." - - - - help: ## Show this help. @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's \ No newline at end of file From 858561ad55e5056a1dba3412f50110e958e4374a Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 11:04:44 -0700 Subject: [PATCH 11/15] update ndv to 10 --- semantic_model_generator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py index 337ca046..0a97e34b 100644 --- a/semantic_model_generator/main.py +++ b/semantic_model_generator/main.py @@ -170,7 +170,7 @@ def raw_schema_to_semantic_context( schema_name=fqn_table.schema, table_name=fqn_table.table, table_index=0, - ndv_per_column=3, + ndv_per_column=10, # number of sample values to pull per column. columns_df=valid_columns_df_this_table, max_workers=1, ) From acd6b424a9648dd38ef2576b195789a5482c9e9d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 11:10:10 -0700 Subject: [PATCH 12/15] lint --- semantic_model_generator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py index 0a97e34b..9219d5e0 100644 --- a/semantic_model_generator/main.py +++ b/semantic_model_generator/main.py @@ -170,7 +170,7 @@ def raw_schema_to_semantic_context( schema_name=fqn_table.schema, table_name=fqn_table.table, table_index=0, - ndv_per_column=10, # number of sample values to pull per column. + ndv_per_column=10, # number of sample values to pull per column. columns_df=valid_columns_df_this_table, max_workers=1, ) From 820de6b9f65b06670c543799921ffc1326e82317 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 14:25:55 -0700 Subject: [PATCH 13/15] add another test --- .../snowflake_utils/snowflake_connector.py | 1 + .../tests/snowflake_connector_test.py | 57 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/semantic_model_generator/snowflake_utils/snowflake_connector.py b/semantic_model_generator/snowflake_utils/snowflake_connector.py index 6afb9266..408fffd7 100644 --- a/semantic_model_generator/snowflake_utils/snowflake_connector.py +++ b/semantic_model_generator/snowflake_utils/snowflake_connector.py @@ -164,6 +164,7 @@ def _fetch_valid_tables_and_views(conn: SnowflakeConnection) -> pd.DataFrame: def _get_df(query: str) -> pd.DataFrame: cursor = conn.cursor().execute(query) assert cursor is not None, "cursor should not be none here." + df = pd.DataFrame( cursor.fetchall(), columns=[c.name for c in cursor.description] ) diff --git a/semantic_model_generator/tests/snowflake_connector_test.py b/semantic_model_generator/tests/snowflake_connector_test.py index 0ffd509c..702c5b19 100644 --- a/semantic_model_generator/tests/snowflake_connector_test.py +++ b/semantic_model_generator/tests/snowflake_connector_test.py @@ -1,5 +1,5 @@ from unittest import mock -from unittest.mock import call, patch +from unittest.mock import MagicMock, call, patch import pandas as pd import pytest @@ -190,3 +190,58 @@ def test_get_valid_schema_table_columns_df( # Assert that the connection executed the expected queries. query = "select t.TABLE_SCHEMA, t.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE, c.COMMENT as COLUMN_COMMENT\nfrom information_schema.tables as t\njoin information_schema.columns as c on t.table_schema = c.table_schema and t.table_name = c.table_name where t.table_schema ilike 'TEST_SCHEMA_1' AND LOWER(t.table_name) in ('table_1') \norder by 1, 2, c.ordinal_position" mock_conn.cursor().execute.assert_any_call(query) + + +@pytest.fixture +def snowflake_data(): + return [ + # This mimics the return value of cursor.fetchall() for tables and views + ([("table1", "schema1", "A table comment")], [("column1", "dtype")]), + ([("view1", "schema1", "A view comment")], [("column1", "dtype")]), + ] + + +@pytest.fixture +def expected_df(): + # Expected DataFrame structure based on mocked fetchall data + return pd.DataFrame( + { + snowflake_connector._TABLE_NAME_COL: ["table1", "view1"], + snowflake_connector._TABLE_SCHEMA_COL: ["schema1", "schema1"], + snowflake_connector._TABLE_COMMENT_COL: [ + "A table comment", + "A view comment", + ], + } + ) + + +def test_fetch_valid_tables_and_views(snowflake_data, expected_df): + # Mock SnowflakeConnection and cursor + mock_conn = mock.MagicMock() + mock_cursor = mock_conn.cursor.return_value + mock_cursor.execute.return_value = mock_cursor + # Set side effects for fetchall and description based on snowflake_data fixture + mock_cursor.fetchall.side_effect = [snowflake_data[0][0], snowflake_data[1][0]] + + mock_name_one = MagicMock() + mock_name_one.name = "name" + mock_name_two = MagicMock() + mock_name_two.name = "schema_name" + mock_name_three = MagicMock() + mock_name_three.name = "comment" + + mocked_descriptions = [mock_name_one, mock_name_two, mock_name_three] + mock_cursor.description = mocked_descriptions + + # Call the function to test + result_df = snowflake_connector._fetch_valid_tables_and_views(mock_conn) + + # Assert the result is as expected + pd.testing.assert_frame_equal( + result_df.reset_index(drop=True), expected_df.reset_index(drop=True) + ) + + # Verify execute was called with correct queries + mock_cursor.execute.assert_any_call("show tables in database") + mock_cursor.execute.assert_any_call("show views in database") From d7688122a6d2444598c1d2be552a9072fbfcf780 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Apr 2024 14:43:34 -0700 Subject: [PATCH 14/15] feedback and codeowners --- .github/CODEOWNERS | 1 + README.md | 18 ++++++++++-------- semantic_model_generator/main.py | 6 +++--- 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..f318ba24 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @sfc-gh-nsehrawat @sfc-gh-pverhoeven @sfc-gh-rehuang @sfc-gh-nlimtiaco @sfc-gh-jhilgart diff --git a/README.md b/README.md index 2e6c50d6..2f5b96db 100644 --- a/README.md +++ b/README.md @@ -22,32 +22,34 @@ export SNOWFLAKE_HOST = "" ### Generation -You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. Every table should be a physical table present in your database.schema. +You may generate a semantic model for a given list of fully qualified tables following the `{database}.{schema}.{table}` format. Each table in this list should be a physical table or a view present in your database. All generated semantic models by default are saved under `semantic_model_generator/output_models`. ```bash python -m semantic_model_generator.main \ - --fqn_tables "['..','..']" \ - --semantic_model_name "" \ + --fqn_tables "['..','..']" \ + --semantic_model_name "" \ --snowflake_account="" ``` ### Post-Generation -**Important**: After generation, your YAML files will have a series of lines with `# `. Please take the time to fill these out with your business context. In addition, if there are columns included that are not useful for your internal teams, please remove them from the semantic model. +**Important**: After generation, your YAML files will have a series of lines with `# `. Please take the time to fill these out with your business context. + +By default, the generated semantic model will contain all columns from the provided tables/views. However, it's highly encouraged to only keep relevant columns and drop any unwanted columns from the generated semantic model In addition, consider adding the following elements to your semantic model: -1. Logical columns for a given table. - * Example: `col1 - col2` could be the `expr` for a logical col +1. Logical columns for a given table/view that are expressions over physical columns. + * Example: `col1 - col2` could be the `expr` for a logical column. 2. Synonyms. Any additional synonyms for column names. -3. Metrics. Additional metrics with their relevant `expr`. +3. Filters. Additional filters with their relevant `expr`. ## Release -In order to push a new build and release, follow the below steps. +In order to push a new build and release, follow the steps below. 1. Checkout a new branch from main. You must name this branch `release/vYYYY-MM-DD`. The `release/v` prefix is used to trigger a github workflow post-merge. 2. Bump the poetry: diff --git a/semantic_model_generator/main.py b/semantic_model_generator/main.py index 9219d5e0..008e7667 100644 --- a/semantic_model_generator/main.py +++ b/semantic_model_generator/main.py @@ -97,7 +97,7 @@ def _raw_table_to_semantic_context_table( ) else: raise ValueError( - f"Column datatype does not map to a known datatype. Input was = {col.column_type}. If this is a new datatype, please update the constants in snwoflake_connector.py." + f"Column datatype does not map to a known datatype. Input was = {col.column_type}. If this is a new datatype, please update the constants in snowflake_connector.py." ) return semantic_model_pb2.Table( @@ -193,7 +193,7 @@ def append_comment_to_placeholders(yaml_str: str) -> str: Finds all instances of a specified placeholder in a YAML string and appends a given text to these placeholders. This is the homework to fill out after your yaml is generated. - Args: + Parameters: - yaml_str (str): The YAML string to process. Returns: @@ -247,7 +247,7 @@ def generate_base_semantic_context_from_snowflake( """ Generates a base semantic context from specified Snowflake tables and exports it to a YAML file. - Args: + Parameters: fqn_tables: Fully qualified names of Snowflake tables to include in the semantic context. snowflake_account: Identifier of the Snowflake account. semantic_model_name: The human readable model name. This should be semantically meaningful to an organization. From 4f0c97667801142f6f1e104ba58e3fcaa71d0a76 Mon Sep 17 00:00:00 2001 From: Jonathan Hilgart Date: Tue, 9 Apr 2024 14:44:25 -0700 Subject: [PATCH 15/15] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f318ba24..4ccacf75 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @sfc-gh-nsehrawat @sfc-gh-pverhoeven @sfc-gh-rehuang @sfc-gh-nlimtiaco @sfc-gh-jhilgart +* @sfc-gh-nsehrawat @sfc-gh-rehuang @sfc-gh-nlimtiaco @sfc-gh-jhilgart