tox.ini

[tox]
envlist = py{pypy3,3.7,3.8,3.9,3.10,pyjion},checks,lint,pylint,pyupgrade,isort,black,mypy,docs,coverage,integration-storage
skipsdist = true
requires =
    wheel

[testenv]
passenv =
    TERM
    CI
    GITHUB_*
    DOCKER_*
    FORCE_COLOR
    NO_COLOR
deps =
    -r{toxinidir}/requirements-tests.txt
allowlist_externals =
    cp
    echo
    bash
    /bin/bash
    scripts/*.sh
basepython =
    py3.12-dev: python3.12
    pypypy3: pypy3
    pypypy3.7: pypy3.7
    pypypy-3.7: pypy3.7
    pypypy-3.8: pypy3.8
    pypyjion: pyjion
    {py3.6,py3.6-dist,py3.6-dist-wheel}: python3.6
    {py3.7,py3.7-dist,py3.7-dist-wheel}: python3.7
    {docs,checks,black,black-check,lint,pylint,bandit,mypy,micro-benchmarks,coverage,import-timings,isort,isort-check,pyupgrade}: python3.8
    {py3.8,py3.8-windows,integration-storage,py3.8-dist,py3.8-dist-wheel}: python3.8
    {py3.9,py3.9-dist,py3.9-dist-wheel}: python3.9
    {py3.10,py3.10-dist,py3.10-dist-wheel}: python3.10
    {py3.11,py3.11-dist,py3.11-dist-wheel}: python3.11
    {py3.12-dev,py3.12-dev-dist,py3.12-dev-dist-wheel}: python3.12
setenv =
  CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
# NOTE: By default we run tests on CI in parallel to speed up the build
# To avoid per-test function process safety issues we run all tests in a single
# file in the same worker process.
# for pytest-xdist, we want to distribute tests by file aka --dist loadfile
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
           pytest --color=yes -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 --timeout=15 -n auto --dist loadfile --ignore libcloud/test/benchmarks/ --ignore-glob "*test_list_objects_filtering_performance*"

[testenv:py3.6-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.6-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.7-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.7-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.8-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.8-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.9-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.9-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.10-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.10-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.11-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.11-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:py3.12-dev-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"

[testenv:py3.12-dev-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"

[testenv:docs]
deps =
    -r{toxinidir}/requirements-docs.txt
changedir = docs
commands = rstcheck --report-level warning ../CHANGES.rst
           rstcheck --report-level warning ../CHANGES.rst
           rstcheck --report-level warning ../CONTRIBUTING.rst
           python ../contrib/generate_provider_feature_matrix_table.py
           # TODO: Add -W back when we fix all the warnings in docstrings
           sphinx-build -j auto -b html -d {envtmpdir}/doctrees . _build/html

[testenv:provider-tables]
basepython: python3.8
commands = python ./contrib/generate_provider_feature_matrix_table.py

[testenv:scrape-and-publish-provider-prices]
basepython: python3.8
# Needed to avoid urllib3 errors related to old openssl version
# https://github.com/urllib3/urllib3/issues/2168
deps = urllib3==1.26.6
       requests
       jsonnet
       ijson
       tqdm
setenv =
    PYTHONPATH={toxinidir}
passenv =
    TERM
    GCE_API_KEY
    PRICING_DATA_BUCKET_NAME
    AWS_REGION
    AWS_ACCESS_KEY_ID
    AWS_ACCESS_KEY_SECRET
commands =
    python contrib/scrape-gce-prices.py --all
    python contrib/scrape-gce-prices.py
    python contrib/scrape-ec2-prices.py
    python contrib/scrape-azure-prices.py
    # We also store the SHa512 sum so users can check if something has changed
    # by caching and checking the value of the shasum file
    bash -c "(cd libcloud/data/ ; sha256sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha256)"
    bash -c "(cd libcloud/data/ ; sha512sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha512)"
    python contrib/upload-pricing-data-to-s3.py libcloud/data/pricing.json
    echo ""
    echo "Pricing data should now be available at"
    echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json"
    echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha256"
    echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha512"

[testenv:scrape-provider-prices]
basepython: python3.8
# Needed to avoid urllib3 errors related to old openssl version
# https://github.com/urllib3/urllib3/issues/2168
deps = urllib3==1.26.6
       requests
       jsonnet
       ijson
passenv =
    TERM
    GCE_API_KEY
allowlist_externals = sha512sum
                      bash
commands =
    python contrib/scrape-gce-prices.py --all
    python contrib/scrape-gce-prices.py
    python contrib/scrape-ec2-prices.py
    python contrib/scrape-azure-prices.py
    # We also store the SHa512 sum so users can check if something has changed
    # by caching and checking the value of the shasum file
    bash -c "(cd libcloud/data/ ; sha256sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha256)"
    bash -c "(cd libcloud/data/ ; sha512sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha512)"

[testenv:scrape-ec2-prices]
basepython: python3.7
# Needed to avoid urllib3 errors related to old openssl version
# https://github.com/urllib3/urllib3/issues/2168
deps = urllib3==1.26.6
       requests
       jsonnet
       ijson
       tqdm
commands = python contrib/scrape-ec2-prices.py

[testenv:scrape-ec2-sizes]
basepython: python3.7
# Needed to avoid urllib3 errors related to old openssl version
# https://github.com/urllib3/urllib3/issues/2168
deps = urllib3==1.26.6
       requests
       ijson
       tqdm
commands =
    bash -c 'echo "Scrapping EC2 sizes, this may take up to 10 minutes or more since the actual JSON data we download and scrape is very large"'
    bash -c 'python contrib/scrape-ec2-sizes.py'

[testenv:pylint]
setenv =
    PYTHONPATH={toxinidir}
deps =
    -r{toxinidir}/requirements-lint.txt
commands = pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/common/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/compute/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/container/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/backup/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/dns/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/storage/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml libcloud/utils/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml demos/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml contrib/
           pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./pyproject.toml pylint_plugins/

[testenv:lint]
deps =
    -r{toxinidir}/requirements-lint.txt
commands = flake8 --config ./.flake8 libcloud/
           flake8 --config ./.flake8 libcloud/test/
           flake8 --config ./.flake8 demos/
           flake8 --config ./.flake8 integration/
           flake8 --config ./.flake8 scripts/
           flake8 --config ./.flake8 docs/examples/
           flake8 --config ./.flake8 contrib/
           python -mjson.tool libcloud/data/pricing.json /dev/null
           rstcheck --report-level warning README.rst
           rstcheck --report-level warning CHANGES.rst
           rstcheck --report-level warning CONTRIBUTING.rst
           codespell libcloud/

[testenv:bandit]
deps =
    -r{toxinidir}/requirements-lint.txt
commands = bandit --configfile pyproject.toml -lll -r libcloud/

[testenv:black]
deps =
    -r{toxinidir}/requirements-lint.txt
# NOTE: We need to use bash -c otherwise black will quote the argument and it
# won't expand to the list of files when the command runs
commands =
           bash -c "black --config pyproject.toml *.py"
           black --config pyproject.toml libcloud/
           black --config pyproject.toml docs/examples/
           black --config pyproject.toml docs/
           black --config pyproject.toml demos/
           black --config pyproject.toml contrib/
           black --config pyproject.toml pylint_plugins/
           black --config pyproject.toml integration/

[testenv:black-check]
deps =
    -r{toxinidir}/requirements-lint.txt
# NOTE: We need to use bash -c otherwise black will quote the argument and it
# won't expand to the list of files when the command runs
commands =
           bash -c "black --config pyproject.toml *.py"
           black --config pyproject.toml --check libcloud/
           black --config pyproject.toml --check docs/examples/
           black --config pyproject.toml --check docs/
           black --config pyproject.toml --check demos/
           black --config pyproject.toml --check contrib/
           black --config pyproject.toml --check pylint_plugins/
           black --config pyproject.toml --check integration/

[testenv:checks]
commands =
    bash ./scripts/check_file_names.sh
    python ./scripts/check_asf_license_headers.py .

[testenv:integration-compute]
deps = -r{toxinidir}/integration/compute/requirements.txt

commands = python -m integration.compute

[testenv:integration-storage]
passenv =
    AZURE_CLIENT_SECRET
    AWS_ACCESS_KEY_ID
    AWS_ACCESS_KEY_SECRET
    GITHUB_*

setenv =
  AZURE_CLIENT_ID=16cd65a3-dfa2-4272-bcdb-842cbbedb1b7
  AZURE_TENANT_ID=982317c6-fb7e-4e92-abcd-196557e41c5b
  AZURE_SUBSCRIPTION_ID=d6d608a6-e0c8-42ae-a548-2f41793709d2
  # Actual secret token is defined as part of Github repo secret
deps =
    -r{toxinidir}/requirements-tests.txt
    -r{toxinidir}/integration/storage/requirements.txt
commands = pytest --color=yes -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 integration/storage

[testenv:coverage]
deps =
    -r{toxinidir}/requirements-tests.txt
setenv =
  CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
           coverage run --source=libcloud -m pytest --benchmark-disable

[testenv:coverage-ci]
passenv =
    TERM
    TOXENV
    CI
    GITHUB_*
deps =
    -r{toxinidir}/requirements-tests.txt
setenv =
  CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
           coverage run --source=libcloud -m pytest --benchmark-disable
           coverage xml

[testenv:isort]
deps =
    -r{toxinidir}/requirements-lint.txt
commands =
    isort {toxinidir}

[testenv:isort-check]
deps =
    -r{toxinidir}/requirements-lint.txt
commands =
    isort --check {toxinidir}

[testenv:pyupgrade]
deps =
    -r{toxinidir}/requirements-lint.txt
commands =
    bash -c "find libcloud/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find contrib/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find demos/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find pylint_plugins/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find integration/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find scripts/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"
    bash -c "find docs/examples/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only"

    isort {toxinidir}

[testenv:mypy]
deps =
    -r{toxinidir}/requirements-mypy.txt
commands =
    mypy --no-incremental libcloud/common/
    mypy --no-incremental libcloud/compute/
    mypy --no-incremental libcloud/storage/
    mypy --no-incremental libcloud/dns/
    mypy --no-incremental libcloud/container/
    mypy --no-incremental example_compute.py
    mypy --no-incremental example_storage.py
    mypy --no-incremental example_dns.py
    mypy --no-incremental example_container.py

[testenv:micro-benchmarks]
commands =
    cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
    pytest --color=yes -s -v --timeout 60 --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-histogram=benchmark_histograms/benchmark  --benchmark-group-by=group,param:sort_objects libcloud/test/benchmarks/test_list_objects_filtering_performance.py
    pytest --color=yes -s -v --timeout 60 --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-histogram=benchmark_histograms/benchmark --benchmark-group-by=group,func,param:read_in_chunks_func libcloud/test/benchmarks/test_read_in_chunks.py

[testenv:import-timings]
setenv =
    PYTHONPATH={toxinidir}
deps = requests
       profimp==0.1.0
commands =
    bash -c "./scripts/time_imports.sh"