diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..41066c8 --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,25 @@ +# Autogenerated. Do not edit this by hand, use `copier update`. +--- +_commit: 0.2.6 +_src_path: https://github.com/lkubb/salt-extension-copier +author: EITR Technologies, LLC +author_email: eitr@devops.tech +docs_url: '' +license: apache +loaders: + - cache + - module + - pillar + - sdb + - state +max_salt_version: 3006 +no_saltext_namespace: false +package_name: consul +project_name: consul +python_requires: '3.8' +salt_version: '3005' +source_url: https://github.com/salt-extensions/saltext-consul +ssh_fixtures: false +summary: Salt Extension for interacting with Consul +tracker_url: https://github.com/salt-extensions/saltext-consul/issues +url: https://github.com/salt-extensions/saltext-consul diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..879568e --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,11 @@ +### Description of Issue + + +### Setup +(Please provide relevant configs and/or SLS files (Be sure to remove sensitive info).) + +### Steps to Reproduce Issue +(Include debug logs if possible and relevant.) + +### Versions Report +(Provided by running `salt --versions-report`. Please also mention any differences in master/minion versions.) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b232a42 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,48 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug, needs-triage +assignees: '' + +--- + +**Description** +A clear and concise description of what the bug is. + +**Setup** +(Please provide relevant configs and/or SLS files (be sure to remove sensitive info. There is no general set-up of Salt.) + +Please be as specific as possible and give set-up details. + +- [ ] on-prem machine +- [ ] VM (Virtualbox, KVM, etc. please specify) +- [ ] VM running on a cloud service, please be explicit and add details +- [ ] container (Kubernetes, Docker, containerd, etc. please specify) +- [ ] or a combination, please be explicit +- [ ] jails if it is FreeBSD +- [ ] classic packaging +- [ ] onedir packaging +- [ ] used bootstrap to install + + +**Steps to Reproduce the behavior** +(Include debug logs if possible and relevant) + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Versions Report** +
salt --versions-report +(Provided by running salt --versions-report. Please also mention any differences in master/minion versions.) + +```yaml +PASTE HERE +``` +
+ +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4cf0d28 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: true +contact_links: + - name: Salt Community Slack + url: https://saltstackcommunity.slack.com/ + about: Please ask and answer questions here. + - name: Salt-Users Forum + url: https://groups.google.com/forum/#!forum/salt-users + about: Please ask and answer questions here. + - name: Salt on LiberaChat + url: https://web.libera.chat/#salt + about: Please ask and answer questions here. + - name: Security vulnerabilities + email: saltproject-security.pdl@broadcom.com + about: Please report security vulnerabilities here. diff --git a/.github/ISSUE_TEMPLATE/docs.md b/.github/ISSUE_TEMPLATE/docs.md new file mode 100644 index 0000000..59af749 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/docs.md @@ -0,0 +1,23 @@ +--- +name: Docs +about: Issue related to Salt Documentation +title: "[DOCS]" +labels: documentation, needs-triage +assignees: '' + +--- + +**Description** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Suggested Fix** +What did you expect to see in the documentation that is missing or needs updating? + +**Type of documentation** +This could be module documentation or a guide. + +**Location or format of documentation** +Insert page URL if applicable. + +**Additional context** +Add any other context or screenshots here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..0e0a390 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE REQUEST]" +labels: feature, needs-triage +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/tech-debt.md b/.github/ISSUE_TEMPLATE/tech-debt.md new file mode 100644 index 0000000..5a546c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/tech-debt.md @@ -0,0 +1,13 @@ +--- +name: Tech Debt +about: Issue is related to tech debt. This includes compatibility changes for newer versions of software and OSes that salt interacts with. +title: "[TECH DEBT]" +labels: tech-debt +assignees: '' + +--- + +### Description of the tech debt to be addressed, include links and screenshots + +### Versions Report +(Provided by running `salt --versions-report`. Please also mention any differences in master/minion versions.) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7d63a7c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +### What does this PR do? + +### What issues does this PR fix or reference? +Fixes: + +### Previous Behavior +Remove this section if not relevant + +### New Behavior +Remove this section if not relevant + +### Merge requirements satisfied? +**[NOTICE] Bug fixes or features added to Salt require tests.** + +- [ ] Docs +- [ ] Changelog - https://docs.saltproject.io/en/master/topics/development/changelog.html +- [ ] Tests written/updated + +### Commits signed with GPG? +Yes/No + +Please review [Salt's Contributing Guide](https://docs.saltproject.io/en/master/topics/development/contributing.html) for best practices. + +See GitHub's [page on GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) for more information about signing commits with GPG. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..c612d7a --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,11 @@ +name: Pull Request or Push + +on: [push, pull_request] + +jobs: + ci: + name: CI + uses: salt-extensions/central-artifacts/.github/workflows/ci.yml@main + permissions: + contents: write + pull-requests: read diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 0000000..9f86b4e --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,26 @@ +name: Tagged Releases + +on: + push: + tags: + - "v*" + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Extract tag name + id: get_version + run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/v})" >> $GITHUB_ENV + + - name: CI + uses: salt-extensions/central-artifacts/.github/workflows/ci.yml@main + with: + release: true + version: ${{ env.VERSION }} + permissions: + contents: write + pull-requests: read diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 6e535cb..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,417 +0,0 @@ - -name: Testing - -on: [push, pull_request] - -jobs: - Pre-Commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set Cache Key - run: echo "PY=$(python --version --version | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - name: Install System Deps - run: | - sudo apt-get update - sudo apt-get install -y libxml2 libxml2-dev libxslt-dev - - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - uses: pre-commit/action@v1.0.1 - - Docs: - runs-on: ubuntu-latest - needs: Pre-Commit - - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 For Nox - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Install Nox - run: | - python -m pip install --upgrade pip - pip install nox - - - name: Install Doc Requirements - run: | - nox --force-color -e docs --install-only - - - name: Build Docs - env: - SKIP_REQUIREMENTS_INSTALL: YES - run: | - nox --force-color -e docs - - Linux: - runs-on: ubuntu-latest - needs: Pre-Commit - - timeout-minutes: 15 - - strategy: - fail-fast: false - max-parallel: 4 - matrix: - python-version: - - 3.6 - - 3.7 - - 3.8 - salt-version: - - 3003 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Nox - run: | - python -m pip install --upgrade pip - pip install nox - - - name: Install Test Requirements - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - run: | - nox --force-color -e tests-3 --install-only - - - name: Test - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - SKIP_REQUIREMENTS_INSTALL: YES - run: | - nox --force-color -e tests-3 -- -vv tests/ - - - name: Create CodeCov Flags - if: always() - id: codecov-flags - run: | - echo ::set-output name=flags::$(python -c "import sys; print('{},{},salt_{}'.format('${{ runner.os }}'.replace('-latest', ''), 'py{}{}'.format(*sys.version_info), '_'.join(str(v) for v in '${{ matrix.salt-version }}'.replace('==', '_').split('.'))))") - - - name: Upload Project Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},project - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-project - REPORT_PATH: artifacts/coverage-project.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Tests Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},tests - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-tests - REPORT_PATH: artifacts/coverage-tests.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Logs - if: always() - uses: actions/upload-artifact@main - with: - name: runtests-${{ runner.os }}-py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}.log - path: artifacts/runtests-*.log - - Windows: - runs-on: windows-latest - needs: Pre-Commit - - timeout-minutes: 40 - - strategy: - fail-fast: false - max-parallel: 3 - matrix: - python-version: - - 3.6 - - 3.7 - salt-version: - - 3003 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Nox - run: | - python -m pip install --upgrade pip - pip install nox - - - name: Install Test Requirements - shell: bash - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - EXTRA_REQUIREMENTS_INSTALL: Cython - run: | - export PATH="/C/Program Files (x86)/Windows Kits/10/bin/10.0.18362.0/x64;$PATH" - nox --force-color -e tests-3 --install-only - - - name: Test - shell: bash - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - SKIP_REQUIREMENTS_INSTALL: YES - run: | - export PATH="/C/Program Files (x86)/Windows Kits/10/bin/10.0.18362.0/x64;$PATH" - nox --force-color -e tests-3 -- -vv tests/ - - - name: Create CodeCov Flags - if: always() - id: codecov-flags - run: | - echo ::set-output name=flags::$(python -c "import sys; print('{},{},salt_{}'.format('${{ runner.os }}'.replace('-latest', ''), 'py{}{}'.format(*sys.version_info), '_'.join(str(v) for v in '${{ matrix.salt-version }}'.replace('==', '_').split('.'))))") - - - name: Upload Project Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},project - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-project - REPORT_PATH: artifacts/coverage-project.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Tests Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},tests - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-tests - REPORT_PATH: artifacts/coverage-tests.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Logs - if: always() - uses: actions/upload-artifact@main - with: - name: runtests-${{ runner.os }}-py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}.log - path: artifacts/runtests-*.log - - macOS: - runs-on: macOS-latest - needs: Pre-Commit - - timeout-minutes: 40 - - strategy: - fail-fast: false - max-parallel: 3 - matrix: - python-version: - - 3.6 - - 3.7 - salt-version: - - 3003 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Nox - run: | - python -m pip install --upgrade pip - pip install nox - - - name: Install Test Requirements - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - run: | - nox --force-color -e tests-3 --install-only - - - name: Test - env: - SALT_REQUIREMENT: salt==${{ matrix.salt-version }} - SKIP_REQUIREMENTS_INSTALL: YES - run: | - nox --force-color -e tests-3 -- -vv tests/ - - - name: Create CodeCov Flags - if: always() - id: codecov-flags - run: | - echo ::set-output name=flags::$(python -c "import sys; print('{},{},salt_{}'.format('${{ runner.os }}'.replace('-latest', ''), 'py{}{}'.format(*sys.version_info), '_'.join(str(v) for v in '${{ matrix.salt-version }}'.replace('==', '_').split('.'))))") - - - name: Upload Project Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},project - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-project - REPORT_PATH: artifacts/coverage-project.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Tests Code Coverage - if: always() - shell: bash - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},tests - REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-tests - REPORT_PATH: artifacts/coverage-tests.xml - run: | - if [ ! -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - if [ -f codecov.sh ]; then - n=0 - until [ "$n" -ge 5 ] - do - if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then - break - fi - n=$((n+1)) - sleep 15 - done - fi - - - name: Upload Logs - if: always() - uses: actions/upload-artifact@main - with: - name: runtests-${{ runner.os }}-py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}.log - path: artifacts/runtests-*.log diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml old mode 100644 new mode 100755 index 561a985..59482db --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ minimum_pre_commit_version: 2.4.0 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.4.0 hooks: - id: check-merge-conflict # Check for files that contain merge conflict strings. - id: trailing-whitespace # Trims trailing whitespace. @@ -37,7 +37,7 @@ repos: - repo: https://github.com/s0undt3ch/salt-rewrite # Automatically rewrite code with known rules - rev: 1.3.3 + rev: 2.5.2 hooks: - id: salt-rewrite alias: rewrite-docstrings @@ -47,7 +47,7 @@ repos: - repo: https://github.com/s0undt3ch/salt-rewrite # Automatically rewrite code with known rules - rev: 1.3.3 + rev: 2.5.2 hooks: - id: salt-rewrite alias: rewrite-tests @@ -59,18 +59,18 @@ repos: rev: v2.37.2 hooks: - id: pyupgrade - name: Rewrite Code to be Py3.6+ + name: Rewrite Code to be Py3.8+ args: [ - --py36-plus + --py38-plus ] exclude: src/saltext/consul/version.py - repo: https://github.com/asottile/reorder_python_imports - rev: v2.6.0 + rev: v3.10.0 hooks: - id: reorder-python-imports args: [ - --py3-plus, + --py38-plus, ] exclude: src/saltext/consul/version.py @@ -80,10 +80,8 @@ repos: - id: black args: [-l 100] exclude: src/saltext/consul/version.py - additional_dependencies: - - click<8.1.0 - - repo: https://github.com/asottile/blacken-docs + - repo: https://github.com/adamchainz/blacken-docs rev: v1.12.1 hooks: - id: blacken-docs @@ -91,12 +89,11 @@ repos: files: ^(docs/.*\.rst|src/saltext/consul/.*\.py)$ additional_dependencies: - black==22.6.0 - - click<8.1.0 # <---- Formatting ----------------------------------------------------------------------------- # ----- Security ------------------------------------------------------------------------------> - repo: https://github.com/PyCQA/bandit - rev: "1.7.0" + rev: "1.7.4" hooks: - id: bandit alias: bandit-salt @@ -104,7 +101,7 @@ repos: args: [--silent, -lll, --skip, B701] exclude: src/saltext/consul/version.py - repo: https://github.com/PyCQA/bandit - rev: "1.7.0" + rev: "1.7.4" hooks: - id: bandit alias: bandit-tests @@ -121,6 +118,7 @@ repos: alias: lint-src name: Lint Source Code files: ^((setup|noxfile)|src/.*)\.py$ + require_serial: true args: - -e - lint-code-pre-commit @@ -133,149 +131,9 @@ repos: alias: lint-tests name: Lint Tests files: ^tests/.*\.py$ + require_serial: true args: - -e - lint-tests-pre-commit - -- # <---- Code Analysis -------------------------------------------------------------------------- - - # ----- Static Test Requirements --------------------------------------------------------------> - - repo: https://github.com/saltstack/pip-tools-compile-impersonate - rev: '4.1' - hooks: - - id: pip-tools-compile - alias: compile-3.6-test-requirements - name: Py3.6 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.6 - - --platform=linux - - requirements/tests.in - - - id: pip-tools-compile - alias: compile-3.7-test-requirements - name: Py3.7 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.7 - - --platform=linux - - requirements/tests.in - - - id: pip-tools-compile - alias: compile-3.8-test-requirements - name: Py3.8 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - requirements/tests.in - - - id: pip-tools-compile - alias: compile-3.9-test-requirements - name: Py3.9 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.9 - - --platform=linux - - requirements/tests.in - # <---- Static Test Requirements --------------------------------------------------------------- - - # ----- Static Lint Requirements --------------------------------------------------------------> - - id: pip-tools-compile - alias: compile-3.6-test-requirements - name: Py3.6 Lint Requirements - files: ^requirements/lint.in$ - pass_filenames: false - args: - - -v - - --py-version=3.6 - - --platform=linux - - requirements/lint.in - - - id: pip-tools-compile - alias: compile-3.7-test-requirements - name: Py3.7 Lint Requirements - files: ^requirements/lint.in$ - pass_filenames: false - args: - - -v - - --py-version=3.7 - - --platform=linux - - requirements/lint.in - - - id: pip-tools-compile - alias: compile-3.8-test-requirements - name: Py3.8 Lint Requirements - files: ^requirements/lint.in$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - requirements/lint.in - - - id: pip-tools-compile - alias: compile-3.9-test-requirements - name: Py3.9 Lint Requirements - files: ^requirements/lint.in$ - pass_filenames: false - args: - - -v - - --py-version=3.9 - - --platform=linux - - requirements/lint.in - # <---- Static Lint Requirements --------------------------------------------------------------- - - # ----- Static Docs Requirements --------------------------------------------------------------> - - id: pip-tools-compile - alias: compile-3.6-test-requirements - name: Py3.6 Docs Requirements - files: ^requirements/docs.in$ - pass_filenames: false - args: - - -v - - --py-version=3.6 - - --platform=linux - - requirements/docs.in - - - id: pip-tools-compile - alias: compile-3.7-test-requirements - name: Py3.7 Docs Requirements - files: ^requirements/docs.in$ - pass_filenames: false - args: - - -v - - --py-version=3.7 - - --platform=linux - - requirements/docs.in - - - id: pip-tools-compile - alias: compile-3.8-test-requirements - name: Py3.8 Docs Requirements - files: ^requirements/docs.in$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - requirements/docs.in - - - id: pip-tools-compile - alias: compile-3.9-test-requirements - name: Py3.9 Docs Requirements - files: ^requirements/docs.in$ - pass_filenames: false - args: - - -v - - --py-version=3.9 - - --platform=linux - - requirements/docs.in - # <---- Static Docs Requirements --------------------------------------------------------------- diff --git a/.pre-commit-hooks/check-cli-examples.py b/.pre-commit-hooks/check-cli-examples.py index b34d329..c060c26 100644 --- a/.pre-commit-hooks/check-cli-examples.py +++ b/.pre-commit-hooks/check-cli-examples.py @@ -4,7 +4,7 @@ import sys CODE_ROOT = pathlib.Path(__file__).resolve().parent.parent -EXECUTION_MODULES_PATH = CODE_ROOT / "src" / "saltext" / " consul" / "modules" +EXECUTION_MODULES_PATH = CODE_ROOT / "src" / "saltext" / "consul" / "modules" def check_cli_examples(files): diff --git a/.pre-commit-hooks/make-autodocs.py b/.pre-commit-hooks/make-autodocs.py index 700af6f..c4a8797 100644 --- a/.pre-commit-hooks/make-autodocs.py +++ b/.pre-commit-hooks/make-autodocs.py @@ -1,59 +1,119 @@ +import ast +import os.path import subprocess -import sys -from enum import IntEnum from pathlib import Path repo_path = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()) -src_dir = repo_path / "src" / " saltext" / "consul" +src_dir = repo_path / "src" / "saltext" / "consul" doc_dir = repo_path / "docs" docs_by_kind = {} +changed_something = False + + +def _find_virtualname(path): + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "__virtualname__": + if isinstance(node.value, ast.Str): + virtualname = node.value.s + break + else: + continue + break + else: + virtualname = path.with_suffix("").name + return virtualname + + +def write_module(rst_path, path, use_virtualname=True): + if use_virtualname: + virtualname = "``" + _find_virtualname(path) + "``" + else: + virtualname = make_import_path(path) + module_contents = f"""\ +{virtualname} +{'='*len(virtualname)} + +.. automodule:: {make_import_path(path)} + :members: +""" + if not rst_path.exists() or rst_path.read_text() != module_contents: + print(rst_path) + rst_path.write_text(module_contents) + return True + return False + + +def write_index(index_rst, import_paths, kind): + if kind == "utils": + header_text = "Utilities" + common_path = os.path.commonpath(tuple(x.replace(".", "/") for x in import_paths)).replace( + "/", "." + ) + if any(x == common_path for x in import_paths): + common_path = common_path[: common_path.rfind(".")] + else: + header_text = ( + "execution modules" if kind.lower() == "modules" else kind.rstrip("s") + " modules" + ) + common_path = import_paths[0][: import_paths[0].rfind(".")] + header = f"{'_'*len(header_text)}\n{header_text.title()}\n{'_'*len(header_text)}" + index_contents = f"""\ +.. all-saltext.consul.{kind}: + +{header} + +.. currentmodule:: {common_path} + +.. autosummary:: + :toctree: + +{chr(10).join(sorted(' '+p[len(common_path)+1:] for p in import_paths))} +""" + if not index_rst.exists() or index_rst.read_text() != index_contents: + print(index_rst) + index_rst.write_text(index_contents) + return True + return False def make_import_path(path): - return ".".join(path.with_suffix("").parts[-4:]) + if path.name == "__init__.py": + path = path.parent + return ".".join(path.relative_to(repo_path / "src").with_suffix("").parts) -for path in Path(__file__).parent.parent.joinpath("src/saltext/consul/").glob("*/*.py"): +for path in src_dir.glob("*/*.py"): if path.name != "__init__.py": kind = path.parent.name - docs_by_kind.setdefault(kind, set()).add(path) + if kind != "utils": + docs_by_kind.setdefault(kind, set()).add(path) + +# Utils can have subdirectories, treat them separately +for path in (src_dir / "utils").rglob("*.py"): + if path.name == "__init__.py" and not path.read_text(): + continue + docs_by_kind.setdefault("utils", set()).add(path) for kind in docs_by_kind: kind_path = doc_dir / "ref" / kind - all_rst = kind_path / "all.rst" + index_rst = kind_path / "index.rst" import_paths = [] for path in sorted(docs_by_kind[kind]): import_path = make_import_path(path) import_paths.append(import_path) - rst_path = kind_path.joinpath(import_path).with_suffix(".rst") - print(rst_path) + rst_path = kind_path / (import_path + ".rst") rst_path.parent.mkdir(parents=True, exist_ok=True) - rst_path.write_text( - f""" -{import_path} -{'='*len(import_path)} - -.. automodule:: {import_path} - :members: -""" - ) + change = write_module(rst_path, path, use_virtualname=kind != "utils") + changed_something = changed_something or change - header_text = ( - "execution modules" if kind.lower() == "modules" else kind.rstrip("s") + " modules" - ) - header = f"{'_'*len(header_text)}\n{header_text.title()}\n{'_'*len(header_text)}" + write_index(index_rst, import_paths, kind) - all_rst.write_text( - f""" -.. all-saltext.consul.{kind}: - -{header} -.. autosummary:: - :toctree: - -{chr(10).join(sorted(' '+p for p in import_paths))} -""" - ) +# Ensure pre-commit realizes we did something +if changed_something: + exit(2) diff --git a/.pylintrc b/.pylintrc old mode 100644 new mode 100755 index 7fb0a22..5692f3b --- a/.pylintrc +++ b/.pylintrc @@ -72,7 +72,7 @@ ignored-modules= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use, and will cap the count on Windows to # avoid hangs. -jobs=1 +jobs=0 # Control the amount of potential inferred values when inferring a single # object. This can help the performance when dealing with large functions or diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..aec3363 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +This project uses [Semantic Versioning](https://semver.org/) - MAJOR.MINOR.PATCH + +# Changelog diff --git a/LICENSE b/LICENSE index a92dcb5..cbfdcf2 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2023 EITR Technologies + Copyright 2024 EITR Technologies, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index b276b66..952de5a 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,86 @@ -# consul +# Salt Extension for Consul Salt Extension for interacting with Consul -## Quickstart +## Security -To get started with your new project: +If you think you've found a security vulnerability, see +[Salt's security guide][security]. - # Create a new venv - python3 -m venv env --prompt consul - source env/bin/activate +## User Documentation - # On mac, you may need to upgrade pip - python -m pip install --upgrade pip +This README is for people aiming to contribute to the project. +If you just want to get started with the extension, check out the +module docstrings (for now, documentation is coming!). - # On WSL or some flavors of linux you may need to install the `enchant` - # library in order to build the docs - sudo apt-get install -y enchant +## Contributing - # Install extension + test/dev/doc dependencies into your environment - python -m pip install -e .[tests,dev,docs] +The saltext-consul project team welcomes contributions from the community. - # Run tests! - python -m nox -e tests-3 +The [Salt Contributing guide][salt-contributing] has a lot of relevant +information, but if you'd like to jump right in here's how to get started: - # skip requirements install for next time - export SKIP_REQUIREMENTS_INSTALL=1 - # Build the docs, serve, and view in your web browser: - python -m nox -e docs && (cd docs/_build/html; python -m webbrowser localhost:8000; python -m http.server; cd -) +```bash +# Clone the repo +git clone --origin salt git@github.com:salt-extensions/saltext-consul.git - # Run the example function - salt-call --local consul.example_function text="Happy Hacking!" +# Change to the repo dir +cd saltext-consul + +# Create a new venv +python3 -m venv env --prompt saltext-consul +source env/bin/activate + +# On mac, you may need to upgrade pip +python -m pip install --upgrade pip + +# On WSL or some flavors of linux you may need to install the `enchant` +# library in order to build the docs +sudo apt-get install -y enchant + +# Install extension + test/dev/doc dependencies into your environment +python -m pip install -e '.[tests,dev,docs]' + +# Run tests! +python -m nox -e tests-3 + +# skip requirements install for next time +export SKIP_REQUIREMENTS_INSTALL=1 + +# Build the docs, serve, and view in your web browser: +python -m nox -e docs && (cd docs/_build/html; python -m webbrowser localhost:8000; python -m http.server; cd -) +``` + +Writing code isn't the only way to contribute! We value contributions in any of +these areas: + +* Documentation - especially examples of how to use this module to solve + specific problems. +* Triaging [issues][issues] and participating in [discussions][discussions] +* Reviewing [Pull Requests][PRs] (we really like + [Conventional Comments][comments]!) + +You could also contribute in other ways: + +* Writing blog posts +* Posting on social media about how you used Salt+Consul to solve your + problems, including videos +* Giving talks at conferences +* Publishing videos +* Asking/answering questions in IRC, Slack, or email groups + +Any of these things are super valuable to our community, and we sincerely +appreciate every contribution! + + +For more information, build the docs and head over to http://localhost:8000/ — +that's where you'll find the rest of the documentation. + + +[security]: https://github.com/saltstack/salt/blob/master/SECURITY.md +[salt-contributing]: https://docs.saltproject.io/en/master/topics/development/contributing.html +[issues]: https://github.com/salt-extensions/saltext-consul/issues +[PRs]: https://github.com/salt-extensions/saltext-consul/pulls +[discussions]: https://github.com/salt-extensions/saltext-consul/discussions +[comments]: https://conventionalcomments.org/ diff --git a/changelog/.template.jinja b/changelog/.template.jinja new file mode 100644 index 0000000..0cf429a --- /dev/null +++ b/changelog/.template.jinja @@ -0,0 +1,15 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} + +### {{ definitions[category]['name'] }} + +{% for text, values in sections[""][category].items() %} +- {{ text }} {{ values|join(', ') }} +{% endfor %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} diff --git a/docs/all.rst b/docs/all.rst deleted file mode 100644 index 3cacbfc..0000000 --- a/docs/all.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. _all the states/modules: - -Complete List of consul -======================= - - -.. toctree:: - :maxdepth: 2 - - ref/modules.rst - - -.. toctree:: - :maxdepth: 2 - - ref/states.rst diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..2f5367b --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,12 @@ +# Changelog + +The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +This project uses [Semantic Versioning](https://semver.org/) - MAJOR.MINOR.PATCH + +```{towncrier-draft-entries} +``` + +```{include} ../CHANGELOG.md +:start-after: '# Changelog' +``` diff --git a/docs/conf.py b/docs/conf.py old mode 100644 new mode 100755 index 16f122d..01c5561 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,8 +9,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import datetime +import email.policy import os import sys +from pathlib import Path try: from importlib_metadata import distribution @@ -25,6 +27,8 @@ # assume we're in the doc/ directory docs_basepath = os.path.abspath(os.path.dirname(".")) +PROJECT_ROOT_DIR = Path(docs_basepath).parent + addtl_paths = ( os.path.join(os.pardir, "src"), # saltext.consul itself (for autodoc) "_ext", # custom Sphinx extensions @@ -44,6 +48,19 @@ copyright_year = f"2021 - {this_year}" project = dist.metadata["Summary"] author = dist.metadata["Author"] + +if author is None: + # Core metadata is serialized differently with pyproject.toml: + # https://packaging.python.org/en/latest/specifications/pyproject-toml/#authors-maintainers + author_email = dist.metadata["Author-email"] + em = email.message_from_string( + f"To: {author_email}", + policy=email.policy.default, + ) + if em["To"].addresses and em["To"].addresses[0]: + author = em["To"].addresses[0].display_name + author = author or "" + copyright = f"{copyright_year}, {author}" # The full version, including alpha/beta/rc tags @@ -62,6 +79,8 @@ # -- General configuration --------------------------------------------------- +linkcheck_ignore = [r"http://localhost:\d+"] + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. @@ -75,6 +94,15 @@ "sphinx.ext.coverage", "sphinx_copybutton", "sphinxcontrib.spelling", + "sphinxcontrib.towncrier.ext", + "myst_parser", + "sphinx_inline_tabs", +] + +myst_enable_extensions = [ + "colon_fence", + "deflist", + "tasklist", ] # Add any paths that contain templates here, relative to this directory. @@ -95,7 +123,7 @@ "sitevars.rst", ] -autosummary_generate = True +autosummary_generate = False # -- Options for HTML output ------------------------------------------------- @@ -136,7 +164,7 @@ # ----- Intersphinx Config ----------------------------------------------------------------------------------------> intersphinx_mapping = { "python": ("https://docs.python.org/3", None), - "pytest": ("https://pytest.readthedocs.io/en/stable", None), + "pytest": ("https://docs.pytest.org/en/stable", None), "salt": ("https://docs.saltproject.io/en/latest", None), } # <---- Intersphinx Config ----------------------------------------------------------------------------------------- @@ -146,6 +174,11 @@ autodoc_mock_imports = ["salt"] # <---- Autodoc Config ----------------------------------------------------------------------------------------------- +# Towncrier draft config +towncrier_draft_autoversion_mode = "sphinx-release" +towncrier_draft_include_empty = True +towncrier_draft_working_directory = str(PROJECT_ROOT_DIR) + def setup(app): app.add_crossref_type( diff --git a/docs/index.rst b/docs/index.rst index 004c647..f20794d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,11 +1,33 @@ -Welcome to consul Documentation! -================================ +``saltext-consul``: Integrate Salt with Consul +============================================== + +Salt Extension for interacting with Consul + +.. toctree:: + :maxdepth: 2 + :caption: Guides + :hidden: + + topics/installation .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: Provided Modules + :hidden: + + ref/cache/index + ref/modules/index + ref/pillar/index + ref/sdb/index + ref/states/index + +.. toctree:: + :maxdepth: 2 + :caption: Reference + :hidden: + + changelog - all.rst Indices and tables ================== diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst deleted file mode 100644 index d28bcc0..0000000 --- a/docs/ref/cache/all/salt.cache.consul.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.cache.consul -================= - -.. automodule:: salt.cache.consul - :members: diff --git a/docs/ref/cache/all.rst b/docs/ref/cache/index.rst similarity index 67% rename from docs/ref/cache/all.rst rename to docs/ref/cache/index.rst index a61e639..f97ece0 100644 --- a/docs/ref/cache/all.rst +++ b/docs/ref/cache/index.rst @@ -1,11 +1,12 @@ - .. all-saltext.consul.cache: _____________ Cache Modules _____________ +.. currentmodule:: saltext.consul.cache + .. autosummary:: :toctree: - saltext.consul.cache.consul + consul diff --git a/docs/ref/cache/saltext.consul.cache.rst b/docs/ref/cache/saltext.consul.cache.consul.rst similarity index 50% rename from docs/ref/cache/saltext.consul.cache.rst rename to docs/ref/cache/saltext.consul.cache.consul.rst index 71d97c9..651b353 100644 --- a/docs/ref/cache/saltext.consul.cache.rst +++ b/docs/ref/cache/saltext.consul.cache.consul.rst @@ -1,6 +1,5 @@ - -saltext.consul.cache.consul -=========================== +``consul`` +========== .. automodule:: saltext.consul.cache.consul :members: diff --git a/docs/ref/modules/all/salt.modules.consul.rst b/docs/ref/modules/all/salt.modules.consul.rst deleted file mode 100644 index 132ec97..0000000 --- a/docs/ref/modules/all/salt.modules.consul.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.modules.consul -=================== - -.. automodule:: salt.modules.consul - :members: diff --git a/docs/ref/modules/all.rst b/docs/ref/modules/index.rst similarity index 63% rename from docs/ref/modules/all.rst rename to docs/ref/modules/index.rst index 5f5ce22..09009cb 100644 --- a/docs/ref/modules/all.rst +++ b/docs/ref/modules/index.rst @@ -1,11 +1,13 @@ - .. all-saltext.consul.modules: _________________ Execution Modules _________________ +.. currentmodule:: saltext.consul.modules + .. autosummary:: :toctree: - saltext.consul.modules.consul + consul + consul_mod diff --git a/docs/ref/modules/saltext.consul.modules.rst b/docs/ref/modules/saltext.consul.modules.consul.rst similarity index 50% rename from docs/ref/modules/saltext.consul.modules.rst rename to docs/ref/modules/saltext.consul.modules.consul.rst index 84549d0..fbeed27 100644 --- a/docs/ref/modules/saltext.consul.modules.rst +++ b/docs/ref/modules/saltext.consul.modules.consul.rst @@ -1,6 +1,5 @@ - -saltext.consul.modules.consul -============================= +``consul`` +========== .. automodule:: saltext.consul.modules.consul :members: diff --git a/docs/ref/modules/saltext.consul.modules.consul_mod.rst b/docs/ref/modules/saltext.consul.modules.consul_mod.rst new file mode 100644 index 0000000..4277ad6 --- /dev/null +++ b/docs/ref/modules/saltext.consul.modules.consul_mod.rst @@ -0,0 +1,5 @@ +``consul`` +========== + +.. automodule:: saltext.consul.modules.consul_mod + :members: diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst deleted file mode 100644 index 29e95b5..0000000 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.pillar.consul_pillar -========================= - -.. automodule:: salt.pillar.consul_pillar - :members: diff --git a/docs/ref/pillar/all.rst b/docs/ref/pillar/index.rst similarity index 65% rename from docs/ref/pillar/all.rst rename to docs/ref/pillar/index.rst index 8f1b8fe..2abd7ad 100644 --- a/docs/ref/pillar/all.rst +++ b/docs/ref/pillar/index.rst @@ -1,11 +1,12 @@ - .. all-saltext.consul.pillar: ______________ Pillar Modules ______________ +.. currentmodule:: saltext.consul.pillar + .. autosummary:: :toctree: - saltext.consul.pillar.consul_pillar + consul_pillar diff --git a/docs/ref/pillar/saltext.consul.pillar.consul_pillar.rst b/docs/ref/pillar/saltext.consul.pillar.consul_pillar.rst new file mode 100644 index 0000000..9a6ef9d --- /dev/null +++ b/docs/ref/pillar/saltext.consul.pillar.consul_pillar.rst @@ -0,0 +1,5 @@ +``consul`` +========== + +.. automodule:: saltext.consul.pillar.consul_pillar + :members: diff --git a/docs/ref/pillar/saltext.consul.pillar.rst b/docs/ref/pillar/saltext.consul.pillar.rst deleted file mode 100644 index 3e99c2d..0000000 --- a/docs/ref/pillar/saltext.consul.pillar.rst +++ /dev/null @@ -1,6 +0,0 @@ - -saltext.consul.pillar.consul_pillar -=================================== - -.. automodule:: saltext.consul.pillar.consul_pillar - :members: diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst deleted file mode 100644 index 043a783..0000000 --- a/docs/ref/sdb/all/salt.sdb.consul.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.sdb.consul -=============== - -.. automodule:: salt.sdb.consul - :members: diff --git a/docs/ref/sdb/all.rst b/docs/ref/sdb/index.rst similarity index 66% rename from docs/ref/sdb/all.rst rename to docs/ref/sdb/index.rst index a57f51c..3c456f2 100644 --- a/docs/ref/sdb/all.rst +++ b/docs/ref/sdb/index.rst @@ -1,11 +1,12 @@ - .. all-saltext.consul.sdb: ___________ Sdb Modules ___________ +.. currentmodule:: saltext.consul.sdb + .. autosummary:: :toctree: - saltext.consul.sdb.consul + consul diff --git a/docs/ref/sdb/saltext.consul.sdb.rst b/docs/ref/sdb/saltext.consul.sdb.consul.rst similarity index 51% rename from docs/ref/sdb/saltext.consul.sdb.rst rename to docs/ref/sdb/saltext.consul.sdb.consul.rst index 53458b2..2eee30b 100644 --- a/docs/ref/sdb/saltext.consul.sdb.rst +++ b/docs/ref/sdb/saltext.consul.sdb.consul.rst @@ -1,6 +1,5 @@ - -saltext.consul.sdb.consul -========================= +``consul`` +========== .. automodule:: saltext.consul.sdb.consul :members: diff --git a/docs/ref/states/all/salt.states.consul.rst b/docs/ref/states/all/salt.states.consul.rst deleted file mode 100644 index a28a69c..0000000 --- a/docs/ref/states/all/salt.states.consul.rst +++ /dev/null @@ -1,6 +0,0 @@ -================== -salt.states.consul -================== - -.. automodule:: salt.states.consul - :members: diff --git a/docs/ref/states/all.rst b/docs/ref/states/index.rst similarity index 61% rename from docs/ref/states/all.rst rename to docs/ref/states/index.rst index 682fb92..f93509a 100644 --- a/docs/ref/states/all.rst +++ b/docs/ref/states/index.rst @@ -1,11 +1,13 @@ - .. all-saltext.consul.states: _____________ State Modules _____________ +.. currentmodule:: saltext.consul.states + .. autosummary:: :toctree: - saltext.consul.states.consul + consul + consul_mod diff --git a/docs/ref/states/saltext.consul.states.rst b/docs/ref/states/saltext.consul.states.consul.rst similarity index 50% rename from docs/ref/states/saltext.consul.states.rst rename to docs/ref/states/saltext.consul.states.consul.rst index bb56ba2..cfcea1a 100644 --- a/docs/ref/states/saltext.consul.states.rst +++ b/docs/ref/states/saltext.consul.states.consul.rst @@ -1,6 +1,5 @@ - -saltext.consul.states.consul -============================ +``consul`` +========== .. automodule:: saltext.consul.states.consul :members: diff --git a/docs/ref/states/saltext.consul.states.consul_mod.rst b/docs/ref/states/saltext.consul.states.consul_mod.rst new file mode 100644 index 0000000..1ca37da --- /dev/null +++ b/docs/ref/states/saltext.consul.states.consul_mod.rst @@ -0,0 +1,5 @@ +``consul`` +========== + +.. automodule:: saltext.consul.states.consul_mod + :members: diff --git a/docs/topics/installation.md b/docs/topics/installation.md new file mode 100644 index 0000000..ab2f145 --- /dev/null +++ b/docs/topics/installation.md @@ -0,0 +1,36 @@ +# Installation + +Generally, extensions need to be installed into the same Python environment Salt uses. + +:::{tab} State +```yaml +Install Salt Consul extension: + pip.installed: + - name: saltext-consul +``` +::: + +:::{tab} Onedir installation +```bash +salt-pip install saltext-consul +``` +::: + +:::{tab} Regular installation +```bash +pip install saltext-consul +``` +::: + +:::{important} +Currently, there is [an issue][issue-second-saltext] where the installation of a Saltext fails silently +if the environment already has another one installed. You can workaround this by +removing all Saltexts and reinstalling them in one transaction. +::: + +:::{hint} +Saltexts are not distributed automatically via the fileserver like custom modules, they need to be installed +on each node you want them to be available on. +::: + +[issue-second-saltext]: https://github.com/saltstack/salt/issues/65433 diff --git a/noxfile.py b/noxfile.py old mode 100644 new mode 100755 index 6c7bafe..c5ef3cc --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,7 @@ nox.options.error_on_missing_interpreters = False # Python versions to test against -PYTHON_VERSIONS = ("3", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10") +PYTHON_VERSIONS = ("3", "3.8", "3.9", "3.10") # Be verbose when running under a CI context CI_RUN = ( os.environ.get("JENKINS_URL") or os.environ.get("CI") or os.environ.get("DRONE") is not None @@ -29,7 +29,7 @@ EXTRA_REQUIREMENTS_INSTALL = os.environ.get("EXTRA_REQUIREMENTS_INSTALL") COVERAGE_VERSION_REQUIREMENT = "coverage==5.2" -SALT_REQUIREMENT = os.environ.get("SALT_REQUIREMENT") or "salt>=3003" +SALT_REQUIREMENT = os.environ.get("SALT_REQUIREMENT") or "salt>=3005" if SALT_REQUIREMENT == "salt==master": SALT_REQUIREMENT = "git+https://github.com/saltstack/salt.git@master" @@ -70,8 +70,8 @@ def _get_session_python_version_info(session): def _get_pydir(session): version_info = _get_session_python_version_info(session) - if version_info < (3, 5): - session.error("Only Python >= 3.5 is supported") + if version_info < (3, 8): + session.error("Only Python >= 3.8 is supported") return f"py{version_info[0]}.{version_info[1]}" diff --git a/pyproject.toml b/pyproject.toml index 6ea51a2..7be35b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,144 @@ [build-system] -requires = ["setuptools>=50.3.2", "wheel", "setuptools-declarative-requirements", "setuptools_scm[toml]>=3.4"] +requires = [ + "wheel", + "setuptools>=50.3.2", + "setuptools_scm[toml]>=3.4", +] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "src/saltext/consul/version.py" write_to_template = "__version__ = \"{version}\"" +[project] +name = "saltext.consul" +description = "Salt Extension for interacting with Consul" +authors = [ + {name = "EITR Technologies, LLC", email = "eitr@devops.tech"}, +] +keywords = [ + "salt-extension", +] +license = {text = "Apache Software License"} +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Cython", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", +] +requires-python = ">= 3.8" +dynamic = ["version"] +dependencies = [ + "python-consul>=0.4.7", +] + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.urls] +Homepage = "https://github.com/salt-extensions/saltext-consul" +Source = "https://github.com/salt-extensions/saltext-consul" +Tracker = "https://github.com/salt-extensions/saltext-consul/issues" + +[project.optional-dependencies] +changelog = ["towncrier==22.12.0"] +dev = [ + "nox", + "pre-commit>=2.4.0", + "pylint", + "saltpylint", +] +docs = [ + "sphinx", + "sphinx-prompt", + "sphinxcontrib-spelling", + "sphinx-copybutton", + "towncrier==22.12.0", + "sphinxcontrib-towncrier", + "myst_parser", + "furo", + "sphinx-inline-tabs", +] +docsauto = ["sphinx-autobuild"] +lint = [ + "pylint", + "saltpylint", +] +tests = [ + "pytest>=6.1.0", + "pytest-salt-factories>=1.0.0rc19", +] + +[project.entry-points."salt.loader"] +"saltext.consul" = "saltext.consul" + +[tool.setuptools] +zip-safe = false +include-package-data = true +platforms = ["any"] + +[tool.setuptools.packages.find] +where = ["src"] +exclude = ["tests"] + +[tool.distutils.bdist_wheel] +# Use this option if your package is pure-python +universal = 1 + +[tool.distutils.sdist] +owner = "root" +group = "root" + +[tool.build_sphinx] +source_dir = "docs" +build_dir = "build/sphinx" + [tool.black] line-length = 100 + +[tool.towncrier] + package = "saltext.consul" + filename = "CHANGELOG.md" + template = "changelog/.template.jinja" + directory = "changelog/" + start_string = "# Changelog\n" + underlines = ["", "", ""] + title_format = "## {version} ({project_date})" + issue_format = "[#{issue}](https://github.com/salt-extensions/saltext-consul/issues/{issue})" + + [[tool.towncrier.type]] + directory = "removed" + name = "Removed" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecated" + name = "Deprecated" + showcontent = true + + [[tool.towncrier.type]] + directory = "changed" + name = "Changed" + showcontent = true + + [[tool.towncrier.type]] + directory = "fixed" + name = "Fixed" + showcontent = true + + [[tool.towncrier.type]] + directory = "added" + name = "Added" + showcontent = true + + [[tool.towncrier.type]] + directory = "security" + name = "Security" + showcontent = true diff --git a/requirements/base.txt b/requirements/base.txt deleted file mode 100644 index bcd4f13..0000000 --- a/requirements/base.txt +++ /dev/null @@ -1 +0,0 @@ -salt>=3003 diff --git a/requirements/docs.in b/requirements/docs.in deleted file mode 100644 index 1542731..0000000 --- a/requirements/docs.in +++ /dev/null @@ -1,5 +0,0 @@ -sphinx -sphinx-material-saltstack -sphinx-prompt -sphinxcontrib-spelling -importlib_metadata; python_version < "3.8" diff --git a/requirements/docs.txt b/requirements/docs.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/lint.in b/requirements/lint.in deleted file mode 100644 index 911de93..0000000 --- a/requirements/lint.in +++ /dev/null @@ -1,2 +0,0 @@ -pylint -saltpylint diff --git a/requirements/lint.txt b/requirements/lint.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/py3.5/docs.txt b/requirements/py3.5/docs.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/py3.5/lint.txt b/requirements/py3.5/lint.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/py3.5/tests.txt b/requirements/py3.5/tests.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/py3.6/docs.txt b/requirements/py3.6/docs.txt deleted file mode 100644 index 273b81d..0000000 --- a/requirements/py3.6/docs.txt +++ /dev/null @@ -1,89 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.6/docs.txt requirements/docs.in -# -alabaster==0.7.13 - # via sphinx -babel==2.11.0 - # via sphinx -beautifulsoup4==4.9.1 - # via sphinx-material-saltstack -certifi==2023.11.17 - # via requests -charset-normalizer==2.0.12 - # via requests -css-html-js-minify==2.5.5 - # via sphinx-material-saltstack -docutils==0.18.1 - # via sphinx -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==4.8.3 ; python_version < "3.8" - # via - # -r requirements/docs.in - # sphinx - # sphinxcontrib-spelling -jinja2==3.0.3 - # via sphinx -lxml==4.5.2 - # via sphinx-material-saltstack -markupsafe==2.0.1 - # via jinja2 -packaging==21.3 - # via sphinx -pyenchant==3.2.2 - # via sphinxcontrib-spelling -pygments==2.14.0 - # via - # sphinx - # sphinx-prompt -pyparsing==3.0.7 - # via packaging -python-slugify[unidecode]==4.0.1 - # via sphinx-material-saltstack -pytz==2023.3.post1 - # via babel -requests==2.27.1 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -soupsieve==2.3.2.post1 - # via beautifulsoup4 -sphinx-material-saltstack==1.0.5 - # via -r requirements/docs.in -sphinx-prompt==1.5.0 - # via -r requirements/docs.in -sphinx==5.3.0 - # via - # -r requirements/docs.in - # sphinx-material-saltstack - # sphinx-prompt - # sphinxcontrib-spelling -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -sphinxcontrib-spelling==7.7.0 - # via -r requirements/docs.in -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via importlib-metadata -unidecode==1.3.7 - # via python-slugify -urllib3==1.26.18 - # via requests -zipp==3.6.0 - # via importlib-metadata diff --git a/requirements/py3.6/lint.txt b/requirements/py3.6/lint.txt deleted file mode 100644 index 4ae2428..0000000 --- a/requirements/py3.6/lint.txt +++ /dev/null @@ -1,41 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.6/lint.txt requirements/lint.in -# -astroid==2.11.7 - # via pylint -dill==0.3.4 - # via pylint -isort==5.10.1 - # via pylint -lazy-object-proxy==1.7.1 - # via astroid -mccabe==0.7.0 - # via pylint -modernize==0.5 - # via saltpylint -platformdirs==2.4.0 - # via pylint -pycodestyle==2.10.0 - # via saltpylint -pylint==2.13.9 - # via - # -r requirements/lint.in - # saltpylint -saltpylint==2023.8.3 - # via -r requirements/lint.in -tomli==1.2.3 - # via pylint -typed-ast==1.5.5 - # via astroid -typing-extensions==4.1.1 - # via - # astroid - # pylint -wrapt==1.16.0 - # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/py3.6/tests.txt b/requirements/py3.6/tests.txt deleted file mode 100644 index 705ae90..0000000 --- a/requirements/py3.6/tests.txt +++ /dev/null @@ -1,67 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.6/tests.txt requirements/tests.in -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-skip-markers -distlib==0.3.8 - # via virtualenv -distro==1.8.0 - # via pytest-skip-markers -filelock==3.4.1 - # via virtualenv -importlib-metadata==4.8.3 - # via - # pluggy - # pytest - # virtualenv -importlib-resources==5.4.0 - # via virtualenv -iniconfig==1.1.1 - # via pytest -msgpack==1.0.5 - # via pytest-salt-factories -packaging==21.3 - # via pytest -platformdirs==2.4.0 - # via virtualenv -pluggy==1.0.0 - # via pytest -psutil==5.9.7 - # via pytest-salt-factories -py==1.11.0 - # via pytest -pyparsing==3.0.7 - # via packaging -pytest-helpers-namespace==2021.12.29 - # via pytest-salt-factories -pytest-salt-factories==0.912.2 - # via -r requirements/tests.in -pytest-skip-markers==1.3.0 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.0.1 - # via - # -r requirements/tests.in - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-skip-markers - # pytest-tempdir -pyzmq==25.1.2 - # via pytest-salt-factories -tomli==1.2.3 - # via pytest -typing-extensions==4.1.1 - # via importlib-metadata -virtualenv==20.17.1 - # via pytest-salt-factories -zipp==3.6.0 - # via - # importlib-metadata - # importlib-resources diff --git a/requirements/py3.7/docs.txt b/requirements/py3.7/docs.txt deleted file mode 100644 index eb87713..0000000 --- a/requirements/py3.7/docs.txt +++ /dev/null @@ -1,87 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.7/docs.txt requirements/docs.in -# -alabaster==0.7.13 - # via sphinx -babel==2.14.0 - # via sphinx -beautifulsoup4==4.9.1 - # via sphinx-material-saltstack -certifi==2023.11.17 - # via requests -charset-normalizer==3.3.2 - # via requests -css-html-js-minify==2.5.5 - # via sphinx-material-saltstack -docutils==0.19 - # via sphinx -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==6.7.0 ; python_version < "3.8" - # via - # -r requirements/docs.in - # sphinx - # sphinxcontrib-spelling -jinja2==3.1.2 - # via sphinx -lxml==4.5.2 - # via sphinx-material-saltstack -markupsafe==2.1.3 - # via jinja2 -packaging==23.2 - # via sphinx -pyenchant==3.2.2 - # via sphinxcontrib-spelling -pygments==2.17.2 - # via - # sphinx - # sphinx-prompt -python-slugify[unidecode]==4.0.1 - # via sphinx-material-saltstack -pytz==2023.3.post1 - # via babel -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -soupsieve==2.4.1 - # via beautifulsoup4 -sphinx-material-saltstack==1.0.5 - # via -r requirements/docs.in -sphinx-prompt==1.5.0 - # via -r requirements/docs.in -sphinx==5.3.0 - # via - # -r requirements/docs.in - # sphinx-material-saltstack - # sphinx-prompt - # sphinxcontrib-spelling -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -sphinxcontrib-spelling==8.0.0 - # via -r requirements/docs.in -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via importlib-metadata -unidecode==1.3.7 - # via python-slugify -urllib3==2.0.7 - # via requests -zipp==3.15.0 - # via importlib-metadata diff --git a/requirements/py3.7/lint.txt b/requirements/py3.7/lint.txt deleted file mode 100644 index 4eda0b4..0000000 --- a/requirements/py3.7/lint.txt +++ /dev/null @@ -1,41 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.7/lint.txt requirements/lint.in -# -astroid==2.15.8 - # via pylint -dill==0.3.7 - # via pylint -isort==5.11.5 - # via pylint -lazy-object-proxy==1.9.0 - # via astroid -mccabe==0.7.0 - # via pylint -modernize==0.5 - # via saltpylint -platformdirs==4.0.0 - # via pylint -pycodestyle==2.10.0 - # via saltpylint -pylint==2.17.7 - # via - # -r requirements/lint.in - # saltpylint -saltpylint==2023.8.3 - # via -r requirements/lint.in -tomli==2.0.1 - # via pylint -tomlkit==0.12.3 - # via pylint -typed-ast==1.5.5 - # via astroid -typing-extensions==4.7.1 - # via - # astroid - # platformdirs - # pylint -wrapt==1.16.0 - # via astroid diff --git a/requirements/py3.7/tests.txt b/requirements/py3.7/tests.txt deleted file mode 100644 index 1945552..0000000 --- a/requirements/py3.7/tests.txt +++ /dev/null @@ -1,63 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.7/tests.txt requirements/tests.in -# -attrs==23.1.0 - # via - # pytest-salt-factories - # pytest-skip-markers -distlib==0.3.8 - # via virtualenv -distro==1.8.0 - # via pytest-skip-markers -exceptiongroup==1.2.0 - # via pytest -filelock==3.12.2 - # via virtualenv -importlib-metadata==6.7.0 - # via - # attrs - # pluggy - # pytest - # virtualenv -iniconfig==2.0.0 - # via pytest -msgpack==1.0.5 - # via pytest-salt-factories -packaging==23.2 - # via pytest -platformdirs==4.0.0 - # via virtualenv -pluggy==1.2.0 - # via pytest -psutil==5.9.7 - # via pytest-salt-factories -pytest-helpers-namespace==2021.12.29 - # via pytest-salt-factories -pytest-salt-factories==0.912.2 - # via -r requirements/tests.in -pytest-skip-markers==1.5.0 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.4.3 - # via - # -r requirements/tests.in - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-skip-markers - # pytest-tempdir -pyzmq==25.1.2 - # via pytest-salt-factories -tomli==2.0.1 - # via pytest -typing-extensions==4.7.1 - # via - # importlib-metadata - # platformdirs -virtualenv==20.25.0 - # via pytest-salt-factories -zipp==3.15.0 - # via importlib-metadata diff --git a/requirements/py3.8/docs.txt b/requirements/py3.8/docs.txt deleted file mode 100644 index 0cc6eed..0000000 --- a/requirements/py3.8/docs.txt +++ /dev/null @@ -1,84 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.8/docs.txt requirements/docs.in -# -alabaster==0.7.13 - # via sphinx -babel==2.14.0 - # via sphinx -beautifulsoup4==4.9.1 - # via sphinx-material-saltstack -certifi==2023.11.17 - # via requests -charset-normalizer==3.3.2 - # via requests -css-html-js-minify==2.5.5 - # via sphinx-material-saltstack -docutils==0.20.1 - # via - # sphinx - # sphinx-prompt -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==7.0.0 - # via sphinx -jinja2==3.1.2 - # via sphinx -lxml==4.5.2 - # via sphinx-material-saltstack -markupsafe==2.1.3 - # via jinja2 -packaging==23.2 - # via sphinx -pyenchant==3.2.2 - # via sphinxcontrib-spelling -pygments==2.17.2 - # via - # sphinx - # sphinx-prompt -python-slugify[unidecode]==4.0.1 - # via sphinx-material-saltstack -pytz==2023.3.post1 - # via babel -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -soupsieve==2.5 - # via beautifulsoup4 -sphinx-material-saltstack==1.0.5 - # via -r requirements/docs.in -sphinx-prompt==1.7.0 - # via -r requirements/docs.in -sphinx==7.1.2 - # via - # -r requirements/docs.in - # sphinx-material-saltstack - # sphinx-prompt - # sphinxcontrib-spelling -sphinxcontrib-applehelp==1.0.4 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.1 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -sphinxcontrib-spelling==8.0.0 - # via -r requirements/docs.in -text-unidecode==1.3 - # via python-slugify -unidecode==1.3.7 - # via python-slugify -urllib3==2.1.0 - # via requests -zipp==3.17.0 - # via importlib-metadata diff --git a/requirements/py3.8/lint.txt b/requirements/py3.8/lint.txt deleted file mode 100644 index d4f1496..0000000 --- a/requirements/py3.8/lint.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.8/lint.txt requirements/lint.in -# -astroid==3.0.2 - # via pylint -dill==0.3.7 - # via pylint -isort==5.13.2 - # via pylint -mccabe==0.7.0 - # via pylint -modernize==0.5 - # via saltpylint -platformdirs==4.1.0 - # via pylint -pycodestyle==2.11.1 - # via saltpylint -pylint==3.0.3 - # via - # -r requirements/lint.in - # saltpylint -saltpylint==2023.8.3 - # via -r requirements/lint.in -tomli==2.0.1 - # via pylint -tomlkit==0.12.3 - # via pylint -typing-extensions==4.9.0 - # via - # astroid - # pylint diff --git a/requirements/py3.8/tests.txt b/requirements/py3.8/tests.txt deleted file mode 100644 index 2282e3d..0000000 --- a/requirements/py3.8/tests.txt +++ /dev/null @@ -1,51 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.8/tests.txt requirements/tests.in -# -attrs==23.1.0 - # via - # pytest-salt-factories - # pytest-skip-markers -distlib==0.3.8 - # via virtualenv -distro==1.8.0 - # via pytest-skip-markers -exceptiongroup==1.2.0 - # via pytest -filelock==3.13.1 - # via virtualenv -iniconfig==2.0.0 - # via pytest -msgpack==1.0.7 - # via pytest-salt-factories -packaging==23.2 - # via pytest -platformdirs==4.1.0 - # via virtualenv -pluggy==1.3.0 - # via pytest -psutil==5.9.7 - # via pytest-salt-factories -pytest-helpers-namespace==2021.12.29 - # via pytest-salt-factories -pytest-salt-factories==0.912.2 - # via -r requirements/tests.in -pytest-skip-markers==1.5.0 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.4.3 - # via - # -r requirements/tests.in - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-skip-markers - # pytest-tempdir -pyzmq==25.1.2 - # via pytest-salt-factories -tomli==2.0.1 - # via pytest -virtualenv==20.25.0 - # via pytest-salt-factories diff --git a/requirements/py3.9/docs.txt b/requirements/py3.9/docs.txt deleted file mode 100644 index 8aab136..0000000 --- a/requirements/py3.9/docs.txt +++ /dev/null @@ -1,87 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.9/docs.txt requirements/docs.in -# -alabaster==0.7.13 - # via sphinx -babel==2.14.0 - # via sphinx -beautifulsoup4==4.9.1 - # via sphinx-material-saltstack -certifi==2023.11.17 - # via requests -charset-normalizer==3.3.2 - # via requests -css-html-js-minify==2.5.5 - # via sphinx-material-saltstack -docutils==0.20.1 - # via - # sphinx - # sphinx-prompt -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==7.0.0 - # via sphinx -jinja2==3.1.2 - # via sphinx -lxml==4.5.2 - # via sphinx-material-saltstack -markupsafe==2.1.3 - # via jinja2 -packaging==23.2 - # via sphinx -pyenchant==3.2.2 - # via sphinxcontrib-spelling -pygments==2.17.2 - # via - # sphinx - # sphinx-prompt -python-slugify[unidecode]==4.0.1 - # via sphinx-material-saltstack -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -soupsieve==2.5 - # via beautifulsoup4 -sphinx-material-saltstack==1.0.5 - # via -r requirements/docs.in -sphinx-prompt==1.8.0 - # via -r requirements/docs.in -sphinx==7.2.6 - # via - # -r requirements/docs.in - # sphinx-material-saltstack - # sphinx-prompt - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml - # sphinxcontrib-spelling -sphinxcontrib-applehelp==1.0.7 - # via sphinx -sphinxcontrib-devhelp==1.0.5 - # via sphinx -sphinxcontrib-htmlhelp==2.0.4 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.6 - # via sphinx -sphinxcontrib-serializinghtml==1.1.9 - # via sphinx -sphinxcontrib-spelling==8.0.0 - # via -r requirements/docs.in -text-unidecode==1.3 - # via python-slugify -unidecode==1.3.7 - # via python-slugify -urllib3==2.1.0 - # via requests -zipp==3.17.0 - # via importlib-metadata diff --git a/requirements/py3.9/lint.txt b/requirements/py3.9/lint.txt deleted file mode 100644 index 5cfdc7c..0000000 --- a/requirements/py3.9/lint.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.9/lint.txt requirements/lint.in -# -astroid==3.0.2 - # via pylint -dill==0.3.7 - # via pylint -isort==5.13.2 - # via pylint -mccabe==0.7.0 - # via pylint -modernize==0.5 - # via saltpylint -platformdirs==4.1.0 - # via pylint -pycodestyle==2.11.1 - # via saltpylint -pylint==3.0.3 - # via - # -r requirements/lint.in - # saltpylint -saltpylint==2023.8.3 - # via -r requirements/lint.in -tomli==2.0.1 - # via pylint -tomlkit==0.12.3 - # via pylint -typing-extensions==4.9.0 - # via - # astroid - # pylint diff --git a/requirements/py3.9/tests.txt b/requirements/py3.9/tests.txt deleted file mode 100644 index c9bc9b5..0000000 --- a/requirements/py3.9/tests.txt +++ /dev/null @@ -1,51 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/py3.9/tests.txt requirements/tests.in -# -attrs==23.1.0 - # via - # pytest-salt-factories - # pytest-skip-markers -distlib==0.3.8 - # via virtualenv -distro==1.8.0 - # via pytest-skip-markers -exceptiongroup==1.2.0 - # via pytest -filelock==3.13.1 - # via virtualenv -iniconfig==2.0.0 - # via pytest -msgpack==1.0.7 - # via pytest-salt-factories -packaging==23.2 - # via pytest -platformdirs==4.1.0 - # via virtualenv -pluggy==1.3.0 - # via pytest -psutil==5.9.7 - # via pytest-salt-factories -pytest-helpers-namespace==2021.12.29 - # via pytest-salt-factories -pytest-salt-factories==0.912.2 - # via -r requirements/tests.in -pytest-skip-markers==1.5.0 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.4.3 - # via - # -r requirements/tests.in - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-skip-markers - # pytest-tempdir -pyzmq==25.1.2 - # via pytest-salt-factories -tomli==2.0.1 - # via pytest -virtualenv==20.25.0 - # via pytest-salt-factories diff --git a/requirements/tests.in b/requirements/tests.in deleted file mode 100644 index 9f29215..0000000 --- a/requirements/tests.in +++ /dev/null @@ -1,2 +0,0 @@ -pytest >= 6.1.0 -pytest-salt-factories >= 0.130.0 diff --git a/requirements/tests.txt b/requirements/tests.txt deleted file mode 100644 index e69de29..0000000 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6b9a92f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,79 +0,0 @@ -[metadata] -name = saltext.consul -description = Salt Extension for interacting with Consul -long_description = file: README.md -long_description_content_type = text/markdown -author = EITR Technologies -author_email = devops@eitr.tech -keywords = salt-extension -url = https://github.com/salt-extensions/saltext-consul -project_urls = - Source=https://github.com/salt-extensions/saltext-consul - Tracker=https://github.com/salt-extensions/saltext-consul/issues -license = Apache Software License -classifiers = - Programming Language :: Python - Programming Language :: Cython - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Development Status :: 4 - Beta - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License -platforms = any - -[options] -zip_safe = False -include_package_data = True -package_dir = - =src -packages = find_namespace: -python_requires = >= 3.5 -setup_requires = - wheel - setuptools>=50.3.2 - setuptools_scm[toml]>=3.4 - setuptools-declarative-requirements -install_requires = - salt>=3003 - # Add other module install requirements above this line - -[options.packages.find] -where = src -exclude = - tests - -# When targetting Salt < 3003, you can remove the other 'options.entry_points' section and use this one -#[options.entry_points] -#salt.loader= -# - -[options.entry_points] -salt.loader= - saltext.consul = saltext.consul - -[requirements-files] -install_requires = requirements/base.txt -tests_require = requirements/tests.txt -extras_require = - dev = requirements/dev.txt - tests = requirements/tests.txt - docs = requirements/docs.txt - docsauto = requirements/docs-auto.txt - changelog = requirements/changelog.txt - -[bdist_wheel] -# Use this option if your package is pure-python -universal = 1 - -[build_sphinx] -source_dir = docs -build_dir = build/sphinx - -[sdist] -owner = root -group = root diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 2b13a36..e69de29 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -1,354 +0,0 @@ -""" -Loader mechanism for caching data, with data expiration, etc. - -.. versionadded:: 2016.11.0 -""" - - -import logging -import time - -import salt.config -import salt.loader -import salt.syspaths -from salt.utils.odict import OrderedDict - -log = logging.getLogger(__name__) - - -def factory(opts, **kwargs): - """ - Creates and returns the cache class. - If memory caching is enabled by opts MemCache class will be instantiated. - If not Cache class will be returned. - """ - if opts.get("memcache_expire_seconds", 0): - cls = MemCache - else: - cls = Cache - return cls(opts, **kwargs) - - -class Cache: - """ - Base caching object providing access to the modular cache subsystem. - - Related configuration options: - - :param cache: - The name of the cache driver to use. This is the name of the python - module of the `salt.cache` package. Default is `localfs`. - - Terminology: - - Salt cache subsystem is organized as a tree with nodes and leafs like a - filesystem. Cache consists of banks. Each bank can contain a number of - keys. Each key can contain a dict or any other object serializable with - `salt.payload`. I.e. any data object in the cache can be - addressed by the path to the bank and the key name: - bank: 'minions/alpha' - key: 'data' - - Bank names should be formatted in a way that can be used as a - directory structure. If slashes are included in the name, then they - refer to a nested structure. - - Key name is a string identifier of a data container (like a file inside a - directory) which will hold the data. - """ - - def __init__(self, opts, cachedir=None, **kwargs): - self.opts = opts - if cachedir is None: - self.cachedir = opts.get("cachedir", salt.syspaths.CACHE_DIR) - else: - self.cachedir = cachedir - self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"]) - self._modules = None - self._kwargs = kwargs - self._kwargs["cachedir"] = self.cachedir - - def __lazy_init(self): - self._modules = salt.loader.cache(self.opts) - fun = "{}.init_kwargs".format(self.driver) - if fun in self.modules: - self._kwargs = self.modules[fun](self._kwargs) - else: - self._kwargs = {} - - @property - def modules(self): - if self._modules is None: - self.__lazy_init() - return self._modules - - def cache(self, bank, key, fun, loop_fun=None, **kwargs): - """ - Check cache for the data. If it is there, check to see if it needs to - be refreshed. - - If the data is not there, or it needs to be refreshed, then call the - callback function (``fun``) with any given ``**kwargs``. - - In some cases, the callback function returns a list of objects which - need to be processed by a second function. If that is the case, then - the second function is passed in as ``loop_fun``. Each item in the - return list from the first function will be the only argument for the - second function. - """ - expire_seconds = kwargs.get("expire", 86400) # 1 day - - updated = self.updated(bank, key) - update_cache = False - if updated is None: - update_cache = True - else: - if int(time.time()) - updated > expire_seconds: - update_cache = True - - data = self.fetch(bank, key) - - if not data or update_cache is True: - if loop_fun is not None: - data = [] - items = fun(**kwargs) - for item in items: - data.append(loop_fun(item)) - else: - data = fun(**kwargs) - self.store(bank, key, data) - - return data - - def store(self, bank, key, data): - """ - Store data using the specified module - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :param key: - The name of the key (or file inside a directory) which will hold - the data. File extensions should not be provided, as they will be - added by the driver itself. - - :param data: - The data which will be stored in the cache. This data should be - in a format which can be serialized by msgpack. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.store".format(self.driver) - return self.modules[fun](bank, key, data, **self._kwargs) - - def fetch(self, bank, key): - """ - Fetch data using the specified module - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :param key: - The name of the key (or file inside a directory) which will hold - the data. File extensions should not be provided, as they will be - added by the driver itself. - - :return: - Return a python object fetched from the cache or an empty dict if - the given path or key not found. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.fetch".format(self.driver) - return self.modules[fun](bank, key, **self._kwargs) - - def updated(self, bank, key): - """ - Get the last updated epoch for the specified key - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :param key: - The name of the key (or file inside a directory) which will hold - the data. File extensions should not be provided, as they will be - added by the driver itself. - - :return: - Return an int epoch time in seconds or None if the object wasn't - found in cache. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.updated".format(self.driver) - return self.modules[fun](bank, key, **self._kwargs) - - def flush(self, bank, key=None): - """ - Remove the key from the cache bank with all the key content. If no key is specified remove - the entire bank with all keys and sub-banks inside. - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :param key: - The name of the key (or file inside a directory) which will hold - the data. File extensions should not be provided, as they will be - added by the driver itself. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.flush".format(self.driver) - return self.modules[fun](bank, key=key, **self._kwargs) - - def list(self, bank): - """ - Lists entries stored in the specified bank. - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :return: - An iterable object containing all bank entries. Returns an empty - iterator if the bank doesn't exists. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.list".format(self.driver) - return self.modules[fun](bank, **self._kwargs) - - def contains(self, bank, key=None): - """ - Checks if the specified bank contains the specified key. - - :param bank: - The name of the location inside the cache which will hold the key - and its associated data. - - :param key: - The name of the key (or file inside a directory) which will hold - the data. File extensions should not be provided, as they will be - added by the driver itself. - - :return: - Returns True if the specified key exists in the given bank and False - if not. - If key is None checks for the bank existense. - - :raises SaltCacheError: - Raises an exception if cache driver detected an error accessing data - in the cache backend (auth, permissions, etc). - """ - fun = "{}.contains".format(self.driver) - return self.modules[fun](bank, key, **self._kwargs) - - -class MemCache(Cache): - """ - Short-lived in-memory cache store keeping values on time and/or size (count) - basis. - """ - - # {: odict({: [atime, data], ...}), ...} - data = {} - - def __init__(self, opts, **kwargs): - super().__init__(opts, **kwargs) - self.expire = opts.get("memcache_expire_seconds", 10) - self.max = opts.get("memcache_max_items", 1024) - self.cleanup = opts.get("memcache_full_cleanup", False) - self.debug = opts.get("memcache_debug", False) - if self.debug: - self.call = 0 - self.hit = 0 - self._storage = None - - @classmethod - def __cleanup(cls, expire): - now = time.time() - for storage in cls.data.values(): - for key, data in list(storage.items()): - if data[0] + expire < now: - del storage[key] - else: - break - - def _get_storage_id(self): - fun = "{}.storage_id".format(self.driver) - if fun in self.modules: - return self.modules[fun](self.kwargs) - else: - return self.driver - - @property - def storage(self): - if self._storage is None: - storage_id = self._get_storage_id() - if storage_id not in MemCache.data: - MemCache.data[storage_id] = OrderedDict() - self._storage = MemCache.data[storage_id] - return self._storage - - def fetch(self, bank, key): - if self.debug: - self.call += 1 - now = time.time() - record = self.storage.pop((bank, key), None) - # Have a cached value for the key - if record is not None and record[0] + self.expire >= now: - if self.debug: - self.hit += 1 - log.debug( - "MemCache stats (call/hit/rate): %s/%s/%s", - self.call, - self.hit, - float(self.hit) / self.call, - ) - # update atime and return - record[0] = now - self.storage[(bank, key)] = record - return record[1] - - # Have no value for the key or value is expired - data = super().fetch(bank, key) - if len(self.storage) >= self.max: - if self.cleanup: - MemCache.__cleanup(self.expire) - if len(self.storage) >= self.max: - self.storage.popitem(last=False) - self.storage[(bank, key)] = [now, data] - return data - - def store(self, bank, key, data): - self.storage.pop((bank, key), None) - super().store(bank, key, data) - if len(self.storage) >= self.max: - if self.cleanup: - MemCache.__cleanup(self.expire) - if len(self.storage) >= self.max: - self.storage.popitem(last=False) - self.storage[(bank, key)] = [time.time(), data] - - def flush(self, bank, key=None): - if key is None: - for bank_, key_ in tuple(self.storage): - if bank == bank_: - self.storage.pop((bank_, key_)) - else: - self.storage.pop((bank, key), None) - super().flush(bank, key) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 61af6cc..6eb259f 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -55,7 +55,6 @@ .. _`python-consul documentation`: https://python-consul.readthedocs.io/en/latest/#consul """ - import logging import time @@ -127,7 +126,7 @@ def store(bank, key, data): api.kv.put(c_key, c_data) api.kv.put(tstamp_key, salt.payload.dumps(int(time.time()))) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f"There was an error writing the key, {c_key}: {exc}") + raise SaltCacheError(f"There was an error writing the key, {c_key}: {exc}") from exc def fetch(bank, key): @@ -141,7 +140,7 @@ def fetch(bank, key): return {} return salt.payload.loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") + raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") from exc def flush(bank, key=None): @@ -159,7 +158,7 @@ def flush(bank, key=None): api.kv.delete(tstamp_key) return api.kv.delete(c_key, recurse=key is None) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f"There was an error removing the key, {c_key}: {exc}") + raise SaltCacheError(f"There was an error removing the key, {c_key}: {exc}") from exc def list_(bank): @@ -169,7 +168,7 @@ def list_(bank): try: _, keys = api.kv.get(bank + "/", keys=True, separator="/") except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f'There was an error getting the key "{bank}": {exc}') + raise SaltCacheError(f'There was an error getting the key "{bank}": {exc}') from exc if keys is None: keys = [] else: @@ -187,10 +186,10 @@ def contains(bank, key): Checks if the specified bank contains the specified key. """ try: - c_key = "{}/{}".format(bank, key or "") + c_key = "{}/{}".format(bank, key or "") # pylint: disable=consider-using-f-string _, value = api.kv.get(c_key, keys=True) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f"There was an error getting the key, {c_key}: {exc}") + raise SaltCacheError(f"There was an error getting the key, {c_key}: {exc}") from exc return value is not None @@ -206,4 +205,4 @@ def updated(bank, key): return None return salt.payload.loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") + raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") from exc diff --git a/src/saltext/consul/loader.py b/src/saltext/consul/loader.py deleted file mode 100644 index 5d2e0b6..0000000 --- a/src/saltext/consul/loader.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Define the required entry-points functions in order for Salt to know -what and from where it should load this extension's loaders -""" -from . import PACKAGE_ROOT # pylint: disable=unused-import diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 205cfb2..6ceac04 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -4,7 +4,6 @@ https://www.consul.io """ - import base64 import http.client import logging @@ -34,9 +33,7 @@ def _get_token(): """ Retrieve Consul configuration """ - return __salt__["config.get"]("consul.token") or __salt__["config.get"]( - "consul:token" - ) + return __salt__["config.get"]("consul.token") or __salt__["config.get"]("consul:token") def _query( @@ -68,7 +65,7 @@ def _query( token = _get_token() headers = {"X-Consul-Token": token, "Content-Type": "application/json"} - base_url = urllib.parse.urljoin(consul_url, "{}/".format(api_version)) + base_url = urllib.parse.urljoin(consul_url, f"{api_version}/") url = urllib.parse.urljoin(base_url, function, False) if method == "GET": @@ -149,13 +146,11 @@ def list_(consul_url=None, token=None, key=None, **kwargs): query_params["recurse"] = "True" function = "kv/" else: - function = "kv/{}".format(key) + function = f"kv/{key}" query_params["keys"] = "True" query_params["separator"] = "/" - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -203,14 +198,12 @@ def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw= raise SaltInvocationError('Required argument "key" is missing.') query_params = {} - function = "kv/{}".format(key) + function = f"kv/{key}" if recurse: query_params["recurse"] = "True" if raw: query_params["raw"] = True - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) if ret["res"]: if decode: @@ -269,9 +262,7 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): for _l2 in conflicting_args: if _l1 in kwargs and _l2 in kwargs and _l1 != _l2: raise SaltInvocationError( - "Using arguments `{}` and `{}` together is invalid.".format( - _l1, _l2 - ) + f"Using arguments `{_l1}` and `{_l2}` together is invalid." ) query_params = {} @@ -286,25 +277,27 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if "cas" in kwargs: if _current["res"]: if kwargs["cas"] == 0: - ret["message"] = "Key {} exists, index must be non-zero.".format(key) + ret["message"] = f"Key {key} exists, index must be non-zero." ret["res"] = False return ret if kwargs["cas"] != _current["data"]["ModifyIndex"]: - ret["message"] = "Key {} exists, but indexes do not match.".format(key) + ret["message"] = f"Key {key} exists, but indexes do not match." ret["res"] = False return ret query_params["cas"] = kwargs["cas"] else: - ret[ - "message" - ] = "Key {} does not exists, CAS argument can not be used.".format(key) + ret["message"] = f"Key {key} does not exists, CAS argument can not be used." ret["res"] = False return ret if "acquire" in kwargs: if kwargs["acquire"] not in available_sessions: - ret["message"] = "{} is not a valid session.".format(kwargs["acquire"]) + ret[ + "message" + ] = "{} is not a valid session.".format( # pylint: disable=consider-using-f-string + kwargs["acquire"] + ) ret["res"] = False return ret @@ -316,18 +309,22 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if _current["data"]["Session"] == kwargs["release"]: query_params["release"] = kwargs["release"] else: - ret["message"] = "{} locked by another session.".format(key) + ret["message"] = f"{key} locked by another session." ret["res"] = False return ret else: - ret["message"] = "{} is not a valid session.".format(kwargs["acquire"]) + ret[ + "message" + ] = "{} is not a valid session.".format( # pylint: disable=consider-using-f-string + kwargs["acquire"] + ) ret["res"] = False else: log.error("Key {0} does not exist. Skipping release.") data = value - function = "kv/{}".format(key) + function = f"kv/{key}" method = "PUT" res = _query( consul_url=consul_url, @@ -340,10 +337,10 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if res["res"]: ret["res"] = True - ret["data"] = "Added key {} with value {}.".format(key, value) + ret["data"] = f"Added key {key} with value {value}." else: ret["res"] = False - ret["data"] = "Unable to add key {} with value {}.".format(key, value) + ret["data"] = f"Unable to add key {key} with value {value}." if "error" in res: ret["error"] = res["error"] return ret @@ -396,7 +393,7 @@ def delete(consul_url=None, token=None, key=None, **kwargs): ret["res"] = False return ret - function = "kv/{}".format(key) + function = f"kv/{key}" res = _query( consul_url=consul_url, token=token, @@ -407,10 +404,10 @@ def delete(consul_url=None, token=None, key=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Deleted key {}.".format(key) + ret["message"] = f"Deleted key {key}." else: ret["res"] = False - ret["message"] = "Unable to delete key {}.".format(key) + ret["message"] = f"Unable to delete key {key}." if "error" in res: ret["error"] = res["error"] return ret @@ -596,7 +593,11 @@ def agent_maintenance(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Agent maintenance mode {}ed.".format(kwargs["enable"]) + ret[ + "message" + ] = "Agent maintenance mode {}ed.".format( # pylint: disable=consider-using-f-string + kwargs["enable"] + ) else: ret["res"] = True ret["message"] = "Unable to change maintenance mode for agent." @@ -635,7 +636,7 @@ def agent_join(consul_url=None, token=None, address=None, **kwargs): if "wan" in kwargs: query_params["wan"] = kwargs["wan"] - function = "agent/join/{}".format(address) + function = f"agent/join/{address}" res = _query( consul_url=consul_url, function=function, @@ -680,7 +681,7 @@ def agent_leave(consul_url=None, token=None, node=None): if not node: raise SaltInvocationError('Required argument "node" is missing.') - function = "agent/force-leave/{}".format(node) + function = f"agent/force-leave/{node}" res = _query( consul_url=consul_url, function=function, @@ -690,10 +691,10 @@ def agent_leave(consul_url=None, token=None, node=None): ) if res["res"]: ret["res"] = True - ret["message"] = "Node {} put in leave state.".format(node) + ret["message"] = f"Node {node} put in leave state." else: ret["res"] = False - ret["message"] = "Unable to change state for {}.".format(node) + ret["message"] = f"Unable to change state for {node}." return ret @@ -770,13 +771,15 @@ def agent_check_register(consul_url=None, token=None, **kwargs): data["TTL"] = kwargs["ttl"] function = "agent/check/register" - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Check {} added to agent.".format(kwargs["name"]) + ret[ + "message" + ] = "Check {} added to agent.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) else: ret["res"] = False ret["message"] = "Unable to add check to agent." @@ -810,11 +813,11 @@ def agent_check_deregister(consul_url=None, token=None, checkid=None): if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - function = "agent/check/deregister/{}".format(checkid) + function = f"agent/check/deregister/{checkid}" res = _query(consul_url=consul_url, function=function, token=token, method="GET") if res["res"]: ret["res"] = True - ret["message"] = "Check {} removed from agent.".format(checkid) + ret["message"] = f"Check {checkid} removed from agent." else: ret["res"] = False ret["message"] = "Unable to remove check from agent." @@ -855,7 +858,7 @@ def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/pass/{}".format(checkid) + function = f"agent/check/pass/{checkid}" res = _query( consul_url=consul_url, function=function, @@ -865,10 +868,10 @@ def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {} marked as passing.".format(checkid) + ret["message"] = f"Check {checkid} marked as passing." else: ret["res"] = False - ret["message"] = "Unable to update check {}.".format(checkid) + ret["message"] = f"Unable to update check {checkid}." return ret @@ -906,7 +909,7 @@ def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/warn/{}".format(checkid) + function = f"agent/check/warn/{checkid}" res = _query( consul_url=consul_url, function=function, @@ -916,10 +919,10 @@ def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {} marked as warning.".format(checkid) + ret["message"] = f"Check {checkid} marked as warning." else: ret["res"] = False - ret["message"] = "Unable to update check {}.".format(checkid) + ret["message"] = f"Unable to update check {checkid}." return ret @@ -957,7 +960,7 @@ def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/fail/{}".format(checkid) + function = f"agent/check/fail/{checkid}" res = _query( consul_url=consul_url, function=function, @@ -967,10 +970,10 @@ def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {} marked as critical.".format(checkid) + ret["message"] = f"Check {checkid} marked as critical." else: ret["res"] = False - ret["message"] = "Unable to update check {}.".format(checkid) + ret["message"] = f"Unable to update check {checkid}." return ret @@ -1074,15 +1077,21 @@ def agent_service_register(consul_url=None, token=None, **kwargs): data["Check"] = check_dd # if empty, ignore it function = "agent/service/register" - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Service {} registered on agent.".format(kwargs["name"]) + ret[ + "message" + ] = "Service {} registered on agent.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) else: ret["res"] = False - ret["message"] = "Unable to register service {}.".format(kwargs["name"]) + ret[ + "message" + ] = "Unable to register service {}.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -1114,16 +1123,14 @@ def agent_service_deregister(consul_url=None, token=None, serviceid=None): if not serviceid: raise SaltInvocationError('Required argument "serviceid" is missing.') - function = "agent/service/deregister/{}".format(serviceid) - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + function = f"agent/service/deregister/{serviceid}" + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Service {} removed from agent.".format(serviceid) + ret["message"] = f"Service {serviceid} removed from agent." else: ret["res"] = False - ret["message"] = "Unable to remove service {}.".format(serviceid) + ret["message"] = f"Unable to remove service {serviceid}." return ret @@ -1168,19 +1175,15 @@ def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwa if "reason" in kwargs: query_params["reason"] = kwargs["reason"] - function = "agent/service/maintenance/{}".format(serviceid) - res = _query( - consul_url=consul_url, token=token, function=function, query_params=query_params - ) + function = f"agent/service/maintenance/{serviceid}" + res = _query(consul_url=consul_url, token=token, function=function, query_params=query_params) if res["res"]: ret["res"] = True - ret["message"] = "Service {} set in maintenance mode.".format(serviceid) + ret["message"] = f"Service {serviceid} set in maintenance mode." else: ret["res"] = False - ret["message"] = "Unable to set service {} to maintenance mode.".format( - serviceid - ) + ret["message"] = f"Unable to set service {serviceid} to maintenance mode." return ret @@ -1255,19 +1258,23 @@ def session_create(consul_url=None, token=None, **kwargs): ret["message"] = ("TTL must be ", "between 0 and 3600.") ret["res"] = False return ret - data["TTL"] = "{}s".format(_ttl) + data["TTL"] = f"{_ttl}s" function = "session/create" - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Created session {}.".format(kwargs["name"]) + ret["message"] = "Created session {}.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) else: ret["res"] = False - ret["message"] = "Unable to create session {}.".format(kwargs["name"]) + ret[ + "message" + ] = "Unable to create session {}.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -1305,9 +1312,7 @@ def session_list(consul_url=None, token=None, return_list=False, **kwargs): query_params["dc"] = kwargs["dc"] function = "session/list" - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) if return_list: _list = [] @@ -1351,7 +1356,7 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "session/destroy/{}".format(session) + function = f"session/destroy/{session}" res = _query( consul_url=consul_url, function=function, @@ -1361,10 +1366,10 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Destroyed Session {}.".format(session) + ret["message"] = f"Destroyed Session {session}." else: ret["res"] = False - ret["message"] = "Unable to destroy session {}.".format(session) + ret["message"] = f"Unable to destroy session {session}." return ret @@ -1402,10 +1407,8 @@ def session_info(consul_url=None, token=None, session=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "session/info/{}".format(session) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"session/info/{session}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1543,9 +1546,7 @@ def catalog_register(consul_url=None, token=None, **kwargs): "warning", "critical", ): - ret[ - "message" - ] = "Check status must be unknown, passing, warning, or critical." + ret["message"] = "Check status must be unknown, passing, warning, or critical." ret["res"] = False return ret data["Check"]["Status"] = kwargs["check_status"] @@ -1560,17 +1561,21 @@ def catalog_register(consul_url=None, token=None, **kwargs): data["Check"]["Notes"] = kwargs["check_notes"] function = "catalog/register" - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Catalog registration for {} successful.".format( + ret[ + "message" + ] = "Catalog registration for {} successful.".format( # pylint: disable=consider-using-f-string kwargs["node"] ) else: ret["res"] = False - ret["message"] = "Catalog registration for {} failed.".format(kwargs["node"]) + ret[ + "message" + ] = "Catalog registration for {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["node"] + ) ret["data"] = data return ret @@ -1621,16 +1626,22 @@ def catalog_deregister(consul_url=None, token=None, **kwargs): data["ServiceID"] = kwargs["serviceid"] function = "catalog/deregister" - res = _query( - consul_url=consul_url, function=function, token=token, method="PUT", data=data - ) + res = _query(consul_url=consul_url, function=function, token=token, method="PUT", data=data) if res["res"]: ret["res"] = True - ret["message"] = "Catalog item {} removed.".format(kwargs["node"]) + ret[ + "message" + ] = "Catalog item {} removed.".format( # pylint: disable=consider-using-f-string + kwargs["node"] + ) else: ret["res"] = False - ret["message"] = "Removing Catalog item {} failed.".format(kwargs["node"]) + ret[ + "message" + ] = "Removing Catalog item {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["node"] + ) return ret @@ -1692,9 +1703,7 @@ def catalog_nodes(consul_url=None, token=None, **kwargs): query_params["dc"] = kwargs["dc"] function = "catalog/nodes" - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1728,9 +1737,7 @@ def catalog_services(consul_url=None, token=None, **kwargs): query_params["dc"] = kwargs["dc"] function = "catalog/services" - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1770,10 +1777,8 @@ def catalog_service(consul_url=None, token=None, service=None, **kwargs): if "tag" in kwargs: query_params["tag"] = kwargs["tag"] - function = "catalog/service/{}".format(service) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"catalog/service/{service}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1810,10 +1815,8 @@ def catalog_node(consul_url=None, token=None, node=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "catalog/node/{}".format(node) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"catalog/node/{node}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1850,10 +1853,8 @@ def health_node(consul_url=None, token=None, node=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "health/node/{}".format(node) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"health/node/{node}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1890,10 +1891,8 @@ def health_checks(consul_url=None, token=None, service=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "health/checks/{}".format(service) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"health/checks/{service}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1941,10 +1940,8 @@ def health_service(consul_url=None, token=None, service=None, **kwargs): if "passing" in kwargs: query_params["passing"] = kwargs["passing"] - function = "health/service/{}".format(service) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"health/service/{service}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -1991,10 +1988,8 @@ def health_state(consul_url=None, token=None, state=None, **kwargs): ret["res"] = False return ret - function = "health/state/{}".format(state) - ret = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params - ) + function = f"health/state/{state}" + ret = _query(consul_url=consul_url, function=function, token=token, query_params=query_params) return ret @@ -2100,16 +2095,20 @@ def acl_create(consul_url=None, token=None, **kwargs): data["Rules"] = kwargs["rules"] function = "acl/create" - res = _query( - consul_url=consul_url, token=token, data=data, method="PUT", function=function - ) + res = _query(consul_url=consul_url, token=token, data=data, method="PUT", function=function) if res["res"]: ret["res"] = True - ret["message"] = "ACL {} created.".format(kwargs["name"]) + ret["message"] = "ACL {} created.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) else: ret["res"] = False - ret["message"] = "Removing Catalog item {} failed.".format(kwargs["name"]) + ret[ + "message" + ] = "Removing Catalog item {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -2163,16 +2162,20 @@ def acl_update(consul_url=None, token=None, **kwargs): data["Rules"] = kwargs["rules"] function = "acl/update" - res = _query( - consul_url=consul_url, token=token, data=data, method="PUT", function=function - ) + res = _query(consul_url=consul_url, token=token, data=data, method="PUT", function=function) if res["res"]: ret["res"] = True - ret["message"] = "ACL {} created.".format(kwargs["name"]) + ret["message"] = "ACL {} created.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) else: ret["res"] = False - ret["message"] = "Updating ACL {} failed.".format(kwargs["name"]) + ret[ + "message" + ] = "Updating ACL {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -2207,17 +2210,21 @@ def acl_delete(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - function = "acl/destroy/{}".format(kwargs["id"]) - res = _query( - consul_url=consul_url, token=token, data=data, method="PUT", function=function - ) + function = "acl/destroy/{}".format(kwargs["id"]) # pylint: disable=consider-using-f-string + res = _query(consul_url=consul_url, token=token, data=data, method="PUT", function=function) if res["res"]: ret["res"] = True - ret["message"] = "ACL {} deleted.".format(kwargs["id"]) + ret["message"] = "ACL {} deleted.".format( # pylint: disable=consider-using-f-string + kwargs["id"] + ) else: ret["res"] = False - ret["message"] = "Removing ACL {} failed.".format(kwargs["id"]) + ret[ + "message" + ] = "Removing ACL {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["id"] + ) return ret @@ -2252,7 +2259,7 @@ def acl_info(consul_url=None, **kwargs): ret["res"] = False return ret - function = "acl/info/{}".format(kwargs["id"]) + function = "acl/info/{}".format(kwargs["id"]) # pylint: disable=consider-using-f-string ret = _query(consul_url=consul_url, data=data, method="GET", function=function) return ret @@ -2288,17 +2295,21 @@ def acl_clone(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - function = "acl/clone/{}".format(kwargs["id"]) - res = _query( - consul_url=consul_url, token=token, data=data, method="PUT", function=function - ) + function = "acl/clone/{}".format(kwargs["id"]) # pylint: disable=consider-using-f-string + res = _query(consul_url=consul_url, token=token, data=data, method="PUT", function=function) if res["res"]: ret["res"] = True - ret["message"] = "ACL {} cloned.".format(kwargs["name"]) + ret["message"] = "ACL {} cloned.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) ret["ID"] = res["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) + ret[ + "message" + ] = "Cloning ACL item {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -2327,9 +2338,7 @@ def acl_list(consul_url=None, token=None, **kwargs): return ret function = "acl/list" - ret = _query( - consul_url=consul_url, token=token, data=data, method="GET", function=function - ) + ret = _query(consul_url=consul_url, token=token, data=data, method="GET", function=function) return ret @@ -2378,7 +2387,7 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): if "tag" in kwargs: query_params = kwargs["tag"] - function = "event/fire/{}".format(name) + function = f"event/fire/{name}" res = _query( consul_url=consul_url, token=token, @@ -2389,11 +2398,15 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Event {} fired.".format(name) + ret["message"] = f"Event {name} fired." ret["data"] = res["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) + ret[ + "message" + ] = "Cloning ACL item {} failed.".format( # pylint: disable=consider-using-f-string + kwargs["name"] + ) return ret @@ -2428,7 +2441,5 @@ def event_list(consul_url=None, token=None, **kwargs): raise SaltInvocationError('Required argument "name" is missing.') function = "event/list/" - ret = _query( - consul_url=consul_url, token=token, query_params=query_params, function=function - ) + ret = _query(consul_url=consul_url, token=token, query_params=query_params, function=function) return ret diff --git a/src/saltext/consul/modules/consul_mod.py b/src/saltext/consul/modules/consul_mod.py new file mode 100644 index 0000000..501a21f --- /dev/null +++ b/src/saltext/consul/modules/consul_mod.py @@ -0,0 +1,27 @@ +""" +Salt execution module +""" +import logging + +log = logging.getLogger(__name__) + +__virtualname__ = "consul" + + +def __virtual__(): + # To force a module not to load return something like: + # return (False, "The consul execution module is not implemented yet") + return __virtualname__ + + +def example_function(text): + """ + This example function should be replaced + + CLI Example: + + .. code-block:: bash + + salt '*' consul.example_function text="foo bar" + """ + return __salt__["test.echo"](text) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4f83ecb..e69de29 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1,1380 +0,0 @@ -""" -Render the pillar data -""" - -import collections -import copy -import fnmatch -import logging -import os -import sys -import time -import traceback - -import tornado.gen - -import salt.channel.client -import salt.fileclient -import salt.loader -import salt.minion -import salt.utils.args -import salt.utils.cache -import salt.utils.crypt -import salt.utils.data -import salt.utils.dictupdate -import salt.utils.url -from salt.exceptions import SaltClientError -from salt.template import compile_template - -# Even though dictupdate is imported, invoking salt.utils.dictupdate.merge here -# causes an UnboundLocalError. This should be investigated and fixed, but until -# then, leave the import directly below this comment intact. -from salt.utils.dictupdate import merge -from salt.utils.odict import OrderedDict -from salt.version import __version__ - -log = logging.getLogger(__name__) - - -def get_pillar( - opts, - grains, - minion_id, - saltenv=None, - ext=None, - funcs=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - clean_cache=False, -): - """ - Return the correct pillar driver based on the file_client option - """ - # When file_client is 'local' this makes the minion masterless - # but sometimes we want the minion to read its files from the local - # filesystem instead of asking for them from the master, but still - # get commands from the master. - # To enable this functionality set file_client=local and - # use_master_when_local=True in the minion config. Then here we override - # the file client to be 'remote' for getting pillar. If we don't do this - # then the minion never sends the event that the master uses to update - # its minion_data_cache. If the master doesn't update the minion_data_cache - # then the SSE salt-master plugin won't see any grains for those minions. - file_client = opts["file_client"] - if opts.get("master_type") == "disable" and file_client == "remote": - file_client = "local" - elif file_client == "local" and opts.get("use_master_when_local"): - file_client = "remote" - - ptype = {"remote": RemotePillar, "local": Pillar}.get(file_client, Pillar) - # If local pillar and we're caching, run through the cache system first - log.debug("Determining pillar cache") - if opts["pillar_cache"]: - log.debug("get_pillar using pillar cache with ext: %s", ext) - return PillarCache( - opts, - grains, - minion_id, - saltenv, - ext=ext, - functions=funcs, - pillar_override=pillar_override, - pillarenv=pillarenv, - clean_cache=clean_cache, - extra_minion_data=extra_minion_data, - ) - return ptype( - opts, - grains, - minion_id, - saltenv, - ext, - functions=funcs, - pillar_override=pillar_override, - pillarenv=pillarenv, - extra_minion_data=extra_minion_data, - ) - - -# TODO: migrate everyone to this one! -def get_async_pillar( - opts, - grains, - minion_id, - saltenv=None, - ext=None, - funcs=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - clean_cache=False, -): - """ - Return the correct pillar driver based on the file_client option - """ - file_client = opts["file_client"] - if opts.get("master_type") == "disable" and file_client == "remote": - file_client = "local" - elif file_client == "local" and opts.get("use_master_when_local"): - file_client = "remote" - ptype = {"remote": AsyncRemotePillar, "local": AsyncPillar}.get( - file_client, AsyncPillar - ) - if file_client == "remote": - # AsyncPillar does not currently support calls to PillarCache - # clean_cache is a kwarg for PillarCache - return ptype( - opts, - grains, - minion_id, - saltenv, - ext, - functions=funcs, - pillar_override=pillar_override, - pillarenv=pillarenv, - extra_minion_data=extra_minion_data, - clean_cache=clean_cache, - ) - return ptype( - opts, - grains, - minion_id, - saltenv, - ext, - functions=funcs, - pillar_override=pillar_override, - pillarenv=pillarenv, - extra_minion_data=extra_minion_data, - ) - - -class RemotePillarMixin: - """ - Common remote pillar functionality - """ - - def get_ext_pillar_extra_minion_data(self, opts): - """ - Returns the extra data from the minion's opts dict (the config file). - - This data will be passed to external pillar functions. - """ - - def get_subconfig(opts_key): - """ - Returns a dict containing the opts key subtree, while maintaining - the opts structure - """ - ret_dict = aux_dict = {} - config_val = opts - subkeys = opts_key.split(":") - # Build an empty dict with the opts path - for subkey in subkeys[:-1]: - aux_dict[subkey] = {} - aux_dict = aux_dict[subkey] - if not config_val.get(subkey): - # The subkey is not in the config - return {} - config_val = config_val[subkey] - if subkeys[-1] not in config_val: - return {} - aux_dict[subkeys[-1]] = config_val[subkeys[-1]] - return ret_dict - - extra_data = {} - if "pass_to_ext_pillars" in opts: - if not isinstance(opts["pass_to_ext_pillars"], list): - log.exception("'pass_to_ext_pillars' config is malformed.") - raise SaltClientError("'pass_to_ext_pillars' config is malformed.") - for key in opts["pass_to_ext_pillars"]: - salt.utils.dictupdate.update( - extra_data, - get_subconfig(key), - recursive_update=True, - merge_lists=True, - ) - log.trace("ext_pillar_extra_data = %s", extra_data) - return extra_data - - -class AsyncRemotePillar(RemotePillarMixin): - """ - Get the pillar from the master - """ - - def __init__( - self, - opts, - grains, - minion_id, - saltenv, - ext=None, - functions=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - clean_cache=False, - ): - self.opts = opts - self.opts["saltenv"] = saltenv - self.ext = ext - self.grains = grains - self.minion_id = minion_id - self.channel = salt.channel.client.AsyncReqChannel.factory(opts) - if pillarenv is not None: - self.opts["pillarenv"] = pillarenv - self.pillar_override = pillar_override or {} - if not isinstance(self.pillar_override, dict): - self.pillar_override = {} - log.error("Pillar data must be a dictionary") - self.extra_minion_data = extra_minion_data or {} - if not isinstance(self.extra_minion_data, dict): - self.extra_minion_data = {} - log.error("Extra minion data must be a dictionary") - salt.utils.dictupdate.update( - self.extra_minion_data, - self.get_ext_pillar_extra_minion_data(opts), - recursive_update=True, - merge_lists=True, - ) - self._closing = False - self.clean_cache = clean_cache - - @tornado.gen.coroutine - def compile_pillar(self): - """ - Return a future which will contain the pillar data from the master - """ - load = { - "id": self.minion_id, - "grains": self.grains, - "saltenv": self.opts["saltenv"], - "pillarenv": self.opts["pillarenv"], - "pillar_override": self.pillar_override, - "extra_minion_data": self.extra_minion_data, - "ver": "2", - "cmd": "_pillar", - } - if self.clean_cache: - load["clean_cache"] = self.clean_cache - if self.ext: - load["ext"] = self.ext - try: - start = time.monotonic() - ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( - load, - dictkey="pillar", - ) - except salt.crypt.AuthenticationError as exc: - log.error(exc.message) - raise SaltClientError("Exception getting pillar.") - except salt.exceptions.SaltReqTimeoutError: - raise SaltClientError( - f"Pillar timed out after {int(time.monotonic() - start)} seconds" - ) - except Exception: # pylint: disable=broad-except - log.exception("Exception getting pillar:") - raise SaltClientError("Exception getting pillar.") - - if not isinstance(ret_pillar, dict): - msg = "Got a bad pillar from master, type {}, expecting dict: {}".format( - type(ret_pillar).__name__, ret_pillar - ) - log.error(msg) - # raise an exception! Pillar isn't empty, we can't sync it! - raise SaltClientError(msg) - raise tornado.gen.Return(ret_pillar) - - def destroy(self): - if self._closing: - return - - self._closing = True - self.channel.close() - - # pylint: disable=W1701 - def __del__(self): - self.destroy() - - # pylint: enable=W1701 - - -class RemotePillar(RemotePillarMixin): - """ - Get the pillar from the master - """ - - def __init__( - self, - opts, - grains, - minion_id, - saltenv, - ext=None, - functions=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - ): - self.opts = opts - self.opts["saltenv"] = saltenv - self.ext = ext - self.grains = grains - self.minion_id = minion_id - self.channel = salt.channel.client.ReqChannel.factory(opts) - if pillarenv is not None: - self.opts["pillarenv"] = pillarenv - self.pillar_override = pillar_override or {} - if not isinstance(self.pillar_override, dict): - self.pillar_override = {} - log.error("Pillar data must be a dictionary") - self.extra_minion_data = extra_minion_data or {} - if not isinstance(self.extra_minion_data, dict): - self.extra_minion_data = {} - log.error("Extra minion data must be a dictionary") - salt.utils.dictupdate.update( - self.extra_minion_data, - self.get_ext_pillar_extra_minion_data(opts), - recursive_update=True, - merge_lists=True, - ) - self._closing = False - - def compile_pillar(self): - """ - Return the pillar data from the master - """ - load = { - "id": self.minion_id, - "grains": self.grains, - "saltenv": self.opts["saltenv"], - "pillarenv": self.opts["pillarenv"], - "pillar_override": self.pillar_override, - "extra_minion_data": self.extra_minion_data, - "ver": "2", - "cmd": "_pillar", - } - if self.ext: - load["ext"] = self.ext - - try: - start = time.monotonic() - ret_pillar = self.channel.crypted_transfer_decode_dictentry( - load, - dictkey="pillar", - ) - except salt.crypt.AuthenticationError as exc: - log.error(exc.message) - raise SaltClientError("Exception getting pillar.") - except salt.exceptions.SaltReqTimeoutError: - raise SaltClientError( - f"Pillar timed out after {int(time.monotonic() - start)} seconds" - ) - except Exception: # pylint: disable=broad-except - log.exception("Exception getting pillar:") - raise SaltClientError("Exception getting pillar.") - - if not isinstance(ret_pillar, dict): - log.error( - "Got a bad pillar from master, type %s, expecting dict: %s", - type(ret_pillar).__name__, - ret_pillar, - ) - return {} - return ret_pillar - - def destroy(self): - if hasattr(self, "_closing") and self._closing: - return - - self._closing = True - self.channel.close() - - # pylint: disable=W1701 - def __del__(self): - self.destroy() - - # pylint: enable=W1701 - - -class PillarCache: - """ - Return a cached pillar if it exists, otherwise cache it. - - Pillar caches are structed in two diminensions: minion_id with a dict of - saltenvs. Each saltenv contains a pillar dict - - Example data structure: - - ``` - {'minion_1': - {'base': {'pilar_key_1' 'pillar_val_1'} - } - """ - - # TODO ABC? - def __init__( - self, - opts, - grains, - minion_id, - saltenv, - ext=None, - functions=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - clean_cache=False, - ): - # Yes, we need all of these because we need to route to the Pillar object - # if we have no cache. This is another refactor target. - - # Go ahead and assign these because they may be needed later - self.opts = opts - self.grains = grains - self.minion_id = minion_id - self.ext = ext - self.functions = functions - self.pillar_override = pillar_override - self.pillarenv = pillarenv - self.clean_cache = clean_cache - self.extra_minion_data = extra_minion_data - - if saltenv is None: - self.saltenv = "base" - else: - self.saltenv = saltenv - - # Determine caching backend - self.cache = salt.utils.cache.CacheFactory.factory( - self.opts["pillar_cache_backend"], - self.opts["pillar_cache_ttl"], - minion_cache_path=self._minion_cache_path(minion_id), - ) - - def _minion_cache_path(self, minion_id): - """ - Return the path to the cache file for the minion. - - Used only for disk-based backends - """ - return os.path.join(self.opts["cachedir"], "pillar_cache", minion_id) - - def fetch_pillar(self): - """ - In the event of a cache miss, we need to incur the overhead of caching - a new pillar. - """ - log.debug("Pillar cache getting external pillar with ext: %s", self.ext) - fresh_pillar = Pillar( - self.opts, - self.grains, - self.minion_id, - self.saltenv, - ext=self.ext, - functions=self.functions, - pillar_override=self.pillar_override, - pillarenv=self.pillarenv, - extra_minion_data=self.extra_minion_data, - ) - return fresh_pillar.compile_pillar() - - def clear_pillar(self): - """ - Clear the cache - """ - self.cache.clear() - - return True - - def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs - if self.clean_cache: - self.clear_pillar() - log.debug( - "Scanning pillar cache for information about minion %s and pillarenv %s", - self.minion_id, - self.pillarenv, - ) - if self.opts["pillar_cache_backend"] == "memory": - cache_dict = self.cache - else: - cache_dict = self.cache._dict - - log.debug("Scanning cache: %s", cache_dict) - # Check the cache! - if self.minion_id in self.cache: # Keyed by minion_id - # TODO Compare grains, etc? - if self.pillarenv in self.cache[self.minion_id]: - # We have a cache hit! Send it back. - log.debug( - "Pillar cache hit for minion %s and pillarenv %s", - self.minion_id, - self.pillarenv, - ) - return self.cache[self.minion_id][self.pillarenv] - else: - # We found the minion but not the env. Store it. - fresh_pillar = self.fetch_pillar() - - minion_cache = self.cache[self.minion_id] - minion_cache[self.pillarenv] = fresh_pillar - self.cache[self.minion_id] = minion_cache - - log.debug( - "Pillar cache miss for pillarenv %s for minion %s", - self.pillarenv, - self.minion_id, - ) - return fresh_pillar - else: - # We haven't seen this minion yet in the cache. Store it. - fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} - log.debug("Pillar cache miss for minion %s", self.minion_id) - log.debug("Current pillar cache: %s", cache_dict) # FIXME hack! - return fresh_pillar - - -class Pillar: - """ - Read over the pillar top files and render the pillar data - """ - - def __init__( - self, - opts, - grains, - minion_id, - saltenv, - ext=None, - functions=None, - pillar_override=None, - pillarenv=None, - extra_minion_data=None, - ): - self.minion_id = minion_id - self.ext = ext - if pillarenv is None: - if opts.get("pillarenv_from_saltenv", False): - opts["pillarenv"] = saltenv - # use the local file client - self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) - self.saltenv = saltenv - self.client = salt.fileclient.get_file_client(self.opts, True) - self.avail = self.__gather_avail() - - if opts.get("file_client", "") == "local" and not opts.get( - "use_master_when_local", False - ): - opts["grains"] = grains - - # if we didn't pass in functions, lets load them - if functions is None: - utils = salt.loader.utils(opts) - if opts.get("file_client", "") == "local": - self.functions = salt.loader.minion_mods(opts, utils=utils) - else: - self.functions = salt.loader.minion_mods(self.opts, utils=utils) - else: - self.functions = functions - - self.opts["minion_id"] = minion_id - self.matchers = salt.loader.matchers(self.opts) - self.rend = salt.loader.render(self.opts, self.functions) - ext_pillar_opts = copy.deepcopy(self.opts) - # Keep the incoming opts ID intact, ie, the master id - if "id" in opts: - ext_pillar_opts["id"] = opts["id"] - self.merge_strategy = "smart" - if opts.get("pillar_source_merging_strategy"): - self.merge_strategy = opts["pillar_source_merging_strategy"] - - self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) - self.ignored_pillars = {} - self.pillar_override = pillar_override or {} - if not isinstance(self.pillar_override, dict): - self.pillar_override = {} - log.error("Pillar data must be a dictionary") - self.extra_minion_data = extra_minion_data or {} - if not isinstance(self.extra_minion_data, dict): - self.extra_minion_data = {} - log.error("Extra minion data must be a dictionary") - self._closing = False - - def __valid_on_demand_ext_pillar(self, opts): - """ - Check to see if the on demand external pillar is allowed - """ - if not isinstance(self.ext, dict): - log.error("On-demand pillar %s is not formatted as a dictionary", self.ext) - return False - - on_demand = opts.get("on_demand_ext_pillar", []) - try: - invalid_on_demand = {x for x in self.ext if x not in on_demand} - except TypeError: - # Prevent traceback when on_demand_ext_pillar option is malformed - log.error( - "The 'on_demand_ext_pillar' configuration option is " - "malformed, it should be a list of ext_pillar module names" - ) - return False - - if invalid_on_demand: - log.error( - "The following ext_pillar modules are not allowed for " - "on-demand pillar data: %s. Valid on-demand ext_pillar " - "modules are: %s. The valid modules can be adjusted by " - "setting the 'on_demand_ext_pillar' config option.", - ", ".join(sorted(invalid_on_demand)), - ", ".join(on_demand), - ) - return False - return True - - def __gather_avail(self): - """ - Gather the lists of available sls data from the master - """ - avail = {} - for saltenv in self._get_envs(): - avail[saltenv] = self.client.list_states(saltenv) - return avail - - def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): - """ - The options need to be altered to conform to the file client - """ - opts = copy.deepcopy(opts_in) - opts["file_client"] = "local" - if not grains: - opts["grains"] = {} - else: - opts["grains"] = grains - # Allow minion/CLI saltenv/pillarenv to take precedence over master - opts["saltenv"] = saltenv if saltenv is not None else opts.get("saltenv") - opts["pillarenv"] = ( - pillarenv if pillarenv is not None else opts.get("pillarenv") - ) - opts["id"] = self.minion_id - if opts["state_top"].startswith("salt://"): - opts["state_top"] = opts["state_top"] - elif opts["state_top"].startswith("/"): - opts["state_top"] = salt.utils.url.create(opts["state_top"][1:]) - else: - opts["state_top"] = salt.utils.url.create(opts["state_top"]) - if self.ext and self.__valid_on_demand_ext_pillar(opts): - if "ext_pillar" in opts: - opts["ext_pillar"].append(self.ext) - else: - opts["ext_pillar"] = [self.ext] - if "__env__" in opts["pillar_roots"]: - env = opts.get("pillarenv") or opts.get("saltenv") or "base" - if env not in opts["pillar_roots"]: - log.debug( - "pillar environment '%s' maps to __env__ pillar_roots directory", - env, - ) - opts["pillar_roots"][env] = opts["pillar_roots"].pop("__env__") - for idx, root in enumerate(opts["pillar_roots"][env]): - opts["pillar_roots"][env][idx] = opts["pillar_roots"][env][ - idx - ].replace("__env__", env) - else: - log.debug( - "pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", - env, - ) - opts["pillar_roots"].pop("__env__") - return opts - - def _get_envs(self): - """ - Pull the file server environments out of the master options - """ - envs = ["base"] - if "pillar_roots" in self.opts: - envs.extend([x for x in list(self.opts["pillar_roots"]) if x not in envs]) - return envs - - def get_tops(self): - """ - Gather the top files - """ - tops = collections.defaultdict(list) - include = collections.defaultdict(list) - done = collections.defaultdict(list) - errors = [] - # Gather initial top files - try: - saltenvs = set() - if self.opts["pillarenv"]: - # If the specified pillarenv is not present in the available - # pillar environments, do not cache the pillar top file. - if self.opts["pillarenv"] not in self.opts["pillar_roots"]: - log.debug( - "pillarenv '%s' not found in the configured pillar " - "environments (%s)", - self.opts["pillarenv"], - ", ".join(self.opts["pillar_roots"]), - ) - else: - saltenvs.add(self.opts["pillarenv"]) - else: - saltenvs.update(self._get_envs()) - if self.opts.get("pillar_source_merging_strategy", None) == "none": - saltenvs &= {self.saltenv or "base"} - - for saltenv in saltenvs: - top = self.client.cache_file(self.opts["state_top"], saltenv) - if top: - tops[saltenv].append( - compile_template( - top, - self.rend, - self.opts["renderer"], - self.opts["renderer_blacklist"], - self.opts["renderer_whitelist"], - saltenv=saltenv, - _pillar_rend=True, - ) - ) - except Exception as exc: # pylint: disable=broad-except - errors.append(f"Rendering Primary Top file failed, render error:\n{exc}") - log.exception("Pillar rendering failed for minion %s", self.minion_id) - - # Search initial top files for includes - for saltenv, ctops in tops.items(): - for ctop in ctops: - if "include" not in ctop: - continue - for sls in ctop["include"]: - include[saltenv].append(sls) - ctop.pop("include") - # Go through the includes and pull out the extra tops and add them - while include: - pops = [] - for saltenv, states in include.items(): - pops.append(saltenv) - if not states: - continue - for sls in states: - if sls in done[saltenv]: - continue - try: - tops[saltenv].append( - compile_template( - self.client.get_state(sls, saltenv).get("dest", False), - self.rend, - self.opts["renderer"], - self.opts["renderer_blacklist"], - self.opts["renderer_whitelist"], - saltenv=saltenv, - _pillar_rend=True, - ) - ) - except Exception as exc: # pylint: disable=broad-except - errors.append( - "Rendering Top file {} failed, render error:\n{}".format( - sls, exc - ) - ) - done[saltenv].append(sls) - for saltenv in pops: - if saltenv in include: - include.pop(saltenv) - - return tops, errors - - def merge_tops(self, tops): - """ - Cleanly merge the top files - """ - top = collections.defaultdict(OrderedDict) - orders = collections.defaultdict(OrderedDict) - for ctops in tops.values(): - for ctop in ctops: - for saltenv, targets in ctop.items(): - if saltenv == "include": - continue - for tgt in targets: - matches = [] - states = OrderedDict() - orders[saltenv][tgt] = 0 - ignore_missing = False - for comp in ctop[saltenv][tgt]: - if isinstance(comp, dict): - if "match" in comp: - matches.append(comp) - if "order" in comp: - order = comp["order"] - if not isinstance(order, int): - try: - order = int(order) - except ValueError: - order = 0 - orders[saltenv][tgt] = order - if comp.get("ignore_missing", False): - ignore_missing = True - if isinstance(comp, str): - states[comp] = True - if ignore_missing: - if saltenv not in self.ignored_pillars: - self.ignored_pillars[saltenv] = [] - self.ignored_pillars[saltenv].extend(states.keys()) - top[saltenv][tgt] = matches - top[saltenv][tgt].extend(states) - return self.sort_top_targets(top, orders) - - def sort_top_targets(self, top, orders): - """ - Returns the sorted high data from the merged top files - """ - sorted_top = collections.defaultdict(OrderedDict) - # pylint: disable=cell-var-from-loop - for saltenv, targets in top.items(): - sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) - for target in sorted_targets: - sorted_top[saltenv][target] = targets[target] - # pylint: enable=cell-var-from-loop - return sorted_top - - def get_top(self): - """ - Returns the high data derived from the top file - """ - tops, errors = self.get_tops() - try: - merged_tops = self.merge_tops(tops) - except TypeError as err: - merged_tops = OrderedDict() - errors.append("Error encountered while rendering pillar top file.") - return merged_tops, errors - - def top_matches(self, top, reload=False): - """ - Search through the top high data for matches and return the states - that this minion needs to execute. - - Returns: - {'saltenv': ['state1', 'state2', ...]} - - reload - Reload the matcher loader - """ - matches = {} - if reload: - self.matchers = salt.loader.matchers(self.opts) - for saltenv, body in top.items(): - if self.opts["pillarenv"]: - if saltenv != self.opts["pillarenv"]: - continue - for match, data in body.items(): - if self.matchers["confirm_top.confirm_top"]( - match, - data, - self.opts.get("nodegroups", {}), - ): - if saltenv not in matches: - matches[saltenv] = env_matches = [] - else: - env_matches = matches[saltenv] - for item in data: - if isinstance(item, str) and item not in env_matches: - env_matches.append(item) - return matches - - def render_pstate(self, sls, saltenv, mods, defaults=None): - """ - Collect a single pillar sls file and render it - """ - if defaults is None: - defaults = {} - err = "" - errors = [] - state_data = self.client.get_state(sls, saltenv) - fn_ = state_data.get("dest", False) - if not fn_: - if sls in self.ignored_pillars.get(saltenv, []): - log.debug( - "Skipping ignored and missing SLS '%s' in environment '%s'", - sls, - saltenv, - ) - return None, mods, errors - elif self.opts["pillar_roots"].get(saltenv): - msg = ( - "Specified SLS '{}' in environment '{}' is not" - " available on the salt master".format(sls, saltenv) - ) - log.error(msg) - errors.append(msg) - else: - msg = "Specified SLS '{}' in environment '{}' was not found. ".format( - sls, saltenv - ) - if self.opts.get("__git_pillar", False) is True: - msg += ( - "This is likely caused by a git_pillar top file " - "containing an environment other than the one for the " - "branch in which it resides. Each git_pillar " - "branch/tag must have its own top file." - ) - else: - msg += ( - "This could be because SLS '{0}' is in an " - "environment other than '{1}', but '{1}' is " - "included in that environment's Pillar top file. It " - "could also be due to environment '{1}' not being " - "defined in 'pillar_roots'.".format(sls, saltenv) - ) - log.debug(msg) - # return state, mods, errors - return None, mods, errors - state = None - try: - state = compile_template( - fn_, - self.rend, - self.opts["renderer"], - self.opts["renderer_blacklist"], - self.opts["renderer_whitelist"], - saltenv, - sls, - _pillar_rend=True, - **defaults, - ) - except Exception as exc: # pylint: disable=broad-except - msg = f"Rendering SLS '{sls}' failed, render error:\n{exc}" - log.critical(msg, exc_info=True) - if self.opts.get("pillar_safe_render_error", True): - errors.append( - "Rendering SLS '{}' failed. Please see master log for " - "details.".format(sls) - ) - else: - errors.append(msg) - mods[sls] = state - nstate = None - if state: - if not isinstance(state, dict): - msg = f"SLS '{sls}' does not render to a dictionary" - log.error(msg) - errors.append(msg) - else: - if "include" in state: - if not isinstance(state["include"], list): - msg = ( - "Include Declaration in SLS '{}' is not " - "formed as a list".format(sls) - ) - log.error(msg) - errors.append(msg) - else: - # render included state(s) - include_states = [] - for sub_sls in state.pop("include"): - if isinstance(sub_sls, dict): - sub_sls, v = next(iter(sub_sls.items())) - defaults = v.get("defaults", {}) - key = v.get("key", None) - else: - key = None - try: - matched_pstates = fnmatch.filter( - self.avail[saltenv], - sub_sls.lstrip(".").replace("/", "."), - ) - if sub_sls.startswith("."): - if state_data.get("source", "").endswith( - "/init.sls" - ): - include_parts = sls.split(".") - else: - include_parts = sls.split(".")[:-1] - sub_sls = ".".join(include_parts + [sub_sls[1:]]) - matches = fnmatch.filter( - self.avail[saltenv], - sub_sls, - ) - matched_pstates.extend(matches) - except KeyError: - errors.extend( - [ - "No matching pillar environment for environment" - " '{}' found".format(saltenv) - ] - ) - matched_pstates = [sub_sls] - # If matched_pstates is empty, set to sub_sls - if len(matched_pstates) < 1: - matched_pstates = [sub_sls] - for m_sub_sls in matched_pstates: - if m_sub_sls not in mods: - nstate, mods, err = self.render_pstate( - m_sub_sls, saltenv, mods, defaults - ) - else: - nstate = mods[m_sub_sls] - if nstate: - if key: - # If key is x:y, convert it to {x: {y: nstate}} - for key_fragment in reversed(key.split(":")): - nstate = {key_fragment: nstate} - if not self.opts.get( - "pillar_includes_override_sls", False - ): - include_states.append(nstate) - else: - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - if err: - errors += err - if not self.opts.get("pillar_includes_override_sls", False): - # merge included state(s) with the current state - # merged last to ensure that its values are - # authoritative. - include_states.append(state) - state = None - for s in include_states: - if state is None: - state = s - else: - state = merge( - state, - s, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - return state, mods, errors - - def render_pillar(self, matches, errors=None): - """ - Extract the sls pillar files from the matches and render them into the - pillar - """ - pillar = copy.copy(self.pillar_override) - if errors is None: - errors = [] - for saltenv, pstates in matches.items(): - pstatefiles = [] - mods = {} - for sls_match in pstates: - matched_pstates = [] - try: - matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match) - except KeyError: - errors.extend( - [ - "No matching pillar environment for environment " - "'{}' found".format(saltenv) - ] - ) - if matched_pstates: - pstatefiles.extend(matched_pstates) - else: - pstatefiles.append(sls_match) - - for sls in pstatefiles: - pstate, mods, err = self.render_pstate(sls, saltenv, mods) - - if err: - errors += err - - if pstate is not None: - if not isinstance(pstate, dict): - log.error( - "The rendered pillar sls file, '%s' state did " - "not return the expected data format. This is " - "a sign of a malformed pillar sls file. Returned " - "errors: %s", - sls, - ", ".join([f"'{e}'" for e in errors]), - ) - continue - pillar = merge( - pillar, - pstate, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - - return pillar, errors - - def _external_pillar_data(self, pillar, val, key): - """ - Builds actual pillar data structure and updates the ``pillar`` variable - """ - ext = None - args = salt.utils.args.get_function_argspec(self.ext_pillars[key]).args - - if isinstance(val, dict): - if ("extra_minion_data" in args) and self.extra_minion_data: - ext = self.ext_pillars[key]( - self.minion_id, - pillar, - extra_minion_data=self.extra_minion_data, - **val, - ) - else: - ext = self.ext_pillars[key](self.minion_id, pillar, **val) - elif isinstance(val, list): - if ("extra_minion_data" in args) and self.extra_minion_data: - ext = self.ext_pillars[key]( - self.minion_id, - pillar, - *val, - extra_minion_data=self.extra_minion_data, - ) - else: - ext = self.ext_pillars[key](self.minion_id, pillar, *val) - else: - if ("extra_minion_data" in args) and self.extra_minion_data: - ext = self.ext_pillars[key]( - self.minion_id, - pillar, - val, - extra_minion_data=self.extra_minion_data, - ) - else: - ext = self.ext_pillars[key](self.minion_id, pillar, val) - return ext - - def ext_pillar(self, pillar, errors=None): - """ - Render the external pillar data - """ - if errors is None: - errors = [] - try: - # Make sure that on-demand git_pillar is fetched before we try to - # compile the pillar data. git_pillar will fetch a remote when - # the git ext_pillar() func is run, but only for masterless. - if self.ext and "git" in self.ext and self.opts.get("__role") != "minion": - # Avoid circular import - import salt.pillar.git_pillar - import salt.utils.gitfs - - git_pillar = salt.utils.gitfs.GitPillar( - self.opts, - self.ext["git"], - per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY, - global_only=salt.pillar.git_pillar.GLOBAL_ONLY, - ) - git_pillar.fetch_remotes() - except TypeError: - # Handle malformed ext_pillar - pass - if "ext_pillar" not in self.opts: - return pillar, errors - if not isinstance(self.opts["ext_pillar"], list): - errors.append('The "ext_pillar" option is malformed') - log.critical(errors[-1]) - return pillar, errors - ext = None - # Bring in CLI pillar data - if self.pillar_override: - pillar = merge( - pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - - for run in self.opts["ext_pillar"]: - if not isinstance(run, dict): - errors.append('The "ext_pillar" option is malformed') - log.critical(errors[-1]) - return {}, errors - if next(iter(run.keys())) in self.opts.get("exclude_ext_pillar", []): - continue - for key, val in run.items(): - if key not in self.ext_pillars: - log.critical( - "Specified ext_pillar interface %s is unavailable", key - ) - continue - try: - ext = self._external_pillar_data(pillar, val, key) - except Exception as exc: # pylint: disable=broad-except - errors.append( - "Failed to load ext_pillar {}: {}".format( - key, - exc.__str__(), - ) - ) - log.error( - "Exception caught loading ext_pillar '%s':\n%s", - key, - "".join(traceback.format_tb(sys.exc_info()[2])), - ) - if ext: - pillar = merge( - pillar, - ext, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - ext = None - return pillar, errors - - def compile_pillar(self, ext=True): - """ - Render the pillar data and return - """ - top, top_errors = self.get_top() - if ext: - if self.opts.get("ext_pillar_first", False): - self.opts["pillar"], errors = self.ext_pillar(self.pillar_override) - self.rend = salt.loader.render(self.opts, self.functions) - matches = self.top_matches(top, reload=True) - pillar, errors = self.render_pillar(matches, errors=errors) - pillar = merge( - self.opts["pillar"], - pillar, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - else: - matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) - pillar, errors = self.ext_pillar(pillar, errors=errors) - else: - matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) - errors.extend(top_errors) - if self.opts.get("pillar_opts", False): - mopts = dict(self.opts) - if "grains" in mopts: - mopts.pop("grains") - mopts["saltversion"] = __version__ - pillar["master"] = mopts - if "pillar" in self.opts and self.opts.get("ssh_merge_pillar", False): - pillar = merge( - self.opts["pillar"], - pillar, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - if errors: - for error in errors: - log.critical("Pillar render error: %s", error) - pillar["_errors"] = errors - - if self.pillar_override: - pillar = merge( - pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get("pillar_merge_lists", False), - ) - - decrypt_errors = self.decrypt_pillar(pillar) - if decrypt_errors: - pillar.setdefault("_errors", []).extend(decrypt_errors) - return pillar - - def decrypt_pillar(self, pillar): - """ - Decrypt the specified pillar dictionary items, if configured to do so - """ - errors = [] - if self.opts.get("decrypt_pillar"): - decrypt_pillar = self.opts["decrypt_pillar"] - if not isinstance(decrypt_pillar, dict): - decrypt_pillar = salt.utils.data.repack_dictlist( - self.opts["decrypt_pillar"] - ) - if not decrypt_pillar: - errors.append("decrypt_pillar config option is malformed") - for key, rend in decrypt_pillar.items(): - ptr = salt.utils.data.traverse_dict( - pillar, - key, - default=None, - delimiter=self.opts["decrypt_pillar_delimiter"], - ) - if ptr is None: - log.debug("Pillar key %s not present", key) - continue - try: - hash(ptr) - immutable = True - except TypeError: - immutable = False - try: - ret = salt.utils.crypt.decrypt( - ptr, - rend or self.opts["decrypt_pillar_default"], - renderers=self.rend, - opts=self.opts, - valid_rend=self.opts["decrypt_pillar_renderers"], - ) - if immutable: - # Since the key pointed to an immutable type, we need - # to replace it in the pillar dict. First we will find - # the parent, and then we will replace the child key - # with the return data from the renderer. - parent, _, child = key.rpartition( - self.opts["decrypt_pillar_delimiter"] - ) - if not parent: - # key is a top-level key, so the pointer to the - # parent is the pillar dict itself. - ptr = pillar - else: - ptr = salt.utils.data.traverse_dict( - pillar, - parent, - default=None, - delimiter=self.opts["decrypt_pillar_delimiter"], - ) - if ptr is not None: - ptr[child] = ret - except Exception as exc: # pylint: disable=broad-except - msg = f"Failed to decrypt pillar key '{key}': {exc}" - errors.append(msg) - log.error(msg, exc_info=True) - return errors - - def destroy(self): - """ - This method exist in order to be API compatible with RemotePillar - """ - if self._closing: - return - self._closing = True - if self.client: - try: - self.client.destroy() - except AttributeError: - pass - - # pylint: disable=W1701 - def __del__(self): - self.destroy() - - # pylint: enable=W1701 - - -# TODO: actually migrate from Pillar to AsyncPillar to allow for futures in -# ext_pillar etc. -class AsyncPillar(Pillar): - @tornado.gen.coroutine - def compile_pillar(self, ext=True): - ret = super().compile_pillar(ext=ext) - raise tornado.gen.Return(ret) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 2b28b29..fe9dc53 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -134,7 +134,6 @@ - consul: my_consul_config expand_keys=false """ - import logging import re @@ -208,9 +207,7 @@ def ext_pillar(minion_id, pillar, conf): # pylint: disable=W0613 else: opts["profile"] = None - expand_keys_re = re.compile( - "expand_keys=False", re.IGNORECASE - ) # pylint: disable=W1401 + expand_keys_re = re.compile("expand_keys=False", re.IGNORECASE) # pylint: disable=W1401 match = expand_keys_re.search(temp) if match: opts["expand_keys"] = False diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py index 1db699f..72671df 100644 --- a/src/saltext/consul/sdb/consul.py +++ b/src/saltext/consul/sdb/consul.py @@ -26,7 +26,6 @@ The ``driver`` refers to the Consul module, all other options are optional. For option details see: https://python-consul.readthedocs.io/en/latest/#consul """ - from salt.exceptions import CommandExecutionError try: diff --git a/src/saltext/consul/states/consul.py b/src/saltext/consul/states/consul.py index b35b726..f6e425c 100644 --- a/src/saltext/consul/states/consul.py +++ b/src/saltext/consul/states/consul.py @@ -92,18 +92,18 @@ def acl_present( rules Specifies rules for this ACL token. - consul_url : http://locahost:8500 + consul_url : http://localhost:8500 consul URL to query .. note:: - For more information https://www.consul.io/api/acl.html#create-acl-token, https://www.consul.io/api/acl.html#update-acl-token + For more information https://www.consul.io/api/acl.html#bootstrap-acls """ ret = { "name": name, "changes": {}, "result": True, - "comment": 'ACL "{}" exists and is up to date'.format(name), + "comment": f'ACL "{name}" exists and is up to date', } exists = _acl_exists(name, id, token, consul_url) @@ -169,18 +169,18 @@ def acl_absent(name, id=None, token=None, consul_url="http://localhost:8500"): token token to authenticate you Consul query - consul_url : http://locahost:8500 + consul_url : http://localhost:8500 consul URL to query .. note:: - For more information https://www.consul.io/api/acl.html#delete-acl-token + For more information https://www.consul.io/api/acl.html#bootstrap-acls """ ret = { "name": id, "changes": {}, "result": True, - "comment": 'ACL "{}" does not exist'.format(id), + "comment": f'ACL "{id}" does not exist', } exists = _acl_exists(name, id, token, consul_url) @@ -190,9 +190,7 @@ def acl_absent(name, id=None, token=None, consul_url="http://localhost:8500"): ret["comment"] = "The acl exists, it will be deleted" return ret - delete = __salt__["consul.acl_delete"]( - id=exists["id"], token=token, consul_url=consul_url - ) + delete = __salt__["consul.acl_delete"](id=exists["id"], token=token, consul_url=consul_url) if delete["res"]: ret["result"] = True ret["comment"] = "The acl has been deleted" diff --git a/src/saltext/consul/states/consul_mod.py b/src/saltext/consul/states/consul_mod.py new file mode 100644 index 0000000..8e9db1f --- /dev/null +++ b/src/saltext/consul/states/consul_mod.py @@ -0,0 +1,30 @@ +""" +Salt state module +""" +import logging + +log = logging.getLogger(__name__) + +__virtualname__ = "consul" + + +def __virtual__(): + # To force a module not to load return something like: + # return (False, "The consul state module is not implemented yet") + + # Replace this with your own logic + if "consul.example_function" not in __salt__: + return False, "The 'consul' execution module is not available" + return __virtualname__ + + +def exampled(name): + """ + This example function should be replaced + """ + ret = {"name": name, "changes": {}, "result": False, "comment": ""} + value = __salt__["consul.example_function"](name) + if value == name: + ret["result"] = True + ret["comment"] = f"The 'consul.example_function' returned: '{value}'" + return ret diff --git a/tests/pytests/functional/__init__.py b/tests/functional/__init__.py similarity index 100% rename from tests/pytests/functional/__init__.py rename to tests/functional/__init__.py diff --git a/tests/pytests/functional/cache/__init__.py b/tests/functional/cache/__init__.py similarity index 100% rename from tests/pytests/functional/cache/__init__.py rename to tests/functional/cache/__init__.py diff --git a/tests/functional/cache/helpers.py b/tests/functional/cache/helpers.py new file mode 100644 index 0000000..a6f4d4b --- /dev/null +++ b/tests/functional/cache/helpers.py @@ -0,0 +1,218 @@ +import time +from unittest.mock import MagicMock +from unittest.mock import patch + +import pytest +import salt.cache +from salt.exceptions import SaltCacheError + + +def run_common_cache_tests(subtests, cache): + bank = "fnord/kevin/stuart" + # ^^^^ This bank can be just fnord, or fnord/foo, or any mildly reasonable + # or possibly unreasonably nested names. + # + # No. Seriously. Try import string; bank = '/'.join(string.ascii_letters) + # - it works! + # import string; bank = "/".join(string.ascii_letters) + good_key = "roscivs" + bad_key = "monkey" + + with subtests.test("non-existent bank should be empty on cache start"): + assert not cache.contains(bank=bank) + assert cache.list(bank=bank) == [] + + with subtests.test("after storing key in bank it should be in cache list"): + cache.store(bank=bank, key=good_key, data=b"\x01\x04\x05fnordy data") + assert cache.list(bank) == [good_key] + + with subtests.test("after storing value, it should be fetchable"): + expected_data = "trombone pleasantry" + cache.store(bank=bank, key=good_key, data=expected_data) + assert cache.fetch(bank=bank, key=good_key) == expected_data + + with subtests.test("bad key should still be absent from cache"): + assert cache.fetch(bank=bank, key=bad_key) == {} + + with subtests.test("storing new value should update it"): + # Double check that the data was still the old stuff + old_data = expected_data + assert cache.fetch(bank=bank, key=good_key) == old_data + new_data = "stromboli" + cache.store(bank=bank, key=good_key, data=new_data) + assert cache.fetch(bank=bank, key=good_key) == new_data + + with subtests.test("storing complex object works"): + new_thing = { + "some": "data", + 42: "wheee", + "some other": {"sub": {"objects": "here"}}, + } + + cache.store(bank=bank, key=good_key, data=new_thing) + actual_thing = cache.fetch(bank=bank, key=good_key) + if isinstance(cache, salt.cache.MemCache): + # MemCache should actually store the object - everything else + # should create a copy of it. + assert actual_thing is new_thing + else: + assert actual_thing is not new_thing + assert actual_thing == new_thing + + with subtests.test("contains returns true if key in bank"): + assert cache.contains(bank=bank, key=good_key) + + with subtests.test("contains returns true if bank exists and key is None"): + assert cache.contains(bank=bank, key=None) + + with subtests.test("contains returns False when bank not in cache and/or key not in bank"): + assert not cache.contains(bank=bank, key=bad_key) + assert not cache.contains(bank="nonexistent", key=good_key) + assert not cache.contains(bank="nonexistent", key=bad_key) + assert not cache.contains(bank="nonexistent", key=None) + + with subtests.test("flushing nonexistent key should not remove other keys"): + cache.flush(bank=bank, key=bad_key) + assert cache.contains(bank=bank, key=good_key) + + with subtests.test("flushing existing key should not remove bank if no more keys exist"): + pytest.skip("This is impossible with redis. Should we make localfs behave the same way?") + cache.flush(bank=bank, key=good_key) + assert cache.contains(bank=bank) + assert cache.list(bank=bank) == [] + + with subtests.test( + "after existing key is flushed updated should not return a timestamp for that key" + ): + cache.store(bank=bank, key=good_key, data="fnord") + cache.flush(bank=bank, key=good_key) + timestamp = cache.updated(bank=bank, key=good_key) + assert timestamp is None + + with subtests.test( + "after flushing bank containing a good key, updated should not return a timestamp for that key" + ): + cache.store(bank=bank, key=good_key, data="fnord") + cache.flush(bank=bank, key=None) + timestamp = cache.updated(bank=bank, key=good_key) + assert timestamp is None + + with subtests.test("flushing bank with None as key should remove bank"): + cache.flush(bank=bank, key=None) + assert not cache.contains(bank=bank) + + with subtests.test("Exception should happen when flushing None bank"): + # This bit is maybe an accidental API, but currently there is no + # protection at least with the localfs cache when bank is None. If + # bank is None we try to `os.path.normpath` the bank, which explodes + # and is at least the current behavior. If we want to change that + # this test should change. Or be removed altogether. + # TODO: this should actually not raise. Not sure if there's a test that we can do here... or just call the code which will fail if there's actually an exception. -W. Werner, 2021-09-28 + pytest.skip( + "Skipping for now - etcd, redis, and mysql do not raise. Should ensure all backends behave consistently" + ) + with pytest.raises(Exception): + cache.flush(bank=None, key=None) + + with subtests.test("Updated for non-existent key should return None"): + timestamp = cache.updated(bank="nonexistent", key="whatever") + assert timestamp is None + + with subtests.test("Updated for key should return a reasonable time"): + before_storage = int(time.time()) + cache.store(bank="fnord", key="updated test part 2", data="fnord") + after_storage = int(time.time()) + + timestamp = cache.updated(bank="fnord", key="updated test part 2") + + assert before_storage <= timestamp <= after_storage + + with subtests.test("If the module raises SaltCacheError then it should make it out of updated"): + with patch.dict( + cache.modules._dict, + {f"{cache.driver}.updated": MagicMock(side_effect=SaltCacheError)}, + ), pytest.raises(SaltCacheError): + cache.updated(bank="kaboom", key="oops") + + with subtests.test("cache.cache right after a value is cached should not update the cache"): + expected_value = "some cool value yo" + cache.store(bank=bank, key=good_key, data=expected_value) + result = cache.cache( + bank=bank, + key=good_key, + fun=lambda **kwargs: "bad bad value no good", + value="some other value?", + loop_fun=lambda x: "super very no good bad", + ) + fetch_result = cache.fetch(bank=bank, key=good_key) + + assert result == fetch_result == expected_value + + with subtests.test( + "cache.cache should update the value with the result of fun when value was updated longer than expiration", + ), patch( + "salt.cache.Cache.updated", + return_value=42, # Dec 31, 1969... time to update the cache! + autospec=True, + ): + expected_value = "this is the return value woo woo woo" + cache.store(bank=bank, key=good_key, data="not this value") + cache_result = cache.cache( + bank=bank, key=good_key, fun=lambda *args, **kwargs: expected_value + ) + fetch_result = cache.fetch(bank=bank, key=good_key) + + assert cache_result == fetch_result == expected_value + + with subtests.test( + "cache.cache should update the value with all of the outputs from loop_fun if loop_fun was provided", + ), patch( + "salt.cache.Cache.updated", + return_value=42, + autospec=True, + ): + expected_value = "SOME HUGE STRING OKAY?" + + cache.store(bank=bank, key=good_key, data="nope, not me") + cache_result = cache.cache( + bank=bank, + key=good_key, + fun=lambda **kwargs: "some huge string okay?", + loop_fun=str.upper, + ) + fetch_result = cache.fetch(bank=bank, key=good_key) + + assert cache_result == fetch_result + assert "".join(fetch_result) == expected_value + + with subtests.test( + "cache.cache should update the value if the stored value is empty but present and expiry is way in the future" + ), patch( + "salt.cache.Cache.updated", + return_value=time.time() * 2, + autospec=True, + ): + # Unclear if this was intended behavior: currently any falsey data will + # be updated by cache.cache. If this is incorrect, this test should + # be updated or removed. + expected_data = "some random string whatever" + for empty in ("", (), [], {}, 0, 0.0, False, None): + with subtests.test(empty=empty): + cache.store(bank=bank, key=good_key, data=empty) # empty chairs and empty data + cache_result = cache.cache( + bank=bank, key=good_key, fun=lambda **kwargs: expected_data + ) + fetch_result = cache.fetch(bank=bank, key=good_key) + + assert cache_result == fetch_result == expected_data + + with subtests.test("cache.cache should store a value if it does not exist"): + expected_result = "some result plz" + cache.flush(bank=bank, key=None) + assert cache.fetch(bank=bank, key=good_key) == {} + cache_result = cache.cache(bank=bank, key=good_key, fun=lambda **kwargs: expected_result) + fetch_result = cache.fetch(bank=bank, key=good_key) + + assert cache_result == fetch_result + assert fetch_result == expected_result + assert cache_result == fetch_result == expected_result diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/functional/cache/test_consul.py similarity index 96% rename from tests/pytests/functional/cache/test_consul.py rename to tests/functional/cache/test_consul.py index 0a42913..0fb553e 100644 --- a/tests/pytests/functional/cache/test_consul.py +++ b/tests/functional/cache/test_consul.py @@ -3,11 +3,11 @@ import time import pytest -from saltfactories.utils import random_string - import salt.cache import salt.loader -from tests.pytests.functional.cache.helpers import run_common_cache_tests +from saltfactories.utils import random_string + +from tests.functional.cache.helpers import run_common_cache_tests docker = pytest.importorskip("docker") diff --git a/tests/pytests/functional/conftest.py b/tests/functional/conftest.py similarity index 91% rename from tests/pytests/functional/conftest.py rename to tests/functional/conftest.py index 2fb2246..2c7c84a 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -33,8 +33,8 @@ def state_tree_prod(tmp_path_factory): @pytest.fixture(scope="module") def minion_config_defaults(): """ - Functional test modules can provide this fixture to tweak the default configuration dictionary - passed to the minion factory + Functional test modules can provide this fixture to tweak the default + configuration dictionary passed to the minion factory """ return {} @@ -81,8 +81,8 @@ def minion_opts( @pytest.fixture(scope="module") def master_config_defaults(): """ - Functional test modules can provide this fixture to tweak the default configuration dictionary - passed to the master factory + Functional test modules can provide this fixture to tweak the default + configuration dictionary passed to the master factory """ return {} @@ -138,3 +138,13 @@ def reset_loaders_state(loaders): finally: # Reset the loaders state loaders.reset_state() + + +@pytest.fixture(scope="module") +def modules(loaders): + return loaders.modules + + +@pytest.fixture(scope="module") +def states(loaders): + return loaders.states diff --git a/tests/pytests/unit/__init__.py b/tests/functional/modules/__init__.py similarity index 100% rename from tests/pytests/unit/__init__.py rename to tests/functional/modules/__init__.py diff --git a/tests/functional/modules/test_consul.py b/tests/functional/modules/test_consul.py new file mode 100644 index 0000000..c9f8f03 --- /dev/null +++ b/tests/functional/modules/test_consul.py @@ -0,0 +1,16 @@ +import pytest + +pytestmark = [ + pytest.mark.requires_salt_modules("consul.example_function"), +] + + +@pytest.fixture +def consul(modules): + return modules.consul + + +def test_replace_this_this_with_something_meaningful(consul): + echo_str = "Echoed!" + res = consul.example_function(echo_str) + assert res == echo_str diff --git a/tests/pytests/unit/modules/__init__.py b/tests/functional/states/__init__.py similarity index 100% rename from tests/pytests/unit/modules/__init__.py rename to tests/functional/states/__init__.py diff --git a/tests/functional/states/test_consul.py b/tests/functional/states/test_consul.py new file mode 100644 index 0000000..e832233 --- /dev/null +++ b/tests/functional/states/test_consul.py @@ -0,0 +1,18 @@ +import pytest + +pytestmark = [ + pytest.mark.requires_salt_states("consul.exampled"), +] + + +@pytest.fixture +def consul(states): + return states.consul + + +def test_replace_this_this_with_something_meaningful(consul): + echo_str = "Echoed!" + ret = consul.exampled(echo_str) + assert ret.result + assert not ret.changes + assert echo_str in ret.comment diff --git a/tests/pytests/unit/pillar/__init__.py b/tests/integration/modules/__init__.py similarity index 100% rename from tests/pytests/unit/pillar/__init__.py rename to tests/integration/modules/__init__.py diff --git a/tests/integration/modules/test_consul.py b/tests/integration/modules/test_consul.py new file mode 100644 index 0000000..783477a --- /dev/null +++ b/tests/integration/modules/test_consul.py @@ -0,0 +1,13 @@ +import pytest + +pytestmark = [ + pytest.mark.requires_salt_modules("consul.example_function"), +] + + +def test_replace_this_this_with_something_meaningful(salt_call_cli): + echo_str = "Echoed!" + ret = salt_call_cli.run("consul.example_function", echo_str) + assert ret.exitcode == 0 + assert ret.json + assert ret.json == echo_str diff --git a/tests/pytests/unit/states/__init__.py b/tests/integration/states/__init__.py similarity index 100% rename from tests/pytests/unit/states/__init__.py rename to tests/integration/states/__init__.py diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py deleted file mode 100644 index de99d98..0000000 --- a/tests/pytests/integration/conftest.py +++ /dev/null @@ -1,109 +0,0 @@ -""" - tests.pytests.integration.conftest - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - PyTest fixtures -""" - -import logging - -import pytest - -log = logging.getLogger(__name__) - - -@pytest.fixture(scope="package") -def salt_master(salt_master_factory): - """ - A running salt-master fixture - """ - with salt_master_factory.started(): - yield salt_master_factory - - -@pytest.fixture(scope="package") -def salt_minion(salt_master, salt_minion_factory): - """ - A running salt-minion fixture - """ - assert salt_master.is_running() - with salt_minion_factory.started(): - # Sync All - salt_call_cli = salt_minion_factory.salt_call_cli() - ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.returncode == 0, ret - yield salt_minion_factory - - -@pytest.fixture(scope="module") -def salt_sub_minion(salt_master, salt_sub_minion_factory): - """ - A second running salt-minion fixture - """ - assert salt_master.is_running() - with salt_sub_minion_factory.started(): - # Sync All - salt_call_cli = salt_sub_minion_factory.salt_call_cli() - ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.returncode == 0, ret - yield salt_sub_minion_factory - - -@pytest.fixture(scope="package") -def salt_cli(salt_master): - """ - The ``salt`` CLI as a fixture against the running master - """ - assert salt_master.is_running() - return salt_master.salt_cli(timeout=30) - - -@pytest.fixture(scope="package") -def salt_call_cli(salt_minion): - """ - The ``salt-call`` CLI as a fixture against the running minion - """ - assert salt_minion.is_running() - return salt_minion.salt_call_cli(timeout=30) - - -@pytest.fixture(scope="package") -def salt_cp_cli(salt_master): - """ - The ``salt-cp`` CLI as a fixture against the running master - """ - assert salt_master.is_running() - return salt_master.salt_cp_cli(timeout=30) - - -@pytest.fixture(scope="package") -def salt_key_cli(salt_master): - """ - The ``salt-key`` CLI as a fixture against the running master - """ - assert salt_master.is_running() - return salt_master.salt_key_cli(timeout=30) - - -@pytest.fixture(scope="package") -def salt_run_cli(salt_master): - """ - The ``salt-run`` CLI as a fixture against the running master - """ - assert salt_master.is_running() - return salt_master.salt_run_cli(timeout=30) - - -@pytest.fixture(scope="module") -def salt_ssh_cli(salt_master, salt_ssh_roster_file, sshd_config_dir): - """ - The ``salt-ssh`` CLI as a fixture against the running master - """ - assert salt_master.is_running() - return salt_master.salt_ssh_cli( - timeout=180, - roster_file=salt_ssh_roster_file, - target_host="localhost", - client_key=str(sshd_config_dir / "client_key"), - base_script_args=["--ignore-host-keys"], - ) diff --git a/tests/pytests/unit/conftest.py b/tests/unit/conftest.py similarity index 65% rename from tests/pytests/unit/conftest.py rename to tests/unit/conftest.py index e19db6c..3e3fa4a 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,38 +1,30 @@ -import asyncio -import os - import pytest - import salt.config -import salt.transport.tcp -from tests.support.mock import MagicMock, patch @pytest.fixture def minion_opts(tmp_path): """ - Default minion configuration with relative temporary paths to not require root permissions. + Default minion configuration with relative temporary paths to not + require root permissions. """ root_dir = tmp_path / "minion" opts = salt.config.DEFAULT_MINION_OPTS.copy() opts["__role"] = "minion" opts["root_dir"] = str(root_dir) - opts["master_uri"] = "tcp://{ip}:{port}".format( - ip="127.0.0.1", port=opts["master_port"] - ) for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): dirpath = root_dir / name dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" - opts["conf_file"] = os.path.join(opts["conf_dir"], "minion") return opts @pytest.fixture def master_opts(tmp_path): """ - Default master configuration with relative temporary paths to not require root permissions. + Default master configuration with relative temporary paths to not + require root permissions. """ root_dir = tmp_path / "master" opts = salt.config.master_config(None) @@ -43,14 +35,14 @@ def master_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/master.log" - opts["conf_file"] = os.path.join(opts["conf_dir"], "master") return opts @pytest.fixture def syndic_opts(tmp_path): """ - Default master configuration with relative temporary paths to not require root permissions. + Default master configuration with relative temporary paths to not + require root permissions. """ root_dir = tmp_path / "syndic" opts = salt.config.DEFAULT_MINION_OPTS.copy() @@ -62,16 +54,4 @@ def syndic_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/syndic.log" - opts["conf_file"] = os.path.join(opts["conf_dir"], "syndic") return opts - - -@pytest.fixture -def mocked_tcp_pub_client(): - transport = MagicMock(spec=salt.transport.tcp.TCPPubClient) - transport.connect = MagicMock() - future = asyncio.Future() - transport.connect.return_value = future - future.set_result(True) - with patch("salt.transport.tcp.TCPPubClient", transport): - yield diff --git a/requirements/changelog.txt b/tests/unit/modules/__init__.py similarity index 100% rename from requirements/changelog.txt rename to tests/unit/modules/__init__.py diff --git a/tests/pytests/unit/modules/test_consul.py b/tests/unit/modules/test_consul.py similarity index 87% rename from tests/pytests/unit/modules/test_consul.py rename to tests/unit/modules/test_consul.py index 52f1c8e..d61f3ef 100644 --- a/tests/pytests/unit/modules/test_consul.py +++ b/tests/unit/modules/test_consul.py @@ -1,18 +1,16 @@ """ Test case for the consul execution module """ - - import logging +from unittest.mock import MagicMock +from unittest.mock import patch import pytest - import salt.modules.consul as consul import salt.utils.http import salt.utils.json import salt.utils.platform from salt.exceptions import SaltInvocationError -from tests.support.mock import MagicMock, patch log = logging.getLogger(__name__) @@ -64,9 +62,7 @@ def test_get(): } ) with patch.object(consul, "_query", mock_query): - consul_return = consul.get( - consul_url="http://127.0.0.1", key="foo", token="test_token" - ) + consul_return = consul.get(consul_url="http://127.0.0.1", key="foo", token="test_token") _expected = { "data": [ { @@ -99,9 +95,7 @@ def test_get(): } ) with patch.object(consul, "_query", mock_query): - consul_return = consul.get( - consul_url="http://127.0.0.1", key="foo", token="test_token" - ) + consul_return = consul.get(consul_url="http://127.0.0.1", key="foo", token="test_token") _expected = { "data": [ { @@ -253,9 +247,7 @@ def test_agent_maintenance(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_maintenance(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -263,9 +255,7 @@ def test_agent_maintenance(): # no required argument with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = 'Required parameter "enable" is missing.' result = consul.agent_maintenance(consul_url=consul_url) expected = {"message": msg, "res": False} @@ -273,9 +263,7 @@ def test_agent_maintenance(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Agent maintenance mode {}ed." value = "enabl" result = consul.agent_maintenance(consul_url=consul_url, enable=value) @@ -284,9 +272,7 @@ def test_agent_maintenance(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to change maintenance mode for agent." value = "enabl" result = consul.agent_maintenance(consul_url=consul_url, enable=value) @@ -309,9 +295,7 @@ def test_agent_join(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_join(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -319,19 +303,13 @@ def test_agent_join(): # no required argument with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = 'Required parameter "address" is missing.' - pytest.raises( - SaltInvocationError, consul.agent_join, consul_url=consul_url - ) + pytest.raises(SaltInvocationError, consul.agent_join, consul_url=consul_url) with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Agent joined the cluster" result = consul.agent_join(consul_url=consul_url, address="test") expected = {"message": msg, "res": True} @@ -339,9 +317,7 @@ def test_agent_join(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to join the cluster." value = "enabl" result = consul.agent_join(consul_url=consul_url, address="test") @@ -364,9 +340,7 @@ def test_agent_leave(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_join(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -376,18 +350,12 @@ def test_agent_leave(): # no required argument with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): - pytest.raises( - SaltInvocationError, consul.agent_leave, consul_url=consul_url - ) + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): + pytest.raises(SaltInvocationError, consul.agent_leave, consul_url=consul_url) with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Node {} put in leave state." result = consul.agent_leave(consul_url=consul_url, node=node) expected = {"message": msg.format(node), "res": True} @@ -395,9 +363,7 @@ def test_agent_leave(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to change state for {}." result = consul.agent_leave(consul_url=consul_url, node=node) expected = {"message": msg.format(node), "res": False} @@ -419,9 +385,7 @@ def test_agent_check_register(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_check_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -431,9 +395,7 @@ def test_agent_check_register(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_check_register, @@ -460,9 +422,7 @@ def test_agent_check_register(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Check {} added to agent." result = consul.agent_check_register( consul_url=consul_url, @@ -477,9 +437,7 @@ def test_agent_check_register(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to add check to agent." result = consul.agent_check_register( consul_url=consul_url, @@ -508,9 +466,7 @@ def test_agent_check_deregister(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_check_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -520,9 +476,7 @@ def test_agent_check_deregister(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_check_deregister, @@ -531,25 +485,17 @@ def test_agent_check_deregister(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Check {} removed from agent." - result = consul.agent_check_deregister( - consul_url=consul_url, checkid=checkid - ) + result = consul.agent_check_deregister(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": True} assert expected == result with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to remove check from agent." - result = consul.agent_check_deregister( - consul_url=consul_url, checkid=checkid - ) + result = consul.agent_check_deregister(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": False} assert expected == result @@ -569,9 +515,7 @@ def test_agent_check_pass(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_check_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -581,9 +525,7 @@ def test_agent_check_pass(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_check_pass, @@ -592,9 +534,7 @@ def test_agent_check_pass(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Check {} marked as passing." result = consul.agent_check_pass(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": True} @@ -602,9 +542,7 @@ def test_agent_check_pass(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to update check {}." result = consul.agent_check_pass(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": False} @@ -626,9 +564,7 @@ def test_agent_check_warn(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_check_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -638,9 +574,7 @@ def test_agent_check_warn(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_check_warn, @@ -649,9 +583,7 @@ def test_agent_check_warn(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Check {} marked as warning." result = consul.agent_check_warn(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": True} @@ -659,9 +591,7 @@ def test_agent_check_warn(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to update check {}." result = consul.agent_check_warn(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": False} @@ -683,9 +613,7 @@ def test_agent_check_fail(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_check_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -695,9 +623,7 @@ def test_agent_check_fail(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_check_fail, @@ -706,9 +632,7 @@ def test_agent_check_fail(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Check {} marked as critical." result = consul.agent_check_fail(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": True} @@ -716,9 +640,7 @@ def test_agent_check_fail(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to update check {}." result = consul.agent_check_fail(consul_url=consul_url, checkid=checkid) expected = {"message": msg.format(checkid), "res": False} @@ -740,9 +662,7 @@ def test_agent_service_register(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_service_register(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -752,9 +672,7 @@ def test_agent_service_register(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_service_register, @@ -763,9 +681,7 @@ def test_agent_service_register(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Service {} registered on agent." result = consul.agent_service_register( consul_url=consul_url, @@ -780,9 +696,7 @@ def test_agent_service_register(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to register service {}." result = consul.agent_service_register( consul_url=consul_url, @@ -811,9 +725,7 @@ def test_agent_service_deregister(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_service_deregister(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -823,9 +735,7 @@ def test_agent_service_deregister(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_service_deregister, @@ -834,25 +744,17 @@ def test_agent_service_deregister(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Service {} removed from agent." - result = consul.agent_service_deregister( - consul_url=consul_url, serviceid=serviceid - ) + result = consul.agent_service_deregister(consul_url=consul_url, serviceid=serviceid) expected = {"message": msg.format(serviceid), "res": True} assert expected == result with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to remove service {}." - result = consul.agent_service_deregister( - consul_url=consul_url, serviceid=serviceid - ) + result = consul.agent_service_deregister(consul_url=consul_url, serviceid=serviceid) expected = {"message": msg.format(serviceid), "res": False} assert expected == result @@ -872,9 +774,7 @@ def test_agent_service_maintenance(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.agent_service_maintenance(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -884,9 +784,7 @@ def test_agent_service_maintenance(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.agent_service_maintenance, @@ -903,9 +801,7 @@ def test_agent_service_maintenance(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Service {} set in maintenance mode." result = consul.agent_service_maintenance( consul_url=consul_url, serviceid=serviceid, enable=True @@ -915,9 +811,7 @@ def test_agent_service_maintenance(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to set service {} to maintenance mode." result = consul.agent_service_maintenance( consul_url=consul_url, serviceid=serviceid, enable=True @@ -941,9 +835,7 @@ def test_session_create(): # no consul url error with patch.dict(consul.__salt__, {"config.get": mock_nourl}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): result = consul.session_create(consul_url="") expected = {"message": "No Consul URL found.", "res": False} assert expected == result @@ -953,9 +845,7 @@ def test_session_create(): # no required arguments with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): pytest.raises( SaltInvocationError, consul.session_create, @@ -964,9 +854,7 @@ def test_session_create(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Created session {}." result = consul.session_create(consul_url=consul_url, name=name) expected = {"message": msg.format(name), "res": True} @@ -974,9 +862,7 @@ def test_session_create(): with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): with patch.dict(consul.__salt__, {"config.get": mock_url}): - with patch.object( - salt.modules.consul, "session_list", return_value=mock_result - ): + with patch.object(salt.modules.consul, "session_list", return_value=mock_result): msg = "Unable to create session {}." result = consul.session_create(consul_url=consul_url, name=name) expected = {"message": msg.format(name), "res": False} @@ -1041,9 +927,7 @@ def test_session_destroy(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): msg = "Destroyed Session {}." - result = consul.session_destroy( - consul_url=consul_url, session=session, name="test" - ) + result = consul.session_destroy(consul_url=consul_url, session=session, name="test") expected = {"message": msg.format(session), "res": True} assert expected == result @@ -1144,15 +1028,11 @@ def test_catalog_register(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): result = consul.catalog_register( - consul_url=consul_url, - token=token, - node=node, - address=address, - **nodemeta_kwargs + consul_url=consul_url, token=token, node=node, address=address, **nodemeta_kwargs ) expected = { "data": {"Address": address, "Node": node, "NodeMeta": nodemeta}, - "message": "Catalog registration for {} successful.".format(node), + "message": f"Catalog registration for {node} successful.", "res": True, } @@ -1198,7 +1078,7 @@ def test_catalog_deregister(): checkid=checkid, ) expected = { - "message": "Catalog item {} removed.".format(node), + "message": f"Catalog item {node} removed.", "res": True, } @@ -1314,9 +1194,7 @@ def test_catalog_service(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.catalog_service( - consul_url=consul_url, token=token, service=service - ) + result = consul.catalog_service(consul_url=consul_url, token=token, service=service) expected = {"data": "test", "res": True} assert expected == result @@ -1424,9 +1302,7 @@ def test_health_checks(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.health_checks( - consul_url=consul_url, token=token, service=service - ) + result = consul.health_checks(consul_url=consul_url, token=token, service=service) expected = {"data": "test", "res": True} assert expected == result @@ -1462,9 +1338,7 @@ def test_health_service(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.health_service( - consul_url=consul_url, token=token, service=service - ) + result = consul.health_service(consul_url=consul_url, token=token, service=service) expected = {"data": "test", "res": True} assert expected == result @@ -1501,9 +1375,7 @@ def test_health_state(): # state not in allowed states with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.health_state( - consul_url=consul_url, token=token, state=state - ) + result = consul.health_state(consul_url=consul_url, token=token, state=state) expected = { "message": "State must be any, unknown, passing, warning, or critical.", "res": False, @@ -1513,9 +1385,7 @@ def test_health_state(): state = "warning" with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.health_state( - consul_url=consul_url, token=token, state=state - ) + result = consul.health_state(consul_url=consul_url, token=token, state=state) expected = {"data": "test", "res": True} assert expected == result @@ -1604,7 +1474,7 @@ def test_acl_create(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): result = consul.acl_create(consul_url=consul_url, token=token, name=name) - expected = {"message": "ACL {} created.".format(name), "res": True} + expected = {"message": f"ACL {name} created.", "res": True} assert expected == result @@ -1650,10 +1520,8 @@ def test_acl_update(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.acl_update( - consul_url=consul_url, token=token, name=name, id=aclid - ) - expected = {"message": "ACL {} created.".format(name), "res": True} + result = consul.acl_update(consul_url=consul_url, token=token, name=name, id=aclid) + expected = {"message": f"ACL {name} created.", "res": True} assert expected == result @@ -1690,10 +1558,8 @@ def test_acl_delete(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.acl_delete( - consul_url=consul_url, token=token, name=name, id=aclid - ) - expected = {"message": "ACL {} deleted.".format(aclid), "res": True} + result = consul.acl_delete(consul_url=consul_url, token=token, name=name, id=aclid) + expected = {"message": f"ACL {aclid} deleted.", "res": True} assert expected == result @@ -1730,9 +1596,7 @@ def test_acl_info(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.acl_info( - consul_url=consul_url, token=token, name=name, id=aclid - ) + result = consul.acl_info(consul_url=consul_url, token=token, name=name, id=aclid) expected = {"data": "test", "res": True} assert expected == result @@ -1770,12 +1634,10 @@ def test_acl_clone(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.acl_clone( - consul_url=consul_url, token=token, name=name, id=aclid - ) + result = consul.acl_clone(consul_url=consul_url, token=token, name=name, id=aclid) expected = { "ID": aclid, - "message": "ACL {} cloned.".format(name), + "message": f"ACL {name} cloned.", "res": True, } assert expected == result @@ -1804,9 +1666,7 @@ def test_acl_list(): with patch.object(salt.utils.http, "query", return_value=mock_http_result): with patch.dict(consul.__salt__, {"config.get": mock_url}): - result = consul.acl_list( - consul_url=consul_url, token=token, name=name, id=aclid - ) + result = consul.acl_list(consul_url=consul_url, token=token, name=name, id=aclid) expected = {"data": "id1", "res": True} assert expected == result @@ -1845,7 +1705,7 @@ def test_event_fire(): result = consul.event_fire(consul_url=consul_url, token=token, name=name) expected = { "data": "test", - "message": "Event {} fired.".format(name), + "message": f"Event {name} fired.", "res": True, } assert expected == result diff --git a/requirements/dev.txt b/tests/unit/pillar/__init__.py similarity index 100% rename from requirements/dev.txt rename to tests/unit/pillar/__init__.py diff --git a/tests/pytests/unit/pillar/test_consul_pillar.py b/tests/unit/pillar/test_consul_pillar.py similarity index 78% rename from tests/pytests/unit/pillar/test_consul_pillar.py rename to tests/unit/pillar/test_consul_pillar.py index 430f7aa..8682dd6 100644 --- a/tests/pytests/unit/pillar/test_consul_pillar.py +++ b/tests/unit/pillar/test_consul_pillar.py @@ -1,12 +1,11 @@ -import pytest +from unittest.mock import MagicMock +from unittest.mock import patch +import pytest import salt.pillar.consul_pillar as consul_pillar -from tests.support.mock import MagicMock, patch pytestmark = [ - pytest.mark.skipif( - not consul_pillar.consul, reason="python-consul module not installed" - ) + pytest.mark.skipif(not consul_pillar.consul, reason="python-consul module not installed") ] @@ -45,9 +44,7 @@ def base_pillar_data(): def configure_loader_modules(): return { consul_pillar: { - "__opts__": { - "consul_config": {"consul.port": 8500, "consul.host": "172.17.0.15"} - }, + "__opts__": {"consul_config": {"consul.port": 8500, "consul.host": "172.17.0.15"}}, "get_conn": MagicMock(return_value="consul_connection"), } } @@ -55,24 +52,22 @@ def configure_loader_modules(): def test_connection(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, "consul_fetch", MagicMock(return_value=("2232", base_pillar_data)), ): - consul_pillar.ext_pillar( - "testminion", {}, "consul_config root=test-shared/" - ) - consul_pillar.get_conn.assert_called_once_with( - consul_pillar.__opts__, "consul_config" - ) + consul_pillar.ext_pillar("testminion", {}, "consul_config root=test-shared/") + consul_pillar.get_conn.assert_called_once_with(consul_pillar.__opts__, "consul_config") def test_pillar_data(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, @@ -82,16 +77,15 @@ def test_pillar_data(base_pillar_data): pillar_data = consul_pillar.ext_pillar( "testminion", {}, "consul_config root=test-shared/" ) - consul_pillar.consul_fetch.assert_called_once_with( - "consul_connection", "test-shared" - ) + consul_pillar.consul_fetch.assert_called_once_with("consul_connection", "test-shared") assert sorted(pillar_data) == ["sites", "user"] assert "blankvalue" not in pillar_data["user"] def test_blank_root(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, @@ -105,7 +99,8 @@ def test_blank_root(base_pillar_data): def test_pillar_nest(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, @@ -123,7 +118,8 @@ def test_pillar_nest(base_pillar_data): def test_value_parsing(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, @@ -138,7 +134,8 @@ def test_value_parsing(base_pillar_data): def test_non_expansion(base_pillar_data): with patch.dict( - consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + consul_pillar.__salt__, + {"grains.get": MagicMock(return_value=({}))}, # pylint: disable=superfluous-parens ): with patch.object( consul_pillar, diff --git a/requirements/docs-auto.txt b/tests/unit/states/__init__.py similarity index 100% rename from requirements/docs-auto.txt rename to tests/unit/states/__init__.py diff --git a/tests/pytests/unit/states/test_consul.py b/tests/unit/states/test_consul.py similarity index 98% rename from tests/pytests/unit/states/test_consul.py rename to tests/unit/states/test_consul.py index 0236745..9631788 100644 --- a/tests/pytests/unit/states/test_consul.py +++ b/tests/unit/states/test_consul.py @@ -1,14 +1,12 @@ """ Test case for the consul state module """ - - import logging +from unittest.mock import MagicMock +from unittest.mock import patch import pytest - import salt.states.consul as consul -from tests.support.mock import MagicMock, patch log = logging.getLogger(__name__)