diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 4adbdf0..df49976 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -33,31 +33,107 @@ jobs: - name: "Update repository workflows and create PR" env: GH_TOKEN: ${{ github.token }} + # yamllint disable rule:line-length run: | - # Remove update-devops-tooling branch if it exists - git branch -d update-devops-tooling || true - git push origin --delete update-devops-tooling || true - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" + ### SHELL CODE START ### + + REPO_DIR=$(git rev-parse --show-toplevel) + # Ensure working from top-level of GIT repository + CURRENT_DIR=$(pwd) + if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then + echo "Changing directory to: $REPO_DIR" + if ! (cd "$REPO_DIR"); then + echo "Error: unable to change directory"; exit 1 + fi + fi + + # Define a function to allow selective opt-out of devops tooling + OPT_OUT=".devops-exclusions" + perform_operation() { + ELEMENT="$1" + if [ ! -f "$OPT_OUT" ]; then + # Opt-out file does not exist; all operations will be performed + return 1 + else + if grep -Fxq "$ELEMENT" "$OPT_OUT" + then + # Element is excluded from processing + return 0 + else + # Element should be processed + return 1 + fi + fi + } + + # Remove branch, if it exists: update-devops-tooling + if (git rev-parse --verify update-devops-tooling > /dev/null); then + git branch -d update-devops-tooling + elif (git ls-remote origin update-devops-tooling); then + echo "Removing remote branch: update-devops-tooling" + git push origin --delete update-devops-tooling + fi git checkout -b "update-devops-tooling" + + # Configure GIT + TEST=$(git config -l) + if [ -n "$TEST" ]; then + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + fi + FOLDERS=".github .github/workflows scripts" - FILES=".pre-commit-config.yaml .prettierignore .gitignore" for FOLDER in ${FOLDERS}; do + # Check to see if operation should be skipped + if (perform_operation "$FOLDER"); then + echo "Opted out of DevOps folder: $FOLDER" + continue + else # If necessary, create target folder if [ ! -d "$FOLDER" ]; then - mkdir "$FOLDER" + echo "Creating target folder: $FOLDER" + mkdir "$FOLDER" fi # Update folder contents + echo "Updating folder contents: $FOLDER" cp -a .devops/"$FOLDER"/. "$FOLDER" + fi done + # Copy specified files into repository root + FILES=".pre-commit-config.yaml .prettierignore .gitignore" for FILE in ${FILES}; do + if (perform_operation "$FILE"); then + echo "Opted out of DevOps file: $FILE" + else + echo "Copying file: $FILE" cp .devops/"$FILE" "$FILE" + fi done - git add . - git commit -m "Chore: Update DevOps tooling from central repository" - git push --set-upstream origin update-devops-tooling - gh pr create --title \ - "Chore: Pull DevOps tooling from upstream repository" \ - --body 'This process automated by a GitHub workflow: bootstrap.yaml' + + # If no changes required, do not throw an error + if [ -z "$(git status --porcelain)" ]; then + echo "No updates/changes to commit"; exit 0 + else + git add . + if ! (git commit -m "Chore: Update DevOps tooling from central repository [skip-ci]" \ + -m "This commit created by automation/scripting" --no-verify); then + echo "Commit failed; deleting local and remote branches, if necessary" + git checkout main + # Remove branch, if it exists: update-devops-tooling + if (git rev-parse --verify update-devops-tooling > /dev/null); then + git branch -d update-devops-tooling + elif (git ls-remote origin update-devops-tooling); then + echo "Removing remote branch: update-devops-tooling" + git push origin --delete update-devops-tooling + fi + else + git push --set-upstream origin update-devops-tooling + # ToDo: need to verify if we are running in a GHA + gh pr create --title \ + "Chore: Pull DevOps tooling from upstream repository" \ + --body 'Automated by a GitHub workflow: bootstrap.yaml' + fi + fi + ### SHELL CODE END ### diff --git a/.github/workflows/builds.yaml b/.github/workflows/builds.yaml index 94cd68c..8de5ef0 100644 --- a/.github/workflows/builds.yaml +++ b/.github/workflows/builds.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Test builds (matrix)" +name: "🧪 Test builds (Matrix)" # yamllint disable-line rule:truthy on: @@ -12,14 +12,15 @@ on: jobs: pre-release: - # Don't run if pull request is NOT merged - # if: github.event.pull_request.merged == true runs-on: "ubuntu-latest" continue-on-error: true + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Populate environment variables" id: setenv diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml index e6d12ba..33640f5 100644 --- a/.github/workflows/dependencies.yaml +++ b/.github/workflows/dependencies.yaml @@ -5,11 +5,11 @@ name: "⛔️ Update dependencies" on: workflow_dispatch: schedule: - - cron: "0 8 * * FRI" + - cron: "0 8 1 * *" jobs: update-dependencies: - name: "Update Python modules" + name: "Update dependencies" runs-on: ubuntu-latest permissions: # IMPORTANT: mandatory to raise the PR @@ -22,15 +22,27 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: actions/checkout@v4 - - name: Update dependencies - uses: ModeSevenIndustrialSolutions/update-deps-action@v1 + - name: "Set up Python" + uses: actions/setup-python@v5 + + - name: "Update Python dependencies" + uses: pdm-project/update-deps-action@v1 with: sign-off-commit: "true" token: ${{ secrets.GH_TOKEN }} - commit-message: "Chore: Update dependencies and pdm.lock" + commit-message: "Chore: Update dependencies and pdm.lock [skip ci]" pr-title: "Update Python module dependencies" update-strategy: eager # Whether to install PDM plugins before update install-plugins: "false" + + - name: "Export dependencies" + run: | + pdm export --without-hashes -o requirements.txt + + # Ideally, we should export requirements.txt then amend the earlier PR + # update-deps-action could be modified to export PR number as as output + # Or we add the option to export the requirements.txt in that action diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index d3badc0..3796a60 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,5 +1,5 @@ --- -name: "🗒️ Build documentation" +name: "🗒️ Build documentation (Matrix)" # yamllint disable-line rule:truthy on: diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml new file mode 100644 index 0000000..db12e80 --- /dev/null +++ b/.github/workflows/linting.yaml @@ -0,0 +1,56 @@ +--- +name: "⛔️ Standalone linting checks" + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + pull_request: + types: [opened, reopened, edited, synchronize] + branches: + - "*" + - "!update-devops-tooling" + +jobs: + linting: + + name: "Unsupported by pre-commit.ci" + runs-on: "ubuntu-latest" + # Don't run when pull request is merged + if: github.event.pull_request.merged == false + + steps: + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Install linting tools" + run: | + pip install --upgrade pip + pip install pre-commit mypy + + - name: "Run mypy using pre-commit" + run: pre-commit run mypy -a + + # yamllint disable rule:line-length + # yamllint disable rule:comments-indentation + # yamllint disable rule:comments + + # Provided below as an example, in case needed in future + # - name: "Install dependencies" + # run: | + # SOURCE=".pre-commit-config.yaml" + # echo "Install Python dependencies from: $SOURCE" + # echo "With: pip install $PKGS" + # PKGS=$(yq '.repos[] | select (.repo == "https://github.com/pre-commit/mirrors-mypy")' .pre-commit-config.yaml | \ + # grep additional_dependencies | \ + # awk -F: '{print $2}' | \ + # sed "s/\[//g" | \ + # sed "s/\]//g" | \ + # sed "s/,//g" | \ + # sed 's/"//g') + # pip install $PKGS diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d9c0ede..7b99e21 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -76,7 +76,7 @@ jobs: contents: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ @@ -111,7 +111,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ @@ -145,7 +145,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml index a24ce21..9d34824 100644 --- a/.github/workflows/security.yaml +++ b/.github/workflows/security.yaml @@ -4,7 +4,7 @@ # For more information see: # https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: "⛔️ Security auditing" +name: "⛔️ Security auditing (Matrix)" # yamllint disable-line rule:truthy on: @@ -19,10 +19,13 @@ jobs: build: name: "Audit Python dependencies" runs-on: ubuntu-latest + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Checkout repository" uses: actions/checkout@v4 @@ -48,10 +51,3 @@ jobs: - name: "Run: pip-audit" uses: pypa/gh-action-pip-audit@v1.0.8 - with: - ignore-vulns: | - PYSEC-2023-163 - -# Name | Version | ID | -# --- | --- | --- | --- | --- -# numexpr | 2.8.7 | PYSEC-2023-163 | diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index e2f9bdf..ad2a83b 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -86,7 +86,7 @@ jobs: contents: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: Development path: dist/ @@ -132,7 +132,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: Development path: dist/ diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 899529e..bb80cec 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Unit tests" +name: "🧪 Unit tests (Matrix)" # yamllint disable-line rule:truthy on: @@ -14,10 +14,13 @@ jobs: build: name: "Run unit tests" runs-on: ubuntu-latest + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Checkout repository" uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 363e5a9..c7f671c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,7 @@ repos: - id: mixed-line-ending args: ["--fix=lf"] - id: name-tests-test + args: ["--pytest-test-first"] # Do not allow direct push to main/master branches - id: no-commit-to-branch # - id: pretty-format-json @@ -63,7 +64,7 @@ repos: hooks: - id: prettier args: - ['--ignore-unknown', '--no-error-on-unmatched-pattern', '!chart/**'] + ['--ignore-unknown'] # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli @@ -79,12 +80,12 @@ repos: # args: ['--py37-plus'] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black-jupyter @@ -97,7 +98,7 @@ repos: rev: 2.1.1 hooks: - id: bashate - args: ["--ignore=E006"] + args: ["--ignore=E006,E011"] - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.6 @@ -118,7 +119,7 @@ repos: # ] - repo: https://github.com/PyCQA/isort - rev: 5.11.5 + rev: 5.13.2 hooks: - id: isort @@ -154,9 +155,9 @@ repos: # - id: codespell - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.7.1" + rev: "v1.8.0" hooks: - id: mypy verbose: true args: [--show-error-codes] - additional_dependencies: ["types-requests"] + additional_dependencies: ["pytest", "types-requests"] diff --git a/.prettierignore b/.prettierignore index a932ba9..6491e42 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ **/.pre-commit-config.yaml **/*.yaml **/*.yml +**/.git/** diff --git a/scripts/rename-tests.sh b/scripts/rename-tests.sh new file mode 100755 index 0000000..2f03b0b --- /dev/null +++ b/scripts/rename-tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +#set -x + +REPO_NAME=$(basename "$(git rev-parse --show-toplevel)") +echo "Repository name: $REPO_NAME" + +if [ $# -ne 1 ]; then + echo "Usage: $0 [test folder]"; exit 1 +elif [ ! -d "$1" ]; then + echo "Error: specified target was not a folder"; exit 1 +else + # Target specified was a folder + TARGET="$1" +fi + +for TEST in $(find "$TARGET" -type f -name '*_test.py' | xargs -0); do + echo "Processing: $TEST" + FILE_PATH=$(dirname "$TEST") + FILE_NAME=$(basename "$TEST") + STRIPPED="${FILE_NAME//_test.py/.py}" + echo " git mv \"${TEST}\" $FILE_PATH/test_\"${STRIPPED%%}\"" + git mv "${TEST}" "$FILE_PATH"/test_"${STRIPPED%%}" +done diff --git a/scripts/template-to-repo.sh b/scripts/template-to-repo.sh new file mode 100755 index 0000000..21c975c --- /dev/null +++ b/scripts/template-to-repo.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# set -x + +### Shared functions + +# Renames files/folders containing template name +rename_object() { + if [ $# -ne 1 ]; then + echo "Function requires an argumeent: rename_object [filesystem object]"; exit 1 + else + FS_OBJECT="$1" + fi + # Function take a filesystem object as a single argument + FS_OBJECT="$1" + OBJECT_PATH=$(dirname "$FS_OBJECT") + OBJECT_NAME=$(basename "$FS_OBJECT") + + # Check if filesystem object contains template name + if [[ ! "$OBJECT_NAME" == *"$TEMPLATE_NAME"* ]]; then + # Nothing to do; abort early + return + else + NEW_NAME="${OBJECT_NAME//$TEMPLATE_NAME/$REPO_NAME}" + fi + if [[ ! "$OBJECT_NAME" == *"$ALT_TEMPLATE_NAME"* ]]; then + # Nothing to do; abort early + return + else + NEW_NAME="${OBJECT_NAME//$ALT_TEMPLATE_NAME/$ALT_REPO_NAME}" + fi + + # Perform the renaming operation + if [ -d "$FS_OBJECT" ]; then + echo "Renaming folder: $FS_OBJECT" + elif [ -f "$FS_OBJECT" ]; then + echo "Renaming file: $FS_OBJECT" + elif [ -L "$FS_OBJECT" ]; then + echo "Renaming symlink: $FS_OBJECT" + fi + git mv "$OBJECT_PATH/$OBJECT_NAME" "$OBJECT_PATH/$NEW_NAME" +} + +# Checks file content for template name and replaces matching strings +file_content_substitution() { + if [ $# -ne 1 ]; then + echo "Function requires an argument: file_content_substitution [filename]"; exit 1 + else + FILENAME="$1" + fi + if (grep "$TEMPLATE_NAME" "$FILENAME" > /dev/null 2>&1); then + MATCHES=$(grep -c "$TEMPLATE_NAME" "$FILENAME") + if [ "$MATCHES" -eq 1 ]; then + echo "1 content substitution required: $FILENAME" + else + echo "$MATCHES content substitutions required: $FILENAME" + fi + sed -i "s/$TEMPLATE_NAME/$REPO_NAME/g" "$FILENAME" + fi + if (grep "$ALT_TEMPLATE_NAME" "$FILENAME" > /dev/null 2>&1); then + MATCHES=$(grep -c "$ALT_TEMPLATE_NAME" "$FILENAME") + if [ "$MATCHES" -eq 1 ]; then + echo "1 content substitution required: $FILENAME" + else + echo "$MATCHES content substitutions required: $FILENAME" + fi + sed -i "s/$ALT_TEMPLATE_NAME/$ALT_REPO_NAME/g" "$FILENAME" + fi +} + +### Main script entry point + +TEMPLATE_NAME=osc-python-template +ALT_TEMPLATE_NAME="${TEMPLATE_NAME//-/_}" + +if ! (git rev-parse --show-toplevel > /dev/null); then + echo "Error: this folder is not part of a GIT repository"; exit 1 +fi + +REPO_DIR=$(git rev-parse --show-toplevel) +REPO_NAME=$(basename "$REPO_DIR") +ALT_REPO_NAME="${REPO_NAME//-/_}" + +if [ "$TEMPLATE_NAME" == "$REPO_NAME" ]; then + echo "WARNING: template name matches repository name" +else + echo "Template name: $TEMPLATE_NAME" + echo "Alternate name: $ALT_TEMPLATE_NAME" + echo "Repository name: $REPO_NAME" + echo "Alternate name: $ALT_REPO_NAME" +fi + +# Change to top-level of GIT repository +CURRENT_DIR=$(pwd) +if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then + echo "Changing directory to: $REPO_DIR" + if ! (cd "$REPO_DIR"); then + echo "Could not change directory!"; exit 1 + fi +fi + +echo "Processing repository contents..." + +# Rename directories first, as they affect file paths afterwards +for FS_OBJECT in $(find -- * -type d | xargs -0); do + rename_object "$FS_OBJECT" + if [ -f "$FS_OBJECT" ]; then + file_content_substitution "$FS_OBJECT" + fi +done + +for FS_OBJECT in $(find -- * -type f | xargs -0); do + rename_object "$FS_OBJECT" + if [ -f "$FS_OBJECT" ]; then + file_content_substitution "$FS_OBJECT" + fi +done diff --git a/tests/osc_data_extractor_test.py b/tests/test_osc_data_extractor.py similarity index 100% rename from tests/osc_data_extractor_test.py rename to tests/test_osc_data_extractor.py