Skip to content

Commit

Permalink
chore(bors): merge pull request #564
Browse files Browse the repository at this point in the history
564: Upgrade basic sanity test r=niladrih a=niladrih

Changes:
- Adds a pytest for testing basic sanity in upgrade
- Add enhancement to the scripts/helm/install.sh script to support installing from a hosted registry
- Add a script to format python files

Co-authored-by: Niladri Halder <[email protected]>
  • Loading branch information
mayastor-bors and niladrih committed Nov 13, 2024
2 parents e91f57f + c42ed7a commit 33f3d8f
Show file tree
Hide file tree
Showing 14 changed files with 677 additions and 19 deletions.
16 changes: 12 additions & 4 deletions .github/workflows/k8s-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ jobs:
k8s-ci:
runs-on: ubuntu-latest
steps:
- name: Bind mount /dev/sda1 to /nix
run: |
sudo mkdir -p /nix
sudo mount --bind /mnt /nix
lsblk
- uses: actions/checkout@v4
with:
submodules: recursive
Expand All @@ -23,18 +28,21 @@ jobs:
id: build
run: |
TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh')
BIN=$(mktemp -p . -d -t test-bin-XXXXXX)
TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX))
nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG"
RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-bins --build-binary-out $BIN --no-static-linking --skip-publish --debug
RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "bin=$BIN" >> $GITHUB_OUTPUT
echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT
- name: BootStrap k8s cluster
run: |
nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
- name: Load images to Kind cluster
run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix"
- name: Run Pytests
run: nix-shell ./shell.nix --run './scripts/python/test.sh'
run: |
export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }}
export TEST_DIR=${{ steps.build.outputs.bin }}
nix-shell ./shell.nix --run "./scripts/python/test.sh"
- name: The job has failed
if: ${{ failure() }}
run: |
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ __pycache__
/kubectl-plugin

# Pytest assets
/test-bin-*
/test-dir-*
tests/bdd/venv
pytest.log
7 changes: 7 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[pytest]
log_cli = true
log_cli_level = INFO
log_cli_format = %(asctime)s [%(levelname)s] %(message)s
log_cli_date_format = %Y-%m-%d %H:%M:%S
log_file = pytest.log
log_file_level = DEBUG
81 changes: 68 additions & 13 deletions scripts/helm/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,54 @@

set -e

repo_add() {
local -r url=$1
local -r preferred_name=$2

local repo
if [ "$(helm repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then
repo=$(helm repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name")
else
helm repo add "$preferred_name" "$url" > /dev/null
repo=$preferred_name
fi

helm repo update > /dev/null || true

echo "$repo"
}


TIMEOUT="5m"
WAIT=
DRY_RUN=""
CHART=
SCRIPT_DIR="$(dirname "$0")"
CHART_DIR="$SCRIPT_DIR"/../../chart
CHART_SOURCE=$CHART_DIR
DEP_UPDATE=
RELEASE_NAME="mayastor"
K8S_NAMESPACE="mayastor"
FAIL_IF_INSTALLED=
HOSTED=
VERSION=
REGISTRY=
DEFAULT_REGISTRY="https://openebs.github.io/mayastor-extensions"

help() {
cat <<EOF
Usage: $(basename "$0") [COMMAND] [OPTIONS]
Options:
-h, --help Display this text.
--timeout <timeout> How long to wait for helm to complete install (Default: $TIMEOUT).
--wait Wait for helm to complete install.
--dry-run Install helm with --dry-run.
--dep-update Run helm dependency update.
--fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace.
-h, --help Display this text.
--timeout <timeout> How long to wait for helm to complete install (Default: $TIMEOUT).
--wait Wait for helm to complete install.
--dry-run Install helm with --dry-run.
--dep-update Run helm dependency update.
--fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace.
--hosted-chart Install a hosted chart instead of the local chart.
--version <version> Set the version/version-range for the chart. Works only when used with the '--hosted' option.
--registry <registry-url> Set the registry URL for the hosted chart. Works only when used with the '--hosted' option. (Default: $DEFAULT_REGISTRY)
Examples:
$(basename "$0")
Expand All @@ -40,10 +66,6 @@ die() {
exit "${_return}"
}

nvme_ana_check() {
cat /sys/module/nvme_core/parameters/multipath
}

while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
Expand All @@ -67,6 +89,27 @@ while [ "$#" -gt 0 ]; do
--fail-if-installed)
FAIL_IF_INSTALLED="y"
shift;;
--hosted-chart)
HOSTED=1
shift;;
--version*)
if [ "$1" = "--version" ]; then
test $# -lt 2 && die "Missing value for the optional argument '$1'."
VERSION="$2"
shift
else
VERSION="${1#*=}"
fi
shift;;
--registry)
if [ "$1" = "--registry" ]; then
test $# -lt 2 && die "Missing value for the optional argument '$1'."
REGISTRY="$2"
shift
else
REGISTRY="${1#*=}"
fi
shift;;
*)
die "Unknown argument $1!"
shift;;
Expand All @@ -82,7 +125,19 @@ if [ -n "$WAIT" ]; then
WAIT_ARG=" --wait --timeout $TIMEOUT"
fi

if [ "$(helm ls -n openebs -o json | jq --arg release_name "$RELEASE_NAME" 'any(.[]; .name == $release_name)')" = "true" ]; then
VERSION_ARG=
if [ -n "$HOSTED" ]; then
if [ -n "$VERSION" ]; then
VERSION_ARG="--version $VERSION"
fi
if [ -z "$REGISTRY" ]; then
REGISTRY=$DEFAULT_REGISTRY
fi
CHART_SOURCE="$(repo_add "$REGISTRY" "mayastor")/mayastor"
DEP_UPDATE_ARG=
fi

if [ "$(helm ls -n openebs -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
if [ -n "$FAIL_IF_INSTALLED" ]; then
die "ERROR: $already_exists_log" 1
Expand All @@ -91,11 +146,11 @@ if [ "$(helm ls -n openebs -o json | jq --arg release_name "$RELEASE_NAME" 'any(
else
echo "Installing Mayastor Chart"
set -x
helm install "$RELEASE_NAME" "$CHART_DIR" -n "$K8S_NAMESPACE" --create-namespace \
helm install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
--set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \
--set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \
--set="eventing.enabled=false" \
$DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG
$DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG
set +x
fi

Expand Down
122 changes: 122 additions & 0 deletions scripts/python/fmt.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/usr/bin/env bash

SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
ROOT_DIR="$SCRIPT_DIR/../.."

# Imports
source "$ROOT_DIR"/scripts/utils/log.sh

set -e

fmt() {
local -r paths_glob=$1

bash -c "
black --quiet $paths_glob &&
isort --profile=black --quiet $paths_glob &&
autoflake --quiet -r -i --remove-unused-variables --remove-all-unused-imports --expand-star-imports $paths_glob"
}

fmt_diff() {
local -r paths_glob=($(bash -c "echo $1"))

for path in "${paths_glob[@]}"; do
find "$path" -type f -name "*.py" -exec bash -c '
OUTPUT=$(diff -u --color=always <(cat {}) <(black -c "$(cat {})" |
isort --profile=black -d - |
autoflake --remove-unused-variables --remove-all-unused-imports --expand-star-imports -s -)
);
if [ -n "$OUTPUT" ]; then
echo -e "Diff for file {}\n===================================================\n$OUTPUT\n" | cat
fi
' \;
done
}

fmt_check() {
local -r paths_glob=$1

local error=
bash -c "
black --quiet $paths_glob --check &&
isort --profile=black --quiet $paths_glob --check 2> /dev/null &&
autoflake --quiet -r --remove-unused-variables --remove-all-unused-imports --expand-star-imports $paths_glob --check > /dev/null
" || error=$?

if [ -n "$error" ]; then
exit $error
fi
exit 0
}

CHECK=
DIFF=
PATHS_GLOB=
DEFAULT_PATHS_GLOB=$(realpath "$ROOT_DIR"/tests/bdd)/{common,features}

# Print usage options for this script.
print_help() {
cat <<EOF
Usage: $(basename "${0}") [OPTIONS]
Options:
-h, --help Display this text.
--check Don't write the files back, just return the status.
Return code 0 means nothing would change. Return code
1 means some files would be changed.
--diff Don't write the files back, just output a diff to
indicate what changes would've been made.
--paths-glob <glob> Input a paths glob of directories and/or files which
would be parsed recursively. (default: "$DEFAULT_PATHS_GLOB")
Examples:
$(basename "${0}") --check
EOF
}

# Parse args.
while test $# -gt 0; do
arg="$1"
case "$arg" in
--check)
CHECK=1
;;
--diff)
DIFF=1
;;
--paths-glob*)
if [ "$arg" = "--paths-glob" ]; then
test $# -lt 2 && log_fatal "Missing value for the optional argument '$arg'."
PATHS_GLOB="$2"
shift
else
PATHS_GLOB="${arg#*=}"
fi
;;
-h* | --help*)
print_help
exit 0
;;
*)
print_help
log_fatal "unexpected argument '$arg'" 1
;;
esac
shift
done

if [ -z "$PATHS_GLOB" ]; then
PATHS_GLOB=$DEFAULT_PATHS_GLOB
fi

if [ -n "$CHECK" ]; then
fmt_check "$PATHS_GLOB"
fi

if [ -n "$DIFF" ]; then
fmt_diff "$PATHS_GLOB"
fi

if [[ -z "$CHECK" ]] && [[ -z "$DIFF" ]]; then
fmt "$PATHS_GLOB"
fi
1 change: 1 addition & 0 deletions scripts/python/generate-test-tag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
ROOT_DIR="$SCRIPT_DIR/../.."

# Imports
source "$ROOT_DIR"/scripts/utils/log.sh
source "$ROOT_DIR"/scripts/utils/repo.sh

Expand Down
3 changes: 3 additions & 0 deletions shell.nix
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ in
mkShell {
name = "extensions-shell";
buildInputs = [
autoflake
black
cacert
cargo-expand
cargo-udeps
Expand All @@ -27,6 +29,7 @@ mkShell {
cowsay
git
helm-docs
isort
kubectl
kubernetes-helm-wrapped
llvmPackages.libclang
Expand Down
17 changes: 17 additions & 0 deletions tests/bdd/common/environment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import logging
import os

logger = logging.getLogger(__name__)


def get_env(variable: str):
try:
value = os.getenv(variable)
if len(value) == 0:
raise ValueError("Env {variable} is empty")
logger.info(f"Found env {variable}={value}")
return value

except Exception as e:
logger.error(f"Failed to get env {variable}: {e}")
return None
Loading

0 comments on commit 33f3d8f

Please sign in to comment.