diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..716c153 --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +layout_saltext() { + VIRTUAL_ENV="$(python3 tools/initialize.py --print-venv)" + PATH_add "$VIRTUAL_ENV/bin" + export VIRTUAL_ENV +} + +layout_saltext diff --git a/tools/helpers/__init__.py b/tools/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/helpers/cmd.py b/tools/helpers/cmd.py new file mode 100644 index 0000000..9e8eb80 --- /dev/null +++ b/tools/helpers/cmd.py @@ -0,0 +1,286 @@ +""" +Polyfill for very basic ``plumbum`` functionality, no external libs required. +Makes scripts that call a lot of CLI commands much more pleasant to write. +""" + +import os +import platform +import shlex +import shutil +import subprocess +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path + + +class CommandNotFound(RuntimeError): + """ + Raised when a command cannot be found in $PATH + """ + + +@dataclass(frozen=True) +class ProcessResult: + """ + The full process result, returned by ``.run`` methods. + The ``__call__`` ones just return stdout. + """ + + retcode: int + stdout: str | bytes + stderr: str | bytes + argv: tuple + + def check(self, retcode=None): + """ + Check if the retcode is expected. retcode can be a list. + """ + if retcode is None: + expected = [0] + elif not isinstance(retcode, (list, tuple)): + expected = [retcode] + if self.retcode not in expected: + raise ProcessExecutionError(self.argv, self.retcode, self.stdout, self.stderr) + + def __str__(self): + msg = [ + "Process execution result:", + f"Command: {shlex.join(self.argv)}", + f"Retcode: {self.retcode}", + "Stdout: |", + ] + msg += [" " * 10 + "| " + line for line in str(self.stdout).splitlines()] + msg.append("Stderr: |") + msg += [" " * 10 + "| " + line for line in str(self.stderr).splitlines()] + return "\n".join(msg) + + +class ProcessExecutionError(OSError): + """ + Raised by ProcessResult.check when an unexpected retcode was returned. + """ + + def __init__(self, argv, retcode, stdout, stderr): + self.argv = argv + self.retcode = retcode + if isinstance(stdout, bytes): + stdout = ascii(stdout) + if isinstance(stderr, bytes): + stderr = ascii(stderr) + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + msg = [ + "Process finished with unexpected exit code", + f"Retcode: {self.retcode}", + f"Command: {shlex.join(self.argv)}", + "Stdout: |", + ] + msg += [" " * 10 + "| " + line for line in str(self.stdout).splitlines()] + msg.append("Stderr: |") + msg += [" " * 10 + "| " + line for line in str(self.stderr).splitlines()] + return "\n".join(msg) + + +class Local: + """ + Glue for command environment defaults. + Should be treated as a singleton. + + Example: + + local = Local() + + some_cmd = local["some_cmd"] + with local.cwd(some_path), local.env(FOO="bar"): + some_cmd("baz") + + # A changed $PATH requires to rediscover commands. + with local.prepend_path(important_path): + local["other_cmd"]() + with local.venv(venv_path): + local["python"]("-m", "pip", "install", "salt") + + """ + + def __init__(self): + # Explicitly cast values to strings to avoid problems on Windows + self._env = {k: str(v) for k, v in os.environ.items()} + + def __getitem__(self, exe): + """ + Return a LocalCommand in this context. + """ + return LocalCommand(exe, _local=self) + + @property + def path(self): + """ + List of paths in the context's $PATH. + """ + return self._env.get("PATH", "").split(os.pathsep) + + @contextmanager + def cwd(self, path): + """ + Set the default current working directory for commands inside this context. + """ + prev = Path(os.getcwd()) + new = prev / path + os.cwd(new) + try: + yield + finally: + os.cwd(prev) + + @contextmanager + def env(self, **kwargs): + """ + Override default env vars (sourced from the current process' environment) + for commands inside this context. + """ + prev = self._env.copy() + self._env.update((k, str(v)) for k, v in kwargs.items()) + try: + yield + finally: + self._env = prev + + @contextmanager + def path_prepend(self, *args): + """ + Prepend paths to $PATH for commands inside this context. + + Note: If you have saved a reference to an already requested command, + its $PATH will be updated, but it might not be the command + that would have been returned by a new request. + """ + new_path = [str(arg) for arg in args] + self.path + with self.env(PATH=os.pathsep.join(new_path)): + yield + + @contextmanager + def venv(self, venv_dir): + """ + Enter a Python virtual environment. Effectively prepends its bin dir + to $PATH and sets ``VIRTUAL_ENV``. + """ + venv_dir = Path(venv_dir) + if not venv_dir.is_dir() or not (venv_dir / "pyvenv.cfg").exists(): + raise ValueError(f"Not a virtual environment: {venv_dir}") + venv_bin_dir = venv_dir / "bin" + if platform.system() == "Windows": + venv_bin_dir = venv_dir / "Scripts" + with self.path_prepend(venv_bin_dir), self.env(VIRTUAL_ENV=str(venv_dir)): + yield + + +@dataclass(frozen=True) +class Executable: + """ + Utility class used to avoid repeated command lookups. + """ + + _exe: str + + def __str__(self): + return self._exe + + def __repr__(self): + return f"Executable <{self._exe}>" + + +@dataclass(frozen=True) +class Command: + """ + A command object, can be instantiated directly. Does not follow ``Local``. + """ + + exe: Executable | str + args: tuple[str, ...] = () + + def __post_init__(self): + if not isinstance(self.exe, Executable): + if not (full_exe := self._which(self.exe)): + raise CommandNotFound(self.exe) + object.__setattr__(self, "exe", Executable(full_exe)) + + def _which(self, exe): + return shutil.which(exe) + + def _get_env(self, overrides=None): + base = {k: str(v) for k, v in os.environ.items()} + base.update(overrides or {}) + return base + + def __getitem__(self, arg_or_args): + """ + Returns a subcommand with bound parameters. + + Example: + + git = Command("git")["-c", "commit.gpgsign=0"] + # ... + git("add", ".") + git("commit", "-m", "testcommit") + + """ + if not isinstance(arg_or_args, tuple): + arg_or_args = (arg_or_args,) + return type(self)(self.exe, tuple(*self.args, *arg_or_args), _local=self._local) + + def __call__(self, *args, **kwargs): + """ + Run this command and return stdout. + """ + return self.run(*args, **kwargs).stdout + + def __str__(self): + return shlex.join([self.exe] + list(self.args)) + + def __repr__(self): + return f"Command<{self.exe}, {self.args!r}>" + + def run(self, *args, check=True, env=None, **kwargs): + """ + Run this command and return the full output. + """ + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + kwargs.setdefault("text", True) + argv = [str(self.exe), *self.args, *args] + proc = subprocess.run(argv, check=False, env=self._get_env(env), **kwargs) + ret = ProcessResult( + retcode=proc.returncode, + stdout=proc.stdout, + stderr=proc.stderr, + argv=argv, + ) + if check: + ret.check() + return ret + + +@dataclass(frozen=True) +class LocalCommand(Command): + """ + Command returned by Local()["some_command"]. Follows local contexts. + """ + + _local: Local = field(kw_only=True, repr=False) + + def _which(self, exe): + return shutil.which(exe, path=self._local._env.get("PATH", "")) + + def _get_env(self, overrides=None): + base = self._local._env.copy() + base.update(overrides or {}) + return base + + +# Should be imported from here. +local = Local() +# We must assume git is installed +git = local["git"] diff --git a/tools/helpers/copier.py b/tools/helpers/copier.py new file mode 100644 index 0000000..91cf291 --- /dev/null +++ b/tools/helpers/copier.py @@ -0,0 +1,68 @@ +import sys +from functools import wraps +from pathlib import Path + +from . import prompt + +try: + # In case we have it, use it. + # It's always installed in the Copier environment, so if you ensure you + # call this via ``copier_python``, this will work. + import yaml +except ImportError: + yaml = None + + +COPIER_ANSWERS = Path(".copier-answers.yml").resolve() + + +def _needs_answers(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if not COPIER_ANSWERS.exists(): + raise RuntimeError(f"Missing answers file at {COPIER_ANSWERS}") + return func(*args, **kwargs) + + return _wrapper + + +@_needs_answers +def load_answers(): + """ + Load the complete answers file. Depends on PyYAML. + """ + if not yaml: + raise RuntimeError("Missing pyyaml in environment") + with open(COPIER_ANSWERS) as f: + return yaml.safe_load(f) + + +@_needs_answers +def discover_project_name(): + """ + Specifically discover project name. No dependency. + """ + for line in COPIER_ANSWERS.read_text().splitlines(): + if line.startswith("project_name"): + return line.split(":", maxsplit=1)[1].strip() + raise RuntimeError("Failed discovering project name") + + +def finish_task(msg, success, err_exit=False, extra=None): + """ + Print final conclusion of task (migration) run in Copier. + + We usually want to exit with 0, even when something fails, + because a failing task/migration should not crash Copier. + """ + print("\n", file=sys.stderr) + if success: + prompt.pprint(f"\n ✓ {msg}", bold=True, bg=prompt.DARKGREEN, stream=sys.stderr) + elif success is None: + prompt.pprint( + f"\n ✓ {msg}", bold=True, fg=prompt.YELLOW, bg=prompt.DARKGREEN, stream=sys.stderr + ) + success = True + else: + prompt.warn(f" ✗ {msg}", extra) + raise SystemExit(int(not success and err_exit)) diff --git a/tools/helpers/git.py b/tools/helpers/git.py new file mode 100644 index 0000000..5336031 --- /dev/null +++ b/tools/helpers/git.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from .cmd import git + + +def ensure_git(): + """ + Ensure the repository has been initialized. + """ + if Path(".git").is_dir(): + return + git("init", "--initial-branch", "main") + + +def list_untracked(): + """ + List untracked files. + """ + for path in git("ls-files", "-z", "-o", "--exclude-standard").split("\x00"): + if path: + yield path + + +def list_conflicted(): + """ + List files with merge conflicts. + """ + for path in git("diff", "-z", "--name-only", "--diff-filter=U", "--relative").split("\x00"): + if path: + yield path diff --git a/tools/helpers/pre_commit.py b/tools/helpers/pre_commit.py new file mode 100644 index 0000000..29c733e --- /dev/null +++ b/tools/helpers/pre_commit.py @@ -0,0 +1,100 @@ +import re + +from . import prompt +from .cmd import ProcessExecutionError +from .cmd import git +from .cmd import local +from .git import list_untracked + +PRE_COMMIT_TEST_REGEX = re.compile( + r"^(?P[^\n]+?)\.{4,}.*(?PFailed|Passed|Skipped)$" +) +NON_IDEMPOTENT_HOOKS = ( + "trim trailing whitespace", + "mixed line ending", + "fix end of files", + "Remove Python Import Header Comments", + "Check rST doc files exist for modules/states", + "Salt extensions docstrings auto-fixes", + "Rewrite the test suite", + "Rewrite Code to be Py3.", + "isort", + "black", + "blacken-docs", +) + + +def parse_pre_commit(data): + """ + Parse pre-commit output into a list of passing hooks and a mapping of + failing hooks to their output. + """ + passing = [] + failing = {} + cur = None + for line in data.splitlines(): + if match := PRE_COMMIT_TEST_REGEX.match(line): + cur = None + if match.group("resolution") != "Failed": + passing.append(match.group("test")) + continue + cur = match.group("test") + failing[cur] = [] + continue + try: + failing[cur].append(line) + except KeyError: + # in case the parsing logic fails, let's not crash everything + continue + return passing, {test: "\n".join(output).strip() for test, output in failing.items()} + + +def check_pre_commit_rerun(data): + """ + Check if we can expect failing hooks to turn green during a rerun. + """ + _, failing = parse_pre_commit(data) + for hook in failing: + if hook.startswith(NON_IDEMPOTENT_HOOKS): + return True + return False + + +def run_pre_commit(venv, retries=2): + """ + Run pre-commit in a loop until it passes, there is no chance of + autoformatting to make it pass or a maximum number of runs is reached. + + Usually, a maximum of two runs is necessary (if a hook reformats the + output of another later one again). + """ + + def _run_pre_commit_loop(retries_left): + # Ensure pre-commit runs on all paths. + # We don't want to git add . because this removes merge conflicts + git("add", "--intent-to-add", *map(str, list_untracked())) + with local.venv(venv): + try: + local["python"]("-m", "pre_commit", "run", "--all-files") + except ProcessExecutionError as err: + if retries_left > 0 and check_pre_commit_rerun(err.stdout): + return _run_pre_commit_loop(retries_left - 1) + raise + + prompt.status( + "Running pre-commit hooks against all files. This can take a minute, please be patient" + ) + + try: + _run_pre_commit_loop(retries) + return True + except ProcessExecutionError as err: + _, failing = parse_pre_commit(err.stdout) + if failing: + msg = f"Please fix all ({len(failing)}) failing hooks" + else: + msg = f"Output: {err.stderr or err.stdout}" + prompt.warn(f"Pre-commit is failing. {msg}") + for i, failing_hook in enumerate(failing): + prompt.warn(f"✗ Failing hook ({i + 1}): {failing_hook}", failing[failing_hook]) + return False diff --git a/tools/helpers/prompt.py b/tools/helpers/prompt.py new file mode 100644 index 0000000..d0f38c3 --- /dev/null +++ b/tools/helpers/prompt.py @@ -0,0 +1,51 @@ +import platform +import sys + +DARKGREEN = (0, 100, 0) +DARKRED = (139, 0, 0) +YELLOW = (255, 255, 0) + + +def ensure_utf8(): + """ + On Windows, ensure stdout/stderr output uses UTF-8 encoding. + """ + if platform.system() != "Windows": + return + for stream in (sys.stdout, sys.stderr): + if stream.encoding != "utf-8": + stream.reconfigure(encoding="utf-8") + + +def pprint(msg, bold=False, fg=None, bg=None, stream=None): + """ + Ugly helper for printing a bit more fancy output. + Stand-in for questionary/prompt_toolkit. + """ + out = "" + if bold: + out += "\033[1m" + if fg: + red, green, blue = fg + out += f"\033[38;2;{red};{green};{blue}m" + if bg: + red, green, blue = bg + out += f"\033[48;2;{red};{green};{blue}m" + out += msg + if bold or fg or bg: + out += "\033[0m" + print(out, file=stream or sys.stdout) + + +def status(msg, message=None): + out = f"\n → {msg}" + pprint(out, bold=True, fg=DARKGREEN, stream=sys.stderr) + if message: + pprint(message, stream=sys.stderr) + + +def warn(header, message=None): + out = f"\n{header}" + pprint(out, bold=True, bg=DARKRED, stream=sys.stderr) + if message: + pprint(message, stream=sys.stderr) diff --git a/tools/helpers/venv.py b/tools/helpers/venv.py new file mode 100644 index 0000000..bf04216 --- /dev/null +++ b/tools/helpers/venv.py @@ -0,0 +1,96 @@ +from pathlib import Path + +from . import prompt +from .cmd import CommandNotFound +from .cmd import local +from .copier import discover_project_name + +# Should follow the version used for relenv packages, see +# https://github.com/saltstack/salt/blob/master/cicd/shared-gh-workflows-context.yml +RECOMMENDED_PYVER = "3.10" +# For discovery of existing virtual environment, descending priority. +VENV_DIRS = ( + ".venv", + "venv", + ".env", + "env", +) + + +try: + uv = local["uv"] +except CommandNotFound: + uv = None + + +def is_venv(path): + if (venv_path := Path(path)).is_dir and (venv_path / "pyvenv.cfg").exists(): + return venv_path + return False + + +def discover_venv(project_root="."): + base = Path(project_root).resolve() + for name in VENV_DIRS: + if found := is_venv(base / name): + return found + raise RuntimeError(f"No venv found in {base}") + + +def create_venv(project_root=".", directory=None): + base = Path(project_root).resolve() + venv = (base / (directory or VENV_DIRS[0])).resolve() + if is_venv(venv): + raise RuntimeError(f"Venv at {venv} already exists") + prompt.status(f"Creating virtual environment at {venv}") + if uv is not None: + prompt.status("Found `uv`. Creating venv") + uv( + "venv", + "--python", + RECOMMENDED_PYVER, + f"--prompt=saltext-{discover_project_name()}", + ) + prompt.status("Installing pip into venv") + # Ensure there's still a `pip` (+ setuptools/wheel) inside the venv for compatibility + uv("venv", "--seed") + else: + prompt.status("Did not find `uv`. Falling back to `venv`") + try: + python = local[f"python{RECOMMENDED_PYVER}"] + except CommandNotFound: + python = local["python3"] + version = python("--version").split(" ")[1] + if not version.startswith(RECOMMENDED_PYVER): + raise RuntimeError( + f"No `python{RECOMMENDED_PYVER}` executable found in $PATH, exiting" + ) + python("-m", "venv", VENV_DIRS[0], f"--prompt=saltext-{discover_project_name()}") + return venv + + +def ensure_project_venv(project_root=".", reinstall=True): + exists = False + try: + venv = discover_venv(project_root) + prompt.status(f"Found existing virtual environment at {venv}") + exists = True + except RuntimeError: + venv = create_venv(project_root) + if not reinstall: + return venv + prompt.status(("Reinstalling" if exists else "Installing") + " project and dependencies") + with local.venv(venv): + if uv is not None: + uv("pip", "install", "-e", ".[dev,tests,docs]") + else: + try: + # We install uv into the virtualenv, so it might be available now. + # It speeds up this step a lot. + local["uv"]("pip", "install", "-e", ".[dev,tests,docs]") + except CommandNotFound: + local["python"]("-m", "pip", "install", "-e", ".[dev,tests,docs]") + if not exists or not (Path(project_root) / ".git" / "hooks" / "pre-commit").exists(): + prompt.status("Installing pre-commit hooks") + local["python"]("-m", "pre_commit", "install", "--install-hooks") + return venv diff --git a/tools/initialize.py b/tools/initialize.py new file mode 100644 index 0000000..1c07f68 --- /dev/null +++ b/tools/initialize.py @@ -0,0 +1,25 @@ +import sys + +from helpers import prompt +from helpers.copier import finish_task +from helpers.git import ensure_git +from helpers.venv import ensure_project_venv + +if __name__ == "__main__": + try: + prompt.ensure_utf8() + ensure_git() + venv = ensure_project_venv() + except Exception as err: # pylint: disable=broad-except + finish_task( + f"Failed initializing environment: {err}", + False, + True, + extra=( + "No worries, just follow the manual steps documented here: " + "https://salt-extensions.github.io/salt-extension-copier/topics/creation.html#first-steps" + ), + ) + if len(sys.argv) > 1 and sys.argv[1] == "--print-venv": + print(venv) + finish_task("Successfully initialized environment", True)