From 703b5f606da92c20042dba55a20251b986fb947e Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 8 May 2024 10:11:34 +0200 Subject: [PATCH 1/9] (CI) Add option to enable debug --- ci/install-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-package.sh b/ci/install-package.sh index a4320a1b2..29669a529 100755 --- a/ci/install-package.sh +++ b/ci/install-package.sh @@ -134,7 +134,7 @@ then echo -e "\e[35;1mSSH_KEY = ${SSH_KEY_FINGERPRINT}\e[0m" DOCKER_SSH_AUTH_SOCK="/tmp/ssh_auth_sock" - DOCKER_MOUNT_KNOWN_HOSTS_ARGS=("-e" "SSH_AUTH_SOCK=${DOCKER_SSH_AUTH_SOCK}" "--mount" "type=bind,source=$SHARED_DIR/.ssh,target=/tmp/.ssh") + DOCKER_MOUNT_KNOWN_HOSTS_ARGS=("-e" "SSH_AUTH_SOCK=${DOCKER_SSH_AUTH_SOCK}" "--mount" "type=bind,source=${SHARED_DIR}/.ssh,target=/tmp/.ssh") # Used in the print statement to reproduce CI build locally ADDITIONAL_ARGS_LOCAL_INSTALL+=("--shared=/tmp/shared/${PACKAGE}" "--ssh") From 248d48e153b401c5b8b5a67eda7d904c6bed9938 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 9 Jan 2024 09:25:51 +0100 Subject: [PATCH 2/9] (CI)(install) better SSH stuff and logging --- ci/install-package.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/ci/install-package.sh b/ci/install-package.sh index 29669a529..c9747d277 100755 --- a/ci/install-package.sh +++ b/ci/install-package.sh @@ -191,6 +191,7 @@ fi # Docker container can show a header on start-up. We don't want to capture it docker run --detach --tty --name tue-env "${IMAGE_NAME}:${BRANCH_TAG}" DOCKER_HOME=$(docker exec tue-env bash -c 'echo "${HOME}"' | tr -d '\r') +DOCKER_USER=$(docker exec tue-env bash -c 'echo "${USER}"' | tr -d '\r') docker stop tue-env &> /dev/null || true docker rm tue-env &> /dev/null || true @@ -212,13 +213,32 @@ docker exec -t tue-env bash -c 'sudo chown "${USER}":"${USER}" -R ~/.cache/pip' if [ "$USE_SSH" == "true" ] then - docker exec -t tue-env bash -c 'sudo chown "${USER}":"${USER}" -R /tmp/.ssh' + echo -e "\e[35;1msudo chown -R ${DOCKER_USER}:${DOCKER_USER} /tmp/.ssh\e[0m" + docker exec -t tue-env bash -c 'sudo chown -R "${USER}":"${USER}" /tmp/.ssh' - docker exec -t tue-env bash -c "[[ -f /tmp/.ssh/known_hosts ]] && mv ~/.ssh/known_hosts ~/.ssh/known_hosts_container" - docker exec -t tue-env bash -c 'sudo cp -r /tmp/.ssh/* ~/.ssh/ && sudo chown -R "${USER}":"${USER}" ~/.ssh && ls -aln ~/.ssh' + echo -e "\e[35;1mmv ~/.ssh/known_hosts ~/.ssh/known_hosts_container\e[0m" + docker exec -t tue-env bash -c "[[ ! -f /tmp/.ssh/known_hosts ]] || mv ~/.ssh/known_hosts ~/.ssh/known_hosts_container" - docker exec -t tue-env bash -c "[[ -f ~/.ssh/known_hosts && -f ~/.ssh/known_hosts_container ]] && ~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts" + echo -e "\e[35;1msudo cp -r /tmp/.ssh/* ~/.ssh/\e[0m" + docker exec -t tue-env bash -c 'sudo cp -r /tmp/.ssh/* ~/.ssh/' + + echo -e "\e[35;1msudo chown -R ${DOCKER_USER}:${DOCKER_USER} ~/.ssh\e[0m" + docker exec -t tue-env bash -c 'sudo chown -R "${USER}":"${USER}" ~/.ssh' + + echo -e "\e[35;1mls -alF ~/.ssh\e[0m" + docker exec -t tue-env bash -c "ls -alF ~/.ssh" + + echo -e "\e[35;1m~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts\e[0m" + docker exec -t tue-env bash -c "[[ ! -f ~/.ssh/known_hosts || ! -f ~/.ssh/known_hosts_container ]] || ~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts" + + echo -e "\e[35;1meval \"\$(ssh-agent -s)\" && grep -slR \"PRIVATE\" ~/.ssh/ | xargs ssh-add\e[0m" docker exec -e DOCKER_SSH_AUTH_SOCK="$DOCKER_SSH_AUTH_SOCK" -t tue-env bash -c 'eval "$(ssh-agent -s)" && ln -sf "$SSH_AUTH_SOCK" "$DOCKER_SSH_AUTH_SOCK" && grep -slR "PRIVATE" ~/.ssh/ | xargs ssh-add' + + echo -e "\e[35;1mecho -e 'Host *\n StrictHostKeyChecking yes' >> ~/.ssh/config\e[0m" + docker exec -t tue-env bash -c "echo -e 'Host *\n StrictHostKeyChecking yes' >> ~/.ssh/config" + + echo -e "\e[35;1mActive SSH keys:\e[0m" + docker exec -t tue-env bash -c "ssh-add -l 2>/dev/null" | awk '{print $2}' fi # Use docker environment variables in all exec commands instead of script variables From 242786de2c7b4eb3f17870d237ab169dc95c2ee6 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 5 Apr 2024 09:39:32 +0200 Subject: [PATCH 3/9] (CI) support marker and comments in known_hosts --- ci/ssh-merge-known_hosts.py | 56 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/ci/ssh-merge-known_hosts.py b/ci/ssh-merge-known_hosts.py index d0c64ed95..b5eb0b483 100755 --- a/ci/ssh-merge-known_hosts.py +++ b/ci/ssh-merge-known_hosts.py @@ -26,6 +26,20 @@ # Copied from https://blog.ganneff.de/2019/04/ssh-known-hosts-merge-by-key.html import argparse +from collections import defaultdict +from typing import Dict, List, Optional, Set, Tuple, Union + + +def key_dict_factory() -> Dict[str, Union[Set[str], Optional[str]]]: + return {"hosts": set(), "comments": set(), "marker": None} + + +def truncate(s: str, w: int) -> str: + s = s.strip() + if len(s) > w: + s = s[: w - 3].strip() + "..." + return s + parser = argparse.ArgumentParser( description="Merge ssh known host entries by key", @@ -56,26 +70,42 @@ output = stdout -hostkeys = {} +hostkeys: Dict[Tuple[str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory) for kfile in args.files: with open(kfile) as kf: for line in kf: if line[0] == "#": continue - line_splitted = line.rstrip().split(" ") - hosts = line_splitted.pop(0).split(",") - key_type = line_splitted.pop(0) - key = line_splitted[0] - if key not in hostkeys: - hostkeys[key] = {} - hostkeys[key]["hosts"] = [] - hostkeys[key]["key_type"] = key_type - # Store the host entries, uniquify them - hostkeys[key]["hosts"].extend(hosts) + line_splitted: List[str] = line.rstrip().split(" ") + marker: Optional[str] = None + if line_splitted[0].startswith("@"): + marker = line_splitted.pop(0) + hosts: List[str] = line_splitted.pop(0).split(",") + key_type: str = line_splitted.pop(0) + key = line_splitted.pop(0) + unique_key = (key_type, key) + if line_splitted: + if not line_splitted[0].startswith("#"): + raise ValueError(f"Unknown remainder in line: {line}") + comment = " ".join(line_splitted) + hostkeys[unique_key]["comments"].add(comment) + hostkeys[unique_key]["hosts"].update(hosts) + if marker is not None: + if hostkeys[unique_key]["marker"] is not None: + raise ValueError(f"Multiple markers for same key: {truncate(str(unique_key), 50)}") + hostkeys[unique_key]["marker"] = marker # And now output it all -for k, v in hostkeys.items(): - output.write("%s %s %s\n" % (",".join(v["hosts"]), v["key_type"], k)) +for (key_type, key), v in hostkeys.items(): + line_items = [] + if v["marker"] is not None: + line_items.append(v["marker"]) + line_items.append(",".join(sorted(v["hosts"]))) + line_items.append(key_type) + line_items.append(key) + if v["comments"]: + line_items.append(" ".join(v["comments"])) + output.write(f"{' '.join(line_items)}\n") # Write to output file if args.output: From 1fa12074f357fd9ef775514532761087c040d9a2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 5 Apr 2024 09:44:19 +0200 Subject: [PATCH 4/9] (CI) support leading comment line(s) in known_hosts --- ci/ssh-merge-known_hosts.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/ssh-merge-known_hosts.py b/ci/ssh-merge-known_hosts.py index b5eb0b483..2d7684080 100755 --- a/ci/ssh-merge-known_hosts.py +++ b/ci/ssh-merge-known_hosts.py @@ -31,7 +31,7 @@ def key_dict_factory() -> Dict[str, Union[Set[str], Optional[str]]]: - return {"hosts": set(), "comments": set(), "marker": None} + return {"hosts": set(), "comments": set(), "leading_comment_lines": set(), "marker": None} def truncate(s: str, w: int) -> str: @@ -73,8 +73,10 @@ def truncate(s: str, w: int) -> str: hostkeys: Dict[Tuple[str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory) for kfile in args.files: with open(kfile) as kf: + leading_comment_line = None for line in kf: if line[0] == "#": + leading_comment_line = line continue line_splitted: List[str] = line.rstrip().split(" ") marker: Optional[str] = None @@ -90,6 +92,9 @@ def truncate(s: str, w: int) -> str: comment = " ".join(line_splitted) hostkeys[unique_key]["comments"].add(comment) hostkeys[unique_key]["hosts"].update(hosts) + if leading_comment_line is not None: + hostkeys[unique_key]["leading_comment_lines"].add(leading_comment_line) + leading_comment_line = None if marker is not None: if hostkeys[unique_key]["marker"] is not None: raise ValueError(f"Multiple markers for same key: {truncate(str(unique_key), 50)}") @@ -97,6 +102,9 @@ def truncate(s: str, w: int) -> str: # And now output it all for (key_type, key), v in hostkeys.items(): + if v["leading_comment_lines"]: + for line in v["leading_comment_lines"]: + output.write(line) line_items = [] if v["marker"] is not None: line_items.append(v["marker"]) From ca1b7b3ee65773a6ce004da9edc4e71ff3125bfb Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 5 Apr 2024 11:56:37 +0200 Subject: [PATCH 5/9] (docker) restore user provided known_hosts --- dockerfiles/tue-env.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dockerfiles/tue-env.Dockerfile b/dockerfiles/tue-env.Dockerfile index 05281e138..be7e2364a 100644 --- a/dockerfiles/tue-env.Dockerfile +++ b/dockerfiles/tue-env.Dockerfile @@ -66,6 +66,7 @@ WORKDIR /home/"$USER" RUN mkdir -p -m 0700 ~/.ssh ADD ./known_hosts ./.ssh/known_hosts RUN sudo chown $USER_ID:$USER_ID ~/.ssh/known_hosts && sudo chmod 644 ~/.ssh/known_hosts +RUN cp ~/.ssh/known_hosts ~/.ssh/known_hosts.bak # Setup Git HTTPS token authentication RUN { [[ -n "$OAUTH2_TOKEN" ]] && git config --global credential.helper '!f() { printf "%s\n" "username=oauth2" "password=$OAUTH2_TOKEN"; };f'; } || exit 0 @@ -113,6 +114,9 @@ RUN --mount=type=ssh,uid=$USER_ID --mount=type=bind,source=installer/bootstrap.b # Remove apt cache sudo rm -rf /var/lib/apt/lists/* +# Restore known_hosts to one provided by the user +RUN mv -f ~/.ssh/known_hosts.bak ~/.ssh/known_hosts + RUN { [[ -n "$OAUTH2_TOKEN" ]] && git config --global --unset credential.helper; } || exit 0 # ---------------------------------------------------------------- From 73af006b21e0889684d36a3a189038e5c2de8894 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 5 Apr 2024 11:56:59 +0200 Subject: [PATCH 6/9] (docker) add comment --- dockerfiles/tue-env.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/tue-env.Dockerfile b/dockerfiles/tue-env.Dockerfile index be7e2364a..c59183b04 100644 --- a/dockerfiles/tue-env.Dockerfile +++ b/dockerfiles/tue-env.Dockerfile @@ -117,6 +117,7 @@ RUN --mount=type=ssh,uid=$USER_ID --mount=type=bind,source=installer/bootstrap.b # Restore known_hosts to one provided by the user RUN mv -f ~/.ssh/known_hosts.bak ~/.ssh/known_hosts +# Remove Git HTTPS token authentication RUN { [[ -n "$OAUTH2_TOKEN" ]] && git config --global --unset credential.helper; } || exit 0 # ---------------------------------------------------------------- From da18681c586b6f087e4cf394585f96a659e69faa Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 15 May 2024 09:23:35 +0200 Subject: [PATCH 7/9] (CI)(merge-known-hosts) allow multiple leading_comment_lines --- ci/ssh-merge-known_hosts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/ssh-merge-known_hosts.py b/ci/ssh-merge-known_hosts.py index 2d7684080..d921ba9ef 100755 --- a/ci/ssh-merge-known_hosts.py +++ b/ci/ssh-merge-known_hosts.py @@ -73,10 +73,10 @@ def truncate(s: str, w: int) -> str: hostkeys: Dict[Tuple[str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory) for kfile in args.files: with open(kfile) as kf: - leading_comment_line = None + leading_comment_lines = set() for line in kf: if line[0] == "#": - leading_comment_line = line + leading_comment_lines.add(line) continue line_splitted: List[str] = line.rstrip().split(" ") marker: Optional[str] = None @@ -92,9 +92,9 @@ def truncate(s: str, w: int) -> str: comment = " ".join(line_splitted) hostkeys[unique_key]["comments"].add(comment) hostkeys[unique_key]["hosts"].update(hosts) - if leading_comment_line is not None: - hostkeys[unique_key]["leading_comment_lines"].add(leading_comment_line) - leading_comment_line = None + if leading_comment_lines: + hostkeys[unique_key]["leading_comment_lines"].update(leading_comment_lines) + leading_comment_lines = set() if marker is not None: if hostkeys[unique_key]["marker"] is not None: raise ValueError(f"Multiple markers for same key: {truncate(str(unique_key), 50)}") From 0e5048802240212f68132d7c550e7fe912e7ff25 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 15 May 2024 09:31:32 +0200 Subject: [PATCH 8/9] (CI)(merge-known-hosts) each host, key-type, key on own line --- ci/ssh-merge-known_hosts.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/ci/ssh-merge-known_hosts.py b/ci/ssh-merge-known_hosts.py index d921ba9ef..6dee4caa8 100755 --- a/ci/ssh-merge-known_hosts.py +++ b/ci/ssh-merge-known_hosts.py @@ -31,7 +31,7 @@ def key_dict_factory() -> Dict[str, Union[Set[str], Optional[str]]]: - return {"hosts": set(), "comments": set(), "leading_comment_lines": set(), "marker": None} + return {"comments": set(), "leading_comment_lines": set(), "marker": None} def truncate(s: str, w: int) -> str: @@ -70,7 +70,7 @@ def truncate(s: str, w: int) -> str: output = stdout -hostkeys: Dict[Tuple[str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory) +hostkeys: Dict[Tuple[str, str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory) for kfile in args.files: with open(kfile) as kf: leading_comment_lines = set() @@ -85,30 +85,36 @@ def truncate(s: str, w: int) -> str: hosts: List[str] = line_splitted.pop(0).split(",") key_type: str = line_splitted.pop(0) key = line_splitted.pop(0) - unique_key = (key_type, key) + comment: Optional[str] = None if line_splitted: if not line_splitted[0].startswith("#"): raise ValueError(f"Unknown remainder in line: {line}") comment = " ".join(line_splitted) - hostkeys[unique_key]["comments"].add(comment) - hostkeys[unique_key]["hosts"].update(hosts) - if leading_comment_lines: - hostkeys[unique_key]["leading_comment_lines"].update(leading_comment_lines) - leading_comment_lines = set() - if marker is not None: - if hostkeys[unique_key]["marker"] is not None: - raise ValueError(f"Multiple markers for same key: {truncate(str(unique_key), 50)}") - hostkeys[unique_key]["marker"] = marker + for host in hosts: + unique_key = (host, key_type, key) + entry = hostkeys[unique_key] + if comment is not None: + entry["comments"].add(comment) + if leading_comment_lines: + entry["leading_comment_lines"].update(leading_comment_lines) + if marker is not None: + if hostkeys[unique_key]["marker"] is not None: + raise ValueError( + f"Multiple markers for same key: ({truncate(unique_key[0], 25)}, {unique_key[1]}, {truncate(unique_key[2], 25)})" + ) + entry["marker"] = marker + + leading_comment_lines = set() # And now output it all -for (key_type, key), v in hostkeys.items(): +for (host, key_type, key), v in hostkeys.items(): if v["leading_comment_lines"]: for line in v["leading_comment_lines"]: output.write(line) line_items = [] if v["marker"] is not None: line_items.append(v["marker"]) - line_items.append(",".join(sorted(v["hosts"]))) + line_items.append(host) line_items.append(key_type) line_items.append(key) if v["comments"]: From 017e9adb232a351a29552c87d0b238bdbed76bc0 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 10 Jun 2024 13:30:54 +0200 Subject: [PATCH 9/9] Bump VERSION to 1.25.9 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e6a6e7cd3..0e0c284d8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.25.8 +1.25.9