(CI) Improve SSH stuff (#742)

Fixes the SSH issues in CI, see
https://github.com/tue-robotics/hero_bringup/actions/runs/9447499194/job/26019423899

VERSION

@@ -1 +1 @@
-1.25.8
+1.25.9

ci/install-package.sh

@@ -134,7 +134,7 @@ then
     echo -e "\e[35;1mSSH_KEY      = ${SSH_KEY_FINGERPRINT}\e[0m"
 
     DOCKER_SSH_AUTH_SOCK="/tmp/ssh_auth_sock"
-    DOCKER_MOUNT_KNOWN_HOSTS_ARGS=("-e" "SSH_AUTH_SOCK=${DOCKER_SSH_AUTH_SOCK}" "--mount" "type=bind,source=$SHARED_DIR/.ssh,target=/tmp/.ssh")
+    DOCKER_MOUNT_KNOWN_HOSTS_ARGS=("-e" "SSH_AUTH_SOCK=${DOCKER_SSH_AUTH_SOCK}" "--mount" "type=bind,source=${SHARED_DIR}/.ssh,target=/tmp/.ssh")
 
     # Used in the print statement to reproduce CI build locally
     ADDITIONAL_ARGS_LOCAL_INSTALL+=("--shared=/tmp/shared/${PACKAGE}" "--ssh")
@@ -191,6 +191,7 @@ fi
 # Docker container can show a header on start-up. We don't want to capture it
 docker run --detach --tty --name tue-env "${IMAGE_NAME}:${BRANCH_TAG}"
 DOCKER_HOME=$(docker exec tue-env bash -c 'echo "${HOME}"' | tr -d '\r')
+DOCKER_USER=$(docker exec tue-env bash -c 'echo "${USER}"' | tr -d '\r')
 docker stop tue-env  &> /dev/null || true
 docker rm tue-env &> /dev/null || true
 
@@ -212,13 +213,32 @@ docker exec -t tue-env bash -c 'sudo chown "${USER}":"${USER}" -R ~/.cache/pip'
 
 if [ "$USE_SSH" == "true" ]
 then
-    docker exec -t tue-env bash -c 'sudo chown "${USER}":"${USER}" -R /tmp/.ssh'
+    echo -e "\e[35;1msudo chown -R ${DOCKER_USER}:${DOCKER_USER} /tmp/.ssh\e[0m"
+    docker exec -t tue-env bash -c 'sudo chown -R "${USER}":"${USER}" /tmp/.ssh'
 
-    docker exec -t tue-env bash -c "[[ -f /tmp/.ssh/known_hosts ]] && mv ~/.ssh/known_hosts ~/.ssh/known_hosts_container"
-    docker exec -t tue-env bash -c 'sudo cp -r /tmp/.ssh/* ~/.ssh/ && sudo chown -R "${USER}":"${USER}" ~/.ssh && ls -aln ~/.ssh'
+    echo -e "\e[35;1mmv ~/.ssh/known_hosts ~/.ssh/known_hosts_container\e[0m"
+    docker exec -t tue-env bash -c "[[ ! -f /tmp/.ssh/known_hosts ]] || mv ~/.ssh/known_hosts ~/.ssh/known_hosts_container"
 
-    docker exec -t tue-env bash -c "[[ -f ~/.ssh/known_hosts && -f ~/.ssh/known_hosts_container ]] && ~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts"
+    echo -e "\e[35;1msudo cp -r /tmp/.ssh/* ~/.ssh/\e[0m"
+    docker exec -t tue-env bash -c 'sudo cp -r /tmp/.ssh/* ~/.ssh/'
+
+    echo -e "\e[35;1msudo chown -R ${DOCKER_USER}:${DOCKER_USER} ~/.ssh\e[0m"
+    docker exec -t tue-env bash -c 'sudo chown -R "${USER}":"${USER}" ~/.ssh'
+
+    echo -e "\e[35;1mls -alF ~/.ssh\e[0m"
+    docker exec -t tue-env bash -c "ls -alF ~/.ssh"
+
+    echo -e "\e[35;1m~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts\e[0m"
+    docker exec -t tue-env bash -c "[[ ! -f ~/.ssh/known_hosts || ! -f ~/.ssh/known_hosts_container ]] || ~/.tue/ci/ssh-merge-known_hosts.py ~/.ssh/known_hosts_container ~/.ssh/known_hosts --output ~/.ssh/known_hosts"
+
+    echo -e "\e[35;1meval \"\$(ssh-agent -s)\" && grep -slR \"PRIVATE\" ~/.ssh/ | xargs ssh-add\e[0m"
     docker exec -e DOCKER_SSH_AUTH_SOCK="$DOCKER_SSH_AUTH_SOCK" -t tue-env bash -c 'eval "$(ssh-agent -s)" && ln -sf "$SSH_AUTH_SOCK" "$DOCKER_SSH_AUTH_SOCK" && grep -slR "PRIVATE" ~/.ssh/ | xargs ssh-add'
+
+    echo -e "\e[35;1mecho -e 'Host *\n  StrictHostKeyChecking yes' >> ~/.ssh/config\e[0m"
+    docker exec -t tue-env bash -c "echo -e 'Host *\n  StrictHostKeyChecking yes' >> ~/.ssh/config"
+
+    echo -e "\e[35;1mActive SSH keys:\e[0m"
+    docker exec -t tue-env bash -c "ssh-add -l 2>/dev/null" | awk '{print $2}'
 fi
 
 # Use docker environment variables in all exec commands instead of script variables

ci/ssh-merge-known_hosts.py

@@ -26,6 +26,20 @@
 # Copied from https://blog.ganneff.de/2019/04/ssh-known-hosts-merge-by-key.html
 
 import argparse
+from collections import defaultdict
+from typing import Dict, List, Optional, Set, Tuple, Union
+
+
+def key_dict_factory() -> Dict[str, Union[Set[str], Optional[str]]]:
+    return {"comments": set(), "leading_comment_lines": set(), "marker": None}
+
+
+def truncate(s: str, w: int) -> str:
+    s = s.strip()
+    if len(s) > w:
+        s = s[: w - 3].strip() + "..."
+    return s
+
 
 parser = argparse.ArgumentParser(
     description="Merge ssh known host entries by key",
@@ -56,26 +70,56 @@
 
     output = stdout
 
-hostkeys = {}
+hostkeys: Dict[Tuple[str, str, str], Dict[str, Union[Set[str], str]]] = defaultdict(key_dict_factory)
 for kfile in args.files:
     with open(kfile) as kf:
+        leading_comment_lines = set()
         for line in kf:
             if line[0] == "#":
+                leading_comment_lines.add(line)
                 continue
-            line_splitted = line.rstrip().split(" ")
-            hosts = line_splitted.pop(0).split(",")
-            key_type = line_splitted.pop(0)
-            key = line_splitted[0]
-            if key not in hostkeys:
-                hostkeys[key] = {}
-                hostkeys[key]["hosts"] = []
-            hostkeys[key]["key_type"] = key_type
-            # Store the host entries, uniquify them
-            hostkeys[key]["hosts"].extend(hosts)
+            line_splitted: List[str] = line.rstrip().split(" ")
+            marker: Optional[str] = None
+            if line_splitted[0].startswith("@"):
+                marker = line_splitted.pop(0)
+            hosts: List[str] = line_splitted.pop(0).split(",")
+            key_type: str = line_splitted.pop(0)
+            key = line_splitted.pop(0)
+            comment: Optional[str] = None
+            if line_splitted:
+                if not line_splitted[0].startswith("#"):
+                    raise ValueError(f"Unknown remainder in line: {line}")
+                comment = " ".join(line_splitted)
+            for host in hosts:
+                unique_key = (host, key_type, key)
+                entry = hostkeys[unique_key]
+                if comment is not None:
+                    entry["comments"].add(comment)
+                if leading_comment_lines:
+                    entry["leading_comment_lines"].update(leading_comment_lines)
+                if marker is not None:
+                    if hostkeys[unique_key]["marker"] is not None:
+                        raise ValueError(
+                            f"Multiple markers for same key: ({truncate(unique_key[0], 25)}, {unique_key[1]}, {truncate(unique_key[2], 25)})"
+                        )
+                    entry["marker"] = marker
+
+            leading_comment_lines = set()
 
 # And now output it all
-for k, v in hostkeys.items():
-    output.write("%s %s %s\n" % (",".join(v["hosts"]), v["key_type"], k))
+for (host, key_type, key), v in hostkeys.items():
+    if v["leading_comment_lines"]:
+        for line in v["leading_comment_lines"]:
+            output.write(line)
+    line_items = []
+    if v["marker"] is not None:
+        line_items.append(v["marker"])
+    line_items.append(host)
+    line_items.append(key_type)
+    line_items.append(key)
+    if v["comments"]:
+        line_items.append(" ".join(v["comments"]))
+    output.write(f"{' '.join(line_items)}\n")
 
 # Write to output file
 if args.output:

dockerfiles/tue-env.Dockerfile

@@ -66,6 +66,7 @@ WORKDIR /home/"$USER"
 RUN mkdir -p -m 0700 ~/.ssh
 ADD ./known_hosts ./.ssh/known_hosts
 RUN sudo chown $USER_ID:$USER_ID ~/.ssh/known_hosts && sudo chmod 644 ~/.ssh/known_hosts
+RUN cp ~/.ssh/known_hosts ~/.ssh/known_hosts.bak
 
 # Setup Git HTTPS token authentication
 RUN { [[ -n "$OAUTH2_TOKEN" ]] && git config --global credential.helper '!f() { printf "%s\n" "username=oauth2" "password=$OAUTH2_TOKEN"; };f'; } || exit 0
@@ -113,6 +114,10 @@ RUN --mount=type=ssh,uid=$USER_ID --mount=type=bind,source=installer/bootstrap.b
     # Remove apt cache
     sudo rm -rf /var/lib/apt/lists/*
 
+# Restore known_hosts to one provided by the user
+RUN mv -f ~/.ssh/known_hosts.bak ~/.ssh/known_hosts
+
+# Remove Git HTTPS token authentication
 RUN { [[ -n "$OAUTH2_TOKEN" ]] && git config --global --unset credential.helper; } || exit 0
 
 # ----------------------------------------------------------------