Stay beneath data dir / run dir when cleaning up paths during reset

The current way of cleaning up the data directory is too simple to ensure
that the recursive directory removal won't remove data that's not beneath
it.

Re-implement the recursive directory traversal in a way that won't
cross mount points unintentionally, but unmounts everything recursively
instead.

Signed-off-by: Tom Wieczorek <[email protected]>

inttest/reset/obstruct-data-dir.sh

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2024 k0s authors
+#shellcheck shell=ash
+
+set -eu
+
+make_dir() { mkdir -- "$1" && echo "$1"; }
+make_file() { echo "$1" >"$1" && echo "$1"; }
+
+make_bind_mounts() {
+  local real="$1"
+  local target="$2"
+
+  # Directory bind mount
+  make_dir "$real/real_dir"
+  make_file "$real/real_dir/real_dir_info.txt"
+  make_dir "$target/bind_dir"
+  mount --bind -- "$real/real_dir" "$target/bind_dir"
+
+  # File bind mount
+  make_file "$real/real_file.txt"
+  make_file "$target/bind_file.txt"
+  mount --bind -- "$real/real_file.txt" "$target/bind_file.txt"
+
+  # Recursive directory bind mount
+  make_dir "$real/real_recursive_dir"
+  make_file "$real/real_recursive_dir/real_recursive_dir.txt"
+  make_dir "$real/real_recursive_dir/bind_dir"
+  mount --bind -- "$real/real_dir" "$real/real_recursive_dir/bind_dir"
+  make_file "$real/real_recursive_dir/bind_file.txt"
+  mount --bind -- "$real/real_file.txt" "$real/real_recursive_dir/bind_file.txt"
+  make_dir "$target/rbind_dir"
+  mount --rbind -- "$real/real_recursive_dir" "$target/rbind_dir"
+
+  # Directory overmounts
+  make_dir "$real/overmount_dir"
+  make_file "$real/overmount_dir/in_overmount_dir.txt"
+  mount --bind -- "$real/overmount_dir" "$target/bind_dir"
+
+  # File overmounts
+  make_file "$real/overmount_file.txt"
+  mount --bind -- "$real/overmount_file.txt" "$target/bind_file.txt"
+}
+
+obstruct() {
+  local dataDir="$1"
+  local realDir
+
+  realDir="$(mktemp -t -d k0s_reset_inttest.XXXXXX)"
+
+  local dir="$dataDir"/obstructed
+  make_dir "$dir"
+
+  # Directories and files with restricted permissions
+  make_dir "$dir/restricted_dir"
+  make_file "$dir/restricted_dir/no_read_file.txt"
+  chmod 000 -- "$dir/restricted_dir/no_read_file.txt" # No permissions on the file
+  make_dir "$dir/restricted_dir/no_exec_dir"
+  chmod 000 -- "$dir/restricted_dir/no_exec_dir" # No permissions on the directory
+  make_dir "$dir/restricted_dir/no_exec_nonempty_dir"
+  make_file "$dir/restricted_dir/no_exec_nonempty_dir/.hidden_file"
+  chmod 000 -- "$dir/restricted_dir/no_exec_nonempty_dir" # No permissions on the directory
+
+  # Symlinks pointing outside the directory tree
+  make_dir "$realDir/some_dir"
+  make_file "$realDir/some_dir/real_file.txt"
+  ln -s -- "$realDir/some_dir/real_file.txt" "$dir/symlink_to_file" # Symlink to a file
+  ln -s -- "$realDir/some_dir" "$dir/symlink_to_dir"                # Symlink to a directory
+
+  # Bind mounts pointing outside the directory tree
+  make_bind_mounts "$realDir" "$dir"
+
+  # Bind mounts outside the directory tree pointing into it
+  # make_bind_mounts "$dir" "$realDir"
+}
+
+obstruct "$@"

inttest/reset/reset_test.go

@@ -17,6 +17,11 @@ limitations under the License.
 package reset
 
 import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+	"strings"
 	"testing"
 
 	testifysuite "github.com/stretchr/testify/suite"
@@ -28,11 +33,14 @@ type suite struct {
 	common.BootlooseSuite
 }
 
+//go:embed obstruct-data-dir.sh
+var obstructScript []byte
+
 func (s *suite) TestReset() {
 	ctx := s.Context()
 	workerNode := s.WorkerNode(0)
 
-	if ok := s.Run("k0s gets up", func() {
+	if !s.Run("k0s gets up", func() {
 		s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server"))
 		s.Require().NoError(s.RunWorkers())
 
@@ -44,11 +52,7 @@ func (s *suite) TestReset() {
 
 		s.T().Log("waiting to see CNI pods ready")
 		s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start")
-	}); !ok {
-		return
-	}
 
-	s.Run("k0s reset", func() {
 		ssh, err := s.SSH(ctx, workerNode)
 		s.Require().NoError(err)
 		defer ssh.Disconnect()
@@ -57,14 +61,47 @@ func (s *suite) TestReset() {
 		s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory")
 
 		s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims")
+	}) {
+		return
+	}
 
+	var obstructingPaths bytes.Buffer
+
+	if !s.Run("prepare k0s reset", func() {
 		s.NoError(s.StopWorker(workerNode), "Failed to stop k0s")
 
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
+		streams, flushStreams := common.TestLogStreams(s.T(), "obstruct data dir")
+		streams.In = bytes.NewReader(obstructScript)
+		streams.Out = io.MultiWriter(&obstructingPaths, streams.Out)
+		err = ssh.Exec(ctx, "sh -s -- /var/lib/k0s", streams)
+		flushStreams()
+		s.Require().NoError(err)
+	}) {
+		return
+	}
+
+	s.Run("k0s reset", func() {
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
 		streams, flushStreams := common.TestLogStreams(s.T(), "reset")
 		err = ssh.Exec(ctx, "k0s reset --debug", streams)
 		flushStreams()
 		s.NoError(err, "k0s reset didn't exit cleanly")
 
+		for _, path := range strings.Split(string(bytes.TrimSpace(obstructingPaths.Bytes())), "\n") {
+			if strings.HasPrefix(path, "/var/lib/k0s") {
+				s.NoError(ssh.Exec(ctx, fmt.Sprintf("! test -e %q", path), common.SSHStreams{}), "Failed to verify non-existence of %s", path)
+			} else {
+				s.NoError(ssh.Exec(ctx, fmt.Sprintf("test -e %q", path), common.SSHStreams{}), "Failed to verify existence of %s", path)
+			}
+		}
+
 		// /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty
 		s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty")
 		s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists")