Add timestamps to support

Issue: PGO-274

docs/content/reference/pgo_support_export.md

@@ -97,6 +97,7 @@ Collecting PostgresCluster pod logs...
 Collecting monitoring pod logs...
 Collecting Patroni info...
 Collecting processes...
+Collecting system times from containers...
 Collecting PGO CLI logs...
 ┌────────────────────────────────────────────────────────────────
 | Archive file size: 0.02 MiB

internal/cmd/exec.go

@@ -79,3 +79,10 @@ func (exec Executor) processes() (string, string, error) {
 
 	return stdout.String(), stderr.String(), err
 }
+
+// systemTime returns the output of the date command
+func (exec Executor) systemTime() (string, string, error) {
+	var stdout, stderr bytes.Buffer
+	err := exec(nil, &stdout, &stderr, "date")
+	return stdout.String(), stderr.String(), err
+}

internal/cmd/export.go

@@ -260,6 +260,7 @@ Collecting PostgresCluster pod logs...
 Collecting monitoring pod logs...
 Collecting Patroni info...
 Collecting processes...
+Collecting system times from containers...
 Collecting PGO CLI logs...
 ┌────────────────────────────────────────────────────────────────
 | Archive file size: 0.02 MiB
@@ -438,6 +439,11 @@ Collecting PGO CLI logs...
 			err = gatherProcessInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
 		}
 
+		// Exec to get Container system time
+		if err == nil {
+			err = gatherSystemTime(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
+		}
+
 		// Print cli output
 		writeInfo(cmd, "Collecting PGO CLI logs...")
 		path := clusterName + "/logs/cli"
@@ -1078,6 +1084,103 @@ func gatherPatroniInfo(ctx context.Context,
 	return nil
 }
 
+// gatherSystemTime takes a client and buffer and collects system time
+// in each Pod and calculates the delta against client system time.
+func gatherSystemTime(ctx context.Context,
+	clientset *kubernetes.Clientset,
+	config *rest.Config,
+	namespace string,
+	clusterName string,
+	tw *tar.Writer,
+	cmd *cobra.Command,
+) error {
+	writeInfo(cmd, "Collecting system times from containers...")
+	// Get the cluster Pods by label
+	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: "postgres-operator.crunchydata.com/cluster=" + clusterName,
+	})
+	if err != nil {
+		if apierrors.IsForbidden(err) {
+			writeInfo(cmd, err.Error())
+			return nil
+		}
+		return err
+	}
+
+	if len(pods.Items) == 0 {
+		// If we didn't find any resources, skip
+		writeInfo(cmd, "PostgresCluster Pods not found when gathering system time information, skipping")
+		return nil
+	}
+
+	podExec, err := util.NewPodExecutor(config)
+	if err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+	for _, pod := range pods.Items {
+		for _, container := range pod.Spec.Containers {
+			// Attempt to exec in and run 'date' command in the first available container.
+			exec := func(stdin io.Reader, stdout, stderr io.Writer, command ...string,
+			) error {
+				return podExec(namespace, pod.GetName(), container.Name,
+					stdin, stdout, stderr, command...)
+			}
+
+			stdout, stderr, err := Executor(exec).systemTime()
+			if err == nil {
+				buf = writeSystemTime(buf, pod, stdout, stderr)
+				break
+			} else if err != nil {
+				// If we get an RBAC error, let the user know. Otherwise, just
+				// try the next container.
+				if apierrors.IsForbidden(err) {
+					writeInfo(cmd, fmt.Sprintf(
+						"Failed to get system time for Container \"%s\" in Pod \"%s\". Error: \"%s\"",
+						container.Name, pod.GetName(), err.Error()))
+				}
+				continue
+			}
+		}
+	}
+
+	path := clusterName + "/" + "system-time"
+	if err := writeTar(tw, buf.Bytes(), path, cmd); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeSystemTime(buf bytes.Buffer, pod corev1.Pod, stdout, stderr string) bytes.Buffer {
+	// Get client datetime.
+	clientTime := time.Now().UTC()
+	clientDateTimeStr := clientTime.Format(time.UnixDate)
+
+	var deltaStr string
+	var containerDateTimeStr string
+	if containerDateTime, err := time.Parse(time.UnixDate, strings.TrimSpace(stdout)); err == nil {
+		// Calculate difference between client and container datetime.
+		containerDateTimeStr = containerDateTime.Format(time.UnixDate)
+		deltaStr = fmt.Sprint(clientTime.Sub(containerDateTime).Truncate(time.Second))
+	} else {
+		// Parse failed, use stdout instead.
+		containerDateTimeStr = strings.TrimSpace(stdout)
+		deltaStr = "No result"
+	}
+
+	// Build report.
+	fmt.Fprintln(&buf, "Delta: "+deltaStr+"\tPod time: "+containerDateTimeStr+
+		"\tClient time: "+clientDateTimeStr+"\tPod name: "+pod.GetName())
+
+	if stderr != "" {
+		buf.Write([]byte(stderr))
+	}
+
+	return buf
+}
+
 // gatherProcessInfo takes a client and buffer execs into relevant pods to grab
 // running process information for each Pod.
 func gatherProcessInfo(ctx context.Context,

testing/kuttl/e2e/support-export/01--support_export.yaml

@@ -18,14 +18,22 @@ commands:
       exit 1
     }
 
-    # check that the cluster-names file exist and is not empty
+    # check that the cluster-names file exists and is not empty
     if [[ ! -s ./kuttl-support-cluster/cluster-names ]]
     then
       echo "Expected cluster-names file to not be empty"
       eval "$CLEANUP"
       exit 1
     fi
 
+    # check that the system-time file exists and is not empty
+    if [[ ! -s ./kuttl-support-cluster/system-time ]]
+    then
+      echo "Expected system-time file to not be empty"
+      eval "$CLEANUP"
+      exit 1
+    fi
+
     # check that the context file exist and is not empty
     if [[ ! -s ./kuttl-support-cluster/current-context ]]
     then