Test CI for Broken Log Improvements

.github/workflows/build.yml

@@ -30,14 +30,18 @@ jobs:
           secret/data/github/repo/${{ github.repository }}/aws/rke2-ci-uploader/credentials AWS_ACCESS_KEY_ID ;
           secret/data/github/repo/${{ github.repository }}/aws/rke2-ci-uploader/credentials AWS_SECRET_ACCESS_KEY ;
     - name: Build
-      run: |
-        dapper -f Dockerfile --target dapper make dapper-ci
+      run: dapper -f Dockerfile --target dapper make dapper-ci
       env:
         AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
     - name: Test
-      run: |
-        dapper -f Dockerfile --target dapper make test
+      run: dapper -f Dockerfile --target dapper make test
+    - name: Upload Logs on Failure
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: rke2-test-logs
+        path: /tmp/rke2-logs/
   build-arm64:
     runs-on: runs-on,runner=8cpu-linux-arm64,run-id=${{ github.run_id }},image=ubuntu22-full-arm64,hdd=64
     steps:

.github/workflows/pr.yml

@@ -20,11 +20,15 @@ jobs:
         curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper
         chmod +x /usr/local/bin/dapper
     - name: Build
-      run: |
-        dapper -f Dockerfile --target dapper make dapper-ci
+      run: dapper -f Dockerfile --target dapper make dapper-ci
     - name: Test
-      run: |
-        dapper -f Dockerfile --target dapper make test
+      run: dapper -f Dockerfile --target dapper make test
+    - name: Upload Logs on Failure
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: rke2-test-logs
+        path: /tmp/rke2-logs/
   build-arm64:
     runs-on: runs-on,runner=8cpu-linux-arm64,run-id=${{ github.run_id }},image=ubuntu22-full-arm64,hdd=64
     steps:

Dockerfile

@@ -30,7 +30,7 @@ RUN zypper install -y systemd-rpm-macros
 
 # Dapper/Drone/CI environment
 FROM build AS dapper
-ENV DAPPER_ENV GODEBUG GOCOVER REPO TAG GITHUB_ACTION_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY DOCKER_USERNAME DOCKER_PASSWORD GH_TOKEN
+ENV DAPPER_ENV GODEBUG CI GOCOVER REPO TAG GITHUB_ACTION_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY DOCKER_USERNAME DOCKER_PASSWORD GH_TOKEN
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
 ENV DAPPER_OUTPUT ./dist ./bin ./build
@@ -117,7 +117,7 @@ RUN rm -vf /charts/*.sh /charts/*.md /charts/chart_versions.yaml
 # This image includes any host level programs that we might need. All binaries
 # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation.
 # This means bin/foo/bar will become bin/bar when rke2 installs this to the host
-FROM rancher/hardened-kubernetes:v1.30.3-rke2r1-build20240717 AS kubernetes
+FROM rancher/hardened-kubernetes:v1.31.0-rke2r1-build20240815 AS kubernetes
 FROM rancher/hardened-containerd:v1.7.20-k3s1-build20240812 AS containerd
 FROM rancher/hardened-crictl:v1.30.1-build20240812 AS crictl
 FROM rancher/hardened-runc:v1.1.13-build20240821 AS runc

charts/chart_versions.yaml

@@ -32,10 +32,10 @@ charts:
   - version: v0.25.501
     filename: /charts/rke2-flannel.yaml
     bootstrap: true
-  - version: 1.8.000
+  - version: 1.9.000
     filename: /charts/rancher-vsphere-cpi.yaml
     bootstrap: true
-  - version: 3.3.0-rancher100
+  - version: 3.3.1-rancher100
     filename: /charts/rancher-vsphere-csi.yaml
     bootstrap: true
   - version: 0.2.400

developer-docs/image_sources.md

@@ -51,7 +51,7 @@ The **Chart** column indicates the Helm chart that uses this image. `CORE` indic
 | rke2-cilium | rancher/mirrored-cilium-operator-generic | rancher/image-mirror |  | FALSE |
 | rke2-cilium | rancher/mirrored-cilium-startup-script | rancher/image-mirror |  | FALSE |
 |  |  |  |  |  |
-| rancher-vsphere-cpi | rancher/mirrored-cloud-provider-vsphere-cpi-release-manager | rancher/image-mirror |  | FALSE |
+| rancher-vsphere-cpi | rancher/mirrored-cloud-provider-vsphere | rancher/image-mirror |  | FALSE |
 | rancher-vsphere-cpi | rancher/mirrored-cloud-provider-vsphere-csi-release-driver | rancher/image-mirror |  | FALSE |
 | rancher-vsphere-cpi | rancher/mirrored-cloud-provider-vsphere-csi-release-syncer | rancher/image-mirror |  | FALSE |
 |  |  |  |  |  |

pkg/logging/logging.go

@@ -6,8 +6,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/natefinch/lumberjack"
 	"github.com/spf13/pflag"
+	"gopkg.in/natefinch/lumberjack.v2"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
 )

pkg/pebinaryexecutor/pebinary.go

@@ -23,12 +23,12 @@ import (
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
-	"github.com/natefinch/lumberjack"
 	"github.com/rancher/rke2/pkg/bootstrap"
 	"github.com/rancher/rke2/pkg/images"
 	"github.com/rancher/rke2/pkg/logging"
 	win "github.com/rancher/rke2/pkg/windows"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/natefinch/lumberjack.v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/client-go/rest"

pkg/rke2/rke2.go

@@ -18,10 +18,10 @@ import (
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
 	rawServer "github.com/k3s-io/k3s/pkg/server"
-	"github.com/natefinch/lumberjack"
 	"github.com/pkg/errors"
 	"github.com/rancher/rke2/pkg/controllers/cisnetworkpolicy"
 	"github.com/rancher/rke2/pkg/images"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"

pkg/windows/service_windows.go

@@ -72,7 +72,7 @@ func StartService() (bool, error) {
 	// built-in administrators of the Windows machine or by the local system.
 	// If this Win32 event (Global//stackdump-{pid}) is signaled, a goroutine launched by this call
 	// will dump the current stack trace into {windowsTemporaryDirectory}/{default.WindowsServiceName}.{pid}.stack.logs
-	profilings.SetupDumpStacks(version.Program, os.Getpid())
+	profilings.SetupDumpStacks(version.Program, os.Getpid(), os.TempDir())
 
 	go func() {
 		if err := svc.Run(version.Program, Service); err != nil {

scripts/build-images

@@ -68,13 +68,13 @@ EOF
 
 if [ "${GOARCH}" != "arm64" ]; then
 xargs -n1 -t docker image pull --quiet << EOF > build/images-vsphere.txt
-    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-cpi-release-manager:v1.30.1
-    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-driver:v3.3.0
-    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-syncer:v3.3.0
-    ${REGISTRY}/rancher/mirrored-sig-storage-csi-node-driver-registrar:v2.10.1
+    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere:v1.31.0
+    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-driver:v3.3.1
+    ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-syncer:v3.3.1
+    ${REGISTRY}/rancher/mirrored-sig-storage-csi-node-driver-registrar:v2.12.0
     ${REGISTRY}/rancher/mirrored-sig-storage-csi-resizer:v1.10.1
-    ${REGISTRY}/rancher/mirrored-sig-storage-livenessprobe:v2.12.0
-    ${REGISTRY}/rancher/mirrored-sig-storage-csi-attacher:v4.5.1
+    ${REGISTRY}/rancher/mirrored-sig-storage-livenessprobe:v2.14.0
+    ${REGISTRY}/rancher/mirrored-sig-storage-csi-attacher:v4.7.0
     ${REGISTRY}/rancher/mirrored-sig-storage-csi-provisioner:v4.0.1
     ${REGISTRY}/rancher/mirrored-sig-storage-csi-snapshotter:v7.0.2
 EOF

scripts/test-helpers

@@ -117,6 +117,10 @@ export -f verify-valid-versions
 # ---
 
 dump-logs() {
+    # Reduce verbosity in GitHub Actions
+    if [ "$CI" = "true" ]; then
+        set +x
+    fi
     local testID=$(basename $TEST_DIR)
     echo "#---------------------------------"
     echo "#- Begin: logs for run ($testID)"
@@ -141,6 +145,13 @@ dump-logs() {
             docker exec $name crictl pods >$node/logs/crictl-pods.txt
             docker exec $name crictl ps -a >$node/logs/crictl-ps.txt
         fi
+    done
+    # In GitHub Actions, don't dump logs to the console, as they will be uploaded as artifacts
+    if [ "$CI" = "true" ]; then
+        set -x
+        return
+    fi
+    for node in $TEST_DIR/*/*; do
         for log in $node/logs/*.log; do
             echo
             echo "#- Tail: $log"
@@ -279,7 +290,13 @@ export -f test-cleanup
 # ---
 
 test-setup() {
-    export TEST_DIR=$(mktemp -d '/tmp/XXXXXX')
+    # If running in Github Actions, use a known directory, so we can upload logs external to this script
+    if [ "$CI" = 'true' ]; then
+        mkdir -p /tmp/rke2-logs
+        export TEST_DIR=$(mktemp -d '/tmp/rke2-logs/XXXXXX')
+    else
+        export TEST_DIR=$(mktemp -d '/tmp/XXXXXX')
+    fi
     trap test-cleanup EXIT INT TERM
 
     mkdir -p $TEST_DIR/metadata

scripts/version.sh

@@ -32,8 +32,8 @@ REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code;
 PLATFORM=${GOOS}-${GOARCH}
 RELEASE=${PROG}.${PLATFORM}
 # hardcode versions unless set specifically
-KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.30.3}
-KUBERNETES_IMAGE_TAG=${KUBERNETES_IMAGE_TAG:-v1.30.3-rke2r1-build20240717}
+KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.31.0}
+KUBERNETES_IMAGE_TAG=${KUBERNETES_IMAGE_TAG:-v1.31.0-rke2r1-build20240815}
 ETCD_VERSION=${ETCD_VERSION:-v3.5.13-k3s1}
 PAUSE_VERSION=${PAUSE_VERSION:-3.6}
 CCM_VERSION=${CCM_VERSION:-v1.29.3-build20240515}