Fixed Online Upgrade Error (#896)

When we run an online upgrade from Version 24.2.x to 24.3.x, the
operator will fail due to pod facts collection error. The reason is that
secondary subclusters still use the old Vertica Version 24.2.x when
primary subclusters finished the upgrade. As a result, the operator will
call vclusterOps API to collect node details on secondary subclusters,
but secondary subclusters doesn't have that API due to a low Vertica
version.

This PR saved the old Vertica version prior to the upgrade, and used
that version to decide if the operator should call vclusterOps API to
collect node details during the upgrade.

api/v1/helpers.go

@@ -407,6 +407,11 @@ func (v *VerticaDB) IsOnlineUpgradeInProgress() bool {
 	return v.IsStatusConditionTrue(OnlineUpgradeInProgress)
 }
 
+// IsOnlineUpgradeInProgress returns true if an upgrade is in progress
+func (v *VerticaDB) IsUpgradeInProgress() bool {
+	return v.IsStatusConditionTrue(UpgradeInProgress)
+}
+
 // IsStatusConditionTrue returns true when the conditionType is present and set to
 // `metav1.ConditionTrue`
 func (v *VerticaDB) IsStatusConditionTrue(statusCondition string) bool {

api/v1/version.go

@@ -90,6 +90,12 @@ func (v *VerticaDB) GetVerticaVersionStr() (string, bool) {
 	return ver, ok
 }
 
+// GetVerticaVersionStr returns vertica version prior to the upgrade
+func (v *VerticaDB) GetPreviousVerticaVersionStr() (string, bool) {
+	ver, ok := v.ObjectMeta.Annotations[vmeta.PreviousVersionAnnotation]
+	return ver, ok
+}
+
 // MakeVersionInfo will construct an Info struct by extracting the version from the
 // given vdb.  This returns false if it was unable to get the version from the
 // vdb.
@@ -102,6 +108,17 @@ func (v *VerticaDB) MakeVersionInfo() (*version.Info, bool) {
 	return version.MakeInfoFromStr(vdbVer)
 }
 
+// MakePerviousVersionInfo will construct an Info struct by extracting the previous version
+// from the given vdb. This returns false if it was unable to get the version from the vdb.
+func (v *VerticaDB) MakePreviousVersionInfo() (*version.Info, bool) {
+	vdbVer, ok := v.GetPreviousVerticaVersionStr()
+	// If the annotation isn't present, we abort creation of Info
+	if !ok {
+		return nil, false
+	}
+	return version.MakeInfoFromStr(vdbVer)
+}
+
 // MakeVersionInfoCheck is like MakeVersionInfo but returns an error if the
 // version is missing. Use this in places where it is a failure if the version
 // is missing.

pkg/controllers/vdb/imageversion_reconciler.go

@@ -234,6 +234,10 @@ func (v *ImageVersionReconciler) getVersion(ctx context.Context, pod *PodFact) (
 // fail if it detects an invalid upgrade path.
 func (v *ImageVersionReconciler) updateVDBVersion(ctx context.Context, newVersion string) (ctrl.Result, error) {
 	versionAnnotations := vapi.ParseVersionOutput(newVersion)
+	// if we found vertica version is changed, we save previous vertica version to vdb
+	if versionAnnotations[vmeta.VersionAnnotation] != v.Vdb.ObjectMeta.Annotations[vmeta.VersionAnnotation] {
+		versionAnnotations[vmeta.PreviousVersionAnnotation] = v.Vdb.ObjectMeta.Annotations[vmeta.VersionAnnotation]
+	}
 
 	if v.EnforceUpgradePath && !v.Vdb.GetIgnoreUpgradePath() {
 		ok, failureReason, err := v.isUpgradePathSupported(ctx, versionAnnotations)

pkg/controllers/vdb/podfacts.go

@@ -36,6 +36,7 @@ import (
 	"github.com/vertica/vertica-kubernetes/pkg/names"
 	"github.com/vertica/vertica-kubernetes/pkg/paths"
 	config "github.com/vertica/vertica-kubernetes/pkg/vdbconfig"
+	"github.com/vertica/vertica-kubernetes/pkg/version"
 	"github.com/vertica/vertica-kubernetes/pkg/vk8s"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -766,14 +767,29 @@ func (p *PodFacts) checkIsDBCreated(_ context.Context, vdb *vapi.VerticaDB, pf *
 // the operator will call vclusterOps API to collect the node info. Otherwise, the operator will
 // execute vsql inside the pod to collect the node info.
 func (p *PodFacts) makeNodeInfoFetcher(vdb *vapi.VerticaDB, pf *PodFact) catalog.Fetcher {
-	verInfo, ok := vdb.MakeVersionInfo()
+	var verInfo *version.Info
+	var ok bool
+	oldVerInfo, ok1 := vdb.MakePreviousVersionInfo()
+	newVerInfo, ok2 := vdb.MakeVersionInfo()
+	// During the upgrade, we should use the old version to determine if we will call
+	// vclusterOps API to collect node info. The reason is some subclusters might uses
+	// a low version that does not contain vcluster API in the midst of the upgrade.
+	// Apart from the upgrade, we should check current version to make the decision.
+	if vdb.IsUpgradeInProgress() {
+		verInfo = oldVerInfo
+		ok = ok1
+	} else {
+		verInfo = newVerInfo
+		ok = ok2
+	}
 	if verInfo != nil && ok {
 		if !verInfo.IsOlder(vapi.FetchNodeDetailsWithVclusterOpsMinVersion) && vmeta.UseVClusterOps(vdb.Annotations) {
 			return catalog.MakeVCluster(vdb, p.VerticaSUPassword, pf.podIP, p.Log, p.VRec.GetClient(), p.VRec.GetEventRecorder())
 		}
 	} else {
-		p.Log.Info("Cannot get a correct vertica version from the annotation",
-			"vertica version annotation", vdb.ObjectMeta.Annotations[vmeta.VersionAnnotation])
+		p.Log.Info("Cannot get a correct vertica version from the annotations",
+			"vertica version annotation", vdb.ObjectMeta.Annotations[vmeta.VersionAnnotation],
+			"vertica previous version annotation", vdb.ObjectMeta.Annotations[vmeta.PreviousVersionAnnotation])
 	}
 	return catalog.MakeVSQL(vdb, p.PRunner, pf.name, pf.execContainerName, pf.vnodeName)
 }

pkg/meta/annotations.go

@@ -132,6 +132,8 @@ const (
 	VersionAnnotation   = "vertica.com/version"
 	BuildDateAnnotation = "vertica.com/buildDate"
 	BuildRefAnnotation  = "vertica.com/buildRef"
+	// Annotation that records the vertica version prior to the upgrade
+	PreviousVersionAnnotation = "vertica.com/previous-version"
 	// Annotation for the database's revive_instance_id
 	ReviveInstanceIDAnnotation = "vertica.com/revive-instance-id"