From cffa8885201094dfd84e9593e8346b12d4be36ab Mon Sep 17 00:00:00 2001
From: Matt Spilchen <matt.spilchen@cockroachlabs.com>
Date: Thu, 12 Dec 2024 13:45:43 -0400
Subject: [PATCH] ccl/changefeedccl/cdcevent: Fix flake in
 TestEventColumnOrderingWithSchemaChanges

This commit addresses a test flake in
TestEventColumnOrderingWithSchemaChanges. The test relies on the SCHEMA
CHANGE GC job, which triggers the RangeFeedDeleteRange event. Since this
job runs asynchronously, delays in its execution occasionally caused the
test to fail.

To mitigate this, the test has been updated to ensure the job starts
more promptly. Additionally, debugging aids have been added to
facilitate future investigations if the test fails again.

Epic: None
Closes #136371
Release note: None
---
 pkg/ccl/changefeedccl/cdcevent/BUILD.bazel   |  1 +
 pkg/ccl/changefeedccl/cdcevent/event_test.go | 16 ++++++++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel b/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel
index 6ae15d8acbc6..0d2582c6cef4 100644
--- a/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel
+++ b/pkg/ccl/changefeedccl/cdcevent/BUILD.bazel
@@ -78,6 +78,7 @@ go_test(
         "//pkg/sql/schemachanger/scplan",
         "//pkg/sql/sem/eval",
         "//pkg/sql/sem/tree",
+        "//pkg/sql/sqltestutils",
         "//pkg/sql/types",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/skip",
diff --git a/pkg/ccl/changefeedccl/cdcevent/event_test.go b/pkg/ccl/changefeedccl/cdcevent/event_test.go
index a05677b9d53d..a39154be3a61 100644
--- a/pkg/ccl/changefeedccl/cdcevent/event_test.go
+++ b/pkg/ccl/changefeedccl/cdcevent/event_test.go
@@ -26,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
+	"github.com/cockroachdb/cockroach/pkg/sql/sqltestutils"
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -595,7 +596,12 @@ func TestEventColumnOrderingWithSchemaChanges(t *testing.T) {
 		kvserver.RangefeedEnabled.Override(ctx, &l.ClusterSettings().SV, true)
 	}
 
+	// Be aggressive with the SCHEMA CHANGE GC job. For tests involving column
+	// rewrites, this job must run for the tests to succeed. Its execution
+	// triggers RangeFeedDeleteRange events.
 	sqlDB := sqlutils.MakeSQLRunner(db)
+	defer sqltestutils.DisableGCTTLStrictEnforcement(t, db)()
+	sqlDB.Exec(t, "SET CLUSTER SETTING jobs.registry.interval.adopt='1s'")
 
 	type decodeExpectation struct {
 		expectUnwatchedErr bool
@@ -730,8 +736,9 @@ func TestEventColumnOrderingWithSchemaChanges(t *testing.T) {
 		},
 	} {
 		t.Run(tc.testName, func(t *testing.T) {
-			sqlDB.Exec(t, `
-				CREATE TABLE foo (
+			sqlDB.ExecMultiple(t,
+				`DROP TABLE IF EXISTS foo`,
+				`CREATE TABLE foo (
 					i INT,
 					j INT,
 					a STRING,
@@ -748,6 +755,9 @@ func TestEventColumnOrderingWithSchemaChanges(t *testing.T) {
 			popRow, cleanup := cdctest.MakeRangeFeedValueReaderExtended(t, s.ExecutorConfig(), tableDesc)
 			defer cleanup()
 
+			_, err := sqltestutils.AddImmediateGCZoneConfig(db, tableDesc.GetID())
+			require.NoError(t, err)
+
 			targetType := jobspb.ChangefeedTargetSpecification_EACH_FAMILY
 			if tc.familyName != "" {
 				targetType = jobspb.ChangefeedTargetSpecification_COLUMN_FAMILY
@@ -769,8 +779,10 @@ func TestEventColumnOrderingWithSchemaChanges(t *testing.T) {
 			require.NoError(t, err)
 
 			expectedEvents := len(tc.expectMainFamily) + len(tc.expectECFamily)
+			log.Infof(ctx, "expectedEvents: %d\n", expectedEvents)
 			for i := 0; i < expectedEvents; i++ {
 				v, deleteRange := popRow(t)
+				log.Infof(ctx, "event[%d]: v=%+v, deleteRange=%+v", i, v, deleteRange)
 
 				if deleteRange != nil {
 					// Should not see a RangeFeedValue and a RangeFeedDeleteRange