Skip to content

Commit

Permalink
roachtest: better triage schema workload flakes in backup-restore tests
Browse files Browse the repository at this point in the history
Informs #136709

Release note: none
  • Loading branch information
msbutler committed Dec 10, 2024
1 parent 4b94ccb commit b0632b9
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions pkg/cmd/roachtest/tests/backup_restore_roundtrip.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
gosql "database/sql"
"fmt"
"math/rand"
"strings"
"time"

"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
Expand Down Expand Up @@ -254,17 +255,26 @@ func startBackgroundWorkloads(
if err != nil {
return nil, err
}

handleChemaChangeError := func(err error) error {
// If the UNEXPECTED ERROR detail appears, the workload likely flaked.
// Otherwise, the workload could have failed due to other reasons like a node
// crash.
if err != nil && strings.Contains(errors.FlattenDetails(err), "UNEXPECTED ERROR") {
return registry.ErrorWithOwner(registry.OwnerSQLFoundations, errors.Wrapf(err, "schema change workload failed"))
}
return err
}
err = c.RunE(ctx, option.WithNodes(workloadNode), scInit.String())
if err != nil {
return nil, registry.ErrorWithOwner(registry.OwnerSQLFoundations, errors.Wrapf(err, "failed to init schema change workload"))
return nil, handleChemaChangeError(err)
}

run := func() (func(), error) {
tables, err := testUtils.loadTablesForDBs(ctx, l, testRNG, dbs...)
if err != nil {
return nil, err
}

stopBank := workloadWithCancel(m, func(ctx context.Context) error {
return c.RunE(ctx, option.WithNodes(workloadNode), bankRun.String())
})
Expand All @@ -274,7 +284,7 @@ func startBackgroundWorkloads(
})
stopSC := workloadWithCancel(m, func(ctx context.Context) error {
if err := c.RunE(ctx, option.WithNodes(workloadNode), scRun.String()); err != nil {
return registry.ErrorWithOwner(registry.OwnerSQLFoundations, errors.Wrapf(err, "failed to run schema change workload"))
return handleChemaChangeError(err)
}
return nil
})
Expand Down

0 comments on commit b0632b9

Please sign in to comment.