Skip to content

Commit

Permalink
roachtest: fix blocklist
Browse files Browse the repository at this point in the history 
Update the blockList and ignoreList for roachtests so that there's only one for
each version.

fixes #96935

Release note: None
  • Loading branch information
@ZhouXing19
ZhouXing19 committed Feb 13, 2023
1 parent 47331b1 commit 0b52650
Show file tree
Hide file tree
Showing 14 changed files with 52 additions and 560 deletions.
11 changes: 6 additions & 5 deletions pkg/cmd/roachtest/tests/blocklist_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,16 @@ func TestBlocklists(t *testing.T) {
}

blocklists := map[string]blocklist{
"hibernate": hibernateBlockList20_2,
"hibernate": hibernateBlockList,
"pgjdbc": pgjdbcBlockList,
"psycopg": psycopgBlockList20_2,
"psycopg": psycopgBlockList,
"django": djangoBlocklist,
"sqlAlchemy": sqlAlchemyBlocklist,
"libpq": libPQBlocklist20_2,
"gopg": gopgBlockList20_2,
"pgx": pgxBlocklist20_2,
"libpq": libPQBlocklist,
"gopg": gopgBlockList,
"pgx": pgxBlocklist,
"activerecord": activeRecordBlocklist,
"jasyncsql": jasyncSqlBlocklist,
}
type reasonCount struct {
reason string
Expand Down
26 changes: 4 additions & 22 deletions pkg/cmd/roachtest/tests/canary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"regexp"
"sort"
"strconv"
"strings"
"time"

"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
Expand All @@ -37,7 +36,7 @@ import (
// blocklist is a lists of known test errors and failures.
type blocklist map[string]string

// blocklistForVersion contains both a blocklist of known test errors and
// blocklistWithName contains both a blocklist of known test errors and
// failures but also an optional ignorelist for flaky tests.
// When the test suite is run, the results are compared to this list.
// Any passed test that is not on this blocklist is reported as PASS - expected
Expand All @@ -46,26 +45,9 @@ type blocklist map[string]string
// Any failed test that is not on blocklist list is reported as FAIL - unexpected
// Any test on this blocklist that is not run is reported as FAIL - not run
// Ant test in the ignorelist is reported as SKIP if it is run
type blocklistForVersion struct {
versionPrefix string
blocklistname string
blocklist blocklist
ignorelistname string
ignorelist blocklist
}

type blocklistsForVersion []blocklistForVersion

// getLists returns the appropriate blocklist and ignorelist based on the
// cockroach version. This check only looks to ensure that the prefix that
// matches.
func (b blocklistsForVersion) getLists(version string) (string, blocklist, string, blocklist) {
for _, info := range b {
if strings.HasPrefix(version, info.versionPrefix) {
return info.blocklistname, info.blocklist, info.ignorelistname, info.ignorelist
}
}
return "", nil, "", nil
type blocklistWithName struct {
blocklistname string
blocklist blocklist
}

func fetchCockroachVersion(
Expand Down
16 changes: 4 additions & 12 deletions pkg/cmd/roachtest/tests/gopg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,8 @@ func registerGopg(r registry.Registry) {
); err != nil {
t.Fatal(err)
}

blocklistName, expectedFailures, ignorelistName, ignorelist := gopgBlocklists.getLists(version)
if expectedFailures == nil {
t.Fatalf("No gopg blocklist defined for cockroach version %s", version)
}
if ignorelist == nil {
t.Fatalf("No gopg ignorelist defined for cockroach version %s", version)
}
t.L().Printf("Running cockroach version %s, using blocklist %s, using ignorelist %s",
version, blocklistName, ignorelistName)
version, "gopgBlockList", "gopgIgnoreList")

if err := c.RunE(ctx, node, fmt.Sprintf("mkdir -p %s", resultsDirPath)); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -131,7 +123,7 @@ func registerGopg(r registry.Registry) {
// test suites in themselves. Those are run with TestGinkgo test harness.
// First, we parse the result of running TestGinkgo.
if err := gopgParseTestGinkgoOutput(
results, rawResults, expectedFailures, ignorelist,
results, rawResults, gopgBlockList, gopgIgnoreList,
); err != nil {
t.Fatal(err)
}
Expand All @@ -155,9 +147,9 @@ func registerGopg(r registry.Registry) {

xmlResults := []byte(result.Stdout + result.Stderr)

results.parseJUnitXML(t, expectedFailures, ignorelist, xmlResults)
results.parseJUnitXML(t, gopgBlockList, gopgIgnoreList, xmlResults)
results.summarizeFailed(
t, "gopg", blocklistName, expectedFailures, version, gopgSupportedTag,
t, "gopg", "gopgBlockList", gopgBlockList, version, gopgSupportedTag,
0, /* notRunCount */
)
}
Expand Down
57 changes: 2 additions & 55 deletions pkg/cmd/roachtest/tests/gopg_blocklist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,6 @@

package tests

var gopgBlocklists = blocklistsForVersion{
{"v20.2", "gopgBlockList20_2", gopgBlockList20_2, "gopgIgnoreList20_2", gopgIgnoreList20_2},
{"v21.1", "gopgBlockList21_1", gopgBlockList21_1, "gopgIgnoreList21_1", gopgIgnoreList21_1},
{"v21.2", "gopgBlockList21_2", gopgBlockList21_2, "gopgIgnoreList21_2", gopgIgnoreList21_2},
{"v22.1", "gopgBlockList22_1", gopgBlockList22_1, "gopgIgnoreList22_1", gopgIgnoreList22_1},
{"v22.2", "gopgBlockList22_2", gopgBlockList22_2, "gopgIgnoreList22_2", gopgIgnoreList22_2},
{"v23.1", "gopgBlockList23_1", gopgBlockList23_1, "gopgIgnoreList23_1", gopgIgnoreList23_1},
}

// These are lists of known gopg test errors and failures.
// When the gopg test suite is run, the results are compared to this list.
// Any failed test that is on this list is reported as FAIL - expected.
Expand All @@ -27,16 +18,7 @@ var gopgBlocklists = blocklistsForVersion{
// Please keep these lists alphabetized for easy diffing.
// After a failed run, an updated version of this blocklist should be available
// in the test log.

var gopgBlockList23_1 = gopgBlockList22_2

var gopgBlockList22_2 = gopgBlockList22_1

var gopgBlockList22_1 = gopgBlockList21_2

var gopgBlockList21_2 = gopgBlockList21_1

var gopgBlockList21_1 = blocklist{
var gopgBlockList = blocklist{
"pg | BeforeQuery and AfterQuery CopyTo | is called for CopyTo with model": "41608",
"pg | BeforeQuery and AfterQuery CopyTo | is called for CopyTo without model": "41608",
"pg | CopyFrom/CopyTo | copies corrupted data to a table": "41608",
Expand All @@ -59,42 +41,7 @@ var gopgBlockList21_1 = blocklist{
"v10.TestUnixSocket": "31113",
}

var gopgBlockList20_2 = blocklist{
"pg | BeforeQuery and AfterQuery CopyFrom | is called for CopyFrom with model": "41608",
"pg | BeforeQuery and AfterQuery CopyFrom | is called for CopyFrom without model": "41608",
"pg | BeforeQuery and AfterQuery CopyTo | is called for CopyTo with model": "41608",
"pg | BeforeQuery and AfterQuery CopyTo | is called for CopyTo without model": "41608",
"pg | CopyFrom/CopyTo | copies corrupted data to a table": "41608",
"pg | CopyFrom/CopyTo | copies data from a table and to a table": "41608",
"pg | CountEstimate | works": "17511",
"pg | CountEstimate | works when there are no results": "17511",
"pg | CountEstimate | works with GROUP": "17511",
"pg | CountEstimate | works with GROUP when there are no results": "17511",
"pg | Listener | is closed when DB is closed": "41522",
"pg | Listener | listens for notifications": "41522",
"pg | Listener | reconnects on receive error": "41522",
"pg | Listener | returns an error on timeout": "41522",
"pg | Listener | supports concurrent Listen and Receive": "41522",
"v10.ExampleDB_Model_postgresArrayStructTag": "32552",
"v10.TestBigColumn": "41608",
"v10.TestConversion": "32552",
"v10.TestGinkgo": "41522",
"v10.TestGocheck": "17511",
"v10.TestReadColumnValue": "26925",
"v10.TestUnixSocket": "31113",
}

var gopgIgnoreList23_1 = gopgIgnoreList22_2

var gopgIgnoreList22_2 = gopgIgnoreList22_1

var gopgIgnoreList22_1 = gopgIgnoreList21_2

var gopgIgnoreList21_2 = gopgIgnoreList21_1

var gopgIgnoreList21_1 = gopgIgnoreList20_2

var gopgIgnoreList20_2 = blocklist{
var gopgIgnoreList = blocklist{
// These "fetching" tests assume a particular order when ORDER BY clause is
// omitted from the query by the ORM itself.
"pg | ORM slice model | fetches Book relations": "41690",
Expand Down
19 changes: 9 additions & 10 deletions pkg/cmd/roachtest/tests/hibernate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ type hibernateOptions struct {
testDir string
buildCmd,
testCmd string
blocklists blocklistsForVersion
dbSetupFunc func(ctx context.Context, t test.Test, c cluster.Cluster)
blocklistWithName blocklistWithName
dbSetupFunc func(ctx context.Context, t test.Test, c cluster.Cluster)
}

var (
Expand All @@ -40,9 +40,9 @@ var (
testDir: "hibernate-core",
buildCmd: `cd /mnt/data1/hibernate/hibernate-core/ && ./../gradlew test -Pdb=cockroachdb ` +
`--tests org.hibernate.jdbc.util.BasicFormatterTest.*`,
testCmd: "cd /mnt/data1/hibernate/hibernate-core/ && ./../gradlew test -Pdb=cockroachdb",
blocklists: hibernateBlocklists,
dbSetupFunc: nil,
testCmd: "cd /mnt/data1/hibernate/hibernate-core/ && ./../gradlew test -Pdb=cockroachdb",
blocklistWithName: blocklistWithName{blocklistname: "hibernateBlockList", blocklist: hibernateBlockList},
dbSetupFunc: nil,
}
hibernateSpatialOpts = hibernateOptions{
testName: "hibernate-spatial",
Expand All @@ -51,7 +51,7 @@ var (
`--tests org.hibernate.spatial.dialect.postgis.*`,
testCmd: `cd /mnt/data1/hibernate/hibernate-spatial && ` +
`HIBERNATE_CONNECTION_LEAK_DETECTION=true ./../gradlew test -Pdb=cockroachdb_spatial`,
blocklists: hibernateSpatialBlocklists,
blocklistWithName: blocklistWithName{blocklistname: "hibernateSpatialBlockList", blocklist: hibernateSpatialBlockList},
dbSetupFunc: func(ctx context.Context, t test.Test, c cluster.Cluster) {
db := c.Conn(ctx, t.L(), 1)
defer db.Close()
Expand Down Expand Up @@ -169,10 +169,9 @@ func registerHibernate(r registry.Registry, opt hibernateOptions) {
t.Fatal(err)
}

blocklistName, expectedFailures, _, _ := opt.blocklists.getLists(version)
if expectedFailures == nil {
t.Fatalf("No hibernate blocklist defined for cockroach version %s", version)
}
blocklistName := opt.blocklistWithName.blocklistname
expectedFailures := opt.blocklistWithName.blocklist

t.L().Printf("Running cockroach version %s, using blocklist %s", version, blocklistName)

t.Status("running hibernate test suite, will take at least 3 hours")
Expand Down
Loading

0 comments on commit 0b52650

Please sign in to comment.