diff --git a/Makefile b/Makefile index 21fcfcedf1..60350a6046 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ TEST_FLAGS=-race -shuffle=on -timeout 300s PLAYGROUND_DIRECTORY=playground LENS_TEST_DIRECTORY=tests/integration/schema/migrations CLI_TEST_DIRECTORY=tests/integration/cli +CHANGE_DETECTOR_TEST_DIRECTORY=tests/change_detector DEFAULT_TEST_DIRECTORIES=$$(go list ./... | grep -v -e $(LENS_TEST_DIRECTORY) -e $(CLI_TEST_DIRECTORY)) default: @@ -294,8 +295,7 @@ test\:coverage-html: .PHONY: test\:changes test\:changes: - @$(MAKE) deps:lens - env DEFRA_DETECT_DATABASE_CHANGES=true DEFRA_CLIENT_GO=true gotestsum -- ./... -shuffle=on -p 1 + gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... --tags change_detector .PHONY: validate\:codecov validate\:codecov: diff --git a/docs/data_format_changes/i1436-no-change-tests-updated.md b/docs/data_format_changes/i1436-no-change-tests-updated.md new file mode 100644 index 0000000000..89f7305133 --- /dev/null +++ b/docs/data_format_changes/i1436-no-change-tests-updated.md @@ -0,0 +1,3 @@ +# Parallel change detector + +This is is not a breaking change. The change detector has been updated to allow for parallel test runs. There were changes to environment variables and test setup that makes the previous version of the change detector incompatible with this version. diff --git a/tests/change_detector/README.md b/tests/change_detector/README.md new file mode 100644 index 0000000000..4d824fb60f --- /dev/null +++ b/tests/change_detector/README.md @@ -0,0 +1,15 @@ +# Change Detector + +The change detector is used to detect data format changes between versions of DefraDB. + +## How it works + +The tests run using a `source` and `target` branch of DefraDB. Each branch is cloned into a temporary directory and dependencies are installed. + +The test runner executes all of the common test packages available in the `source` and `target` tests directory. + +For each test package execution the following steps occur: + +- Create a temporary data directory. This is used to share data between `source` and `target`. +- Run the `source` version in setup only mode. This creates test fixtures in the shared data directory. +- Run the `target` version in change detector mode. This skips the setup and executes the tests using the shared data directory. diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go new file mode 100644 index 0000000000..ac9bc1a23f --- /dev/null +++ b/tests/change_detector/change_detector_test.go @@ -0,0 +1,205 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +//go:build change_detector + +package change_detector + +import ( + "fmt" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestChanges(t *testing.T) { + sourceRepoDir := t.TempDir() + execClone(t, sourceRepoDir, Repository, SourceBranch) + execMakeDeps(t, sourceRepoDir) + + var targetRepoDir string + if TargetBranch == "" { + // default to the local branch + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + require.NoError(t, err, string(out)) + targetRepoDir = strings.TrimSpace(string(out)) + } else { + // check out the target branch + targetRepoDir = t.TempDir() + execClone(t, targetRepoDir, Repository, TargetBranch) + execMakeDeps(t, targetRepoDir) + } + + if checkIfDatabaseFormatChangesAreDocumented(t, sourceRepoDir, targetRepoDir) { + t.Skip("skipping test with documented database format changes") + } + + targetRepoTestDir := filepath.Join(targetRepoDir, "tests", "integration") + targetRepoPkgList := execList(t, targetRepoTestDir) + + sourceRepoTestDir := filepath.Join(sourceRepoDir, "tests", "integration") + sourceRepoPkgList := execList(t, sourceRepoTestDir) + + sourceRepoPkgMap := make(map[string]bool) + for _, pkg := range sourceRepoPkgList { + sourceRepoPkgMap[pkg] = true + } + + for _, pkg := range targetRepoPkgList { + pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") + t.Run(pkgName, func(t *testing.T) { + if pkg == "" || !sourceRepoPkgMap[pkg] { + t.Skip("skipping unknown or new test package") + } + + t.Parallel() + dataDir := t.TempDir() + + sourceTestPkg := filepath.Join(sourceRepoDir, pkgName) + execTest(t, sourceTestPkg, dataDir, true) + + targetTestPkg := filepath.Join(targetRepoDir, pkgName) + execTest(t, targetTestPkg, dataDir, false) + }) + } +} + +// execList returns a list of all packages in the given directory. +func execList(t *testing.T, dir string) []string { + cmd := exec.Command("go", "list", "./...") + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err, string(out)) + + return strings.Split(string(out), "\n") +} + +// execTest runs the tests in the given directory and sets the data +// directory and setup only environment variables. +func execTest(t *testing.T, dir, dataDir string, setupOnly bool) { + cmd := exec.Command("go", "test", ".", "-count", "1", "-v") + cmd.Dir = dir + cmd.Env = append( + os.Environ(), + fmt.Sprintf("%s=%s", enableEnvName, "true"), + fmt.Sprintf("%s=%s", rootDataDirEnvName, dataDir), + ) + + if setupOnly { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", setupOnlyEnvName, "true")) + } + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +// execClone clones the repo from the given url and branch into the directory. +func execClone(t *testing.T, dir, url, branch string) { + cmd := exec.Command( + "git", + "clone", + "--single-branch", + "--branch", branch, + "--depth", "1", + url, + dir, + ) + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +// execMakeDeps runs make:deps in the given directory. +func execMakeDeps(t *testing.T, dir string) { + cmd := exec.Command("make", "deps:lens") + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, sourceDir, targetDir string) bool { + sourceChanges, ok := getDatabaseFormatDocumentation(t, sourceDir, false) + require.True(t, ok, "Documentation directory not found") + + changes := make(map[string]struct{}, len(sourceChanges)) + for _, f := range sourceChanges { + // Note: we assume flat directory for now - sub directories are not expanded + changes[f.Name()] = struct{}{} + } + + targetChanges, ok := getDatabaseFormatDocumentation(t, targetDir, true) + require.True(t, ok, "Documentation directory not found") + + for _, f := range targetChanges { + if _, isChangeOld := changes[f.Name()]; !isChangeOld { + // If there is a new file in the directory then the change + // has been documented and the test should pass + return true + } + } + + return false +} + +func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend bool) ([]fs.DirEntry, bool) { + startInfo, err := os.Stat(startPath) + require.NoError(t, err) + + var currentDirectory string + if startInfo.IsDir() { + currentDirectory = startPath + } else { + currentDirectory = path.Dir(startPath) + } + + for { + directoryContents, err := os.ReadDir(currentDirectory) + require.NoError(t, err) + + for _, directoryItem := range directoryContents { + directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) + if directoryItem.Name() == documentationDirectoryName { + probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) + require.NoError(t, err) + + for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { + if path.Ext(possibleDocumentationItem.Name()) == ".md" { + // If the directory's name matches the expected, and contains .md files + // we assume it is the documentation directory + return probableFormatChangeDirectoryContents, true + } + } + } else { + if directoryItem.IsDir() { + childContents, directoryFound := getDatabaseFormatDocumentation(t, directoryItemPath, false) + if directoryFound { + return childContents, true + } + } + } + } + + if allowDescend { + // If not found in this directory, continue down the path + currentDirectory = path.Dir(currentDirectory) + require.True(t, currentDirectory != "." && currentDirectory != "/") + } else { + return []fs.DirEntry{}, false + } + } +} diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go new file mode 100644 index 0000000000..4e6e938aa5 --- /dev/null +++ b/tests/change_detector/utils.go @@ -0,0 +1,102 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package change_detector + +import ( + "os" + "path" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +var ( + // Enabled is true when the change detector is running. + Enabled bool + // SetupOnly is true when the change detector is running in setup mode. + SetupOnly bool + // Repository is the url of the repository to run change detector on. + Repository string + // SourceBranch is the name of the source branch to run change detector on. + SourceBranch string + // TargetBranch is the name of the target branch to run change detector on. + TargetBranch string + // rootDatabaseDir is the shared database directory for running tests. + rootDatabaseDir string + // previousTestCaseTestName is the name of the previous test. + previousTestCaseTestName string +) + +const ( + repositoryEnvName = "DEFRA_CHANGE_DETECTOR_REPOSITORY" + sourceBranchEnvName = "DEFRA_CHANGE_DETECTOR_SOURCE_BRANCH" + targetBranchEnvName = "DEFRA_CHANGE_DETECTOR_TARGET_BRANCH" + setupOnlyEnvName = "DEFRA_CHANGE_DETECTOR_SETUP_ONLY" + rootDataDirEnvName = "DEFRA_CHANGE_DETECTOR_ROOT_DATA_DIR" + enableEnvName = "DEFRA_CHANGE_DETECTOR_ENABLE" +) + +const ( + defaultRepository = "https://github.com/sourcenetwork/defradb.git" + defaultSourceBranch = "develop" + documentationDirectoryName = "data_format_changes" +) + +func init() { + Enabled, _ = strconv.ParseBool(os.Getenv(enableEnvName)) + SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) + TargetBranch = os.Getenv(targetBranchEnvName) + rootDatabaseDir = os.Getenv(rootDataDirEnvName) + + if value, ok := os.LookupEnv(repositoryEnvName); ok { + Repository = value + } else { + Repository = defaultRepository + } + + if value, ok := os.LookupEnv(sourceBranchEnvName); ok { + SourceBranch = value + } else { + SourceBranch = defaultSourceBranch + } +} + +// DatabaseDir returns the database directory for change detector test. +func DatabaseDir(t testing.TB) string { + return path.Join(rootDatabaseDir, t.Name()) +} + +// PreTestChecks skips any test that can't be run by the change detector. +func PreTestChecks(t *testing.T, collectionNames []string) { + if !Enabled { + return + } + + if previousTestCaseTestName == t.Name() { + t.Skip("skipping duplicate test") + } + previousTestCaseTestName = t.Name() + + if len(collectionNames) == 0 { + t.Skip("skipping test with no collections") + } + + if SetupOnly { + return + } + + _, err := os.Stat(DatabaseDir(t)) + if os.IsNotExist(err) { + t.Skip("skipping new test package") + } + require.NoError(t, err) +} diff --git a/tests/integration/change_detector.go b/tests/integration/change_detector.go deleted file mode 100644 index 15f17fb16b..0000000000 --- a/tests/integration/change_detector.go +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package tests - -import ( - "context" - "fmt" - "io/fs" - "math/rand" - "os" - "os/exec" - "path" - "runtime" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -var skip bool - -func IsDetectingDbChanges() bool { - return DetectDbChanges -} - -// Returns true if test should pass early -func DetectDbChangesPreTestChecks( - t *testing.T, - collectionNames []string, -) bool { - if skip { - t.SkipNow() - } - - if previousTestCaseTestName == t.Name() { - // The database format changer currently only supports running the first test - // case, if a second case is detected we return early - return true - } - previousTestCaseTestName = t.Name() - - if areDatabaseFormatChangesDocumented { - // If we are checking that database formatting changes have been made and - // documented, and changes are documented, then the tests can all pass. - return true - } - - if len(collectionNames) == 0 { - // If the test doesn't specify any collections, then we can't use it to check - // the database format, so we skip it - t.SkipNow() - } - - if !SetupOnly { - dbDirectory := path.Join(rootDatabaseDir, t.Name()) - _, err := os.Stat(dbDirectory) - if os.IsNotExist(err) { - // This is a new test that does not exist in the target branch, we should - // skip it. - t.SkipNow() - } else { - require.NoError(t, err) - } - } - - return false -} - -func detectDbChangesInit(repository string, targetBranch string) { - badgerFile = true - badgerInMemory = false - - if SetupOnly { - // Only the primary test process should perform the setup below - return - } - - defraTempDir := path.Join(os.TempDir(), "defradb") - changeDetectorTempDir := path.Join(defraTempDir, "tests", "changeDetector") - - latestTargetCommitHash := getLatestCommit(repository, targetBranch) - detectDbChangesCodeDir = path.Join(changeDetectorTempDir, "code", latestTargetCommitHash) - r := rand.New(rand.NewSource(time.Now().Unix())) - randNumber := r.Int() - dbsDir := path.Join(changeDetectorTempDir, "dbs", fmt.Sprint(randNumber)) - - testPackagePath, isIntegrationTest := getTestPackagePath() - if !isIntegrationTest { - skip = true - return - } - rootDatabaseDir = path.Join(dbsDir, strings.ReplaceAll(testPackagePath, "/", "_")) - - _, err := os.Stat(detectDbChangesCodeDir) - // Warning - there is a race condition here, where if running multiple packages in - // parallel (as per default) against a new target commit multiple test pacakges will - // try and clone the target branch at the same time (and will fail). - // This could be solved by using a file lock or similar, however running the change - // detector in parallel is significantly slower than running it serially due to machine - // resource constraints, so I am leaving the race condition in and recommending running - // the change detector with the CLI args `-p 1` - if os.IsNotExist(err) { - cloneCmd := exec.Command( - "git", - "clone", - "-b", - targetBranch, - "--single-branch", - repository, - detectDbChangesCodeDir, - ) - cloneCmd.Stdout = os.Stdout - cloneCmd.Stderr = os.Stderr - err := cloneCmd.Run() - if err != nil { - panic(err) - } - } else if err != nil { - panic(err) - } else { - // Cache must be cleaned, or it might not run the test setup! - // Note: this also acts as a race condition if multiple build are running against the - // same target if this happens some tests might be silently skipped if the - // child-setup fails. Currently I think it is worth it for slightly faster build - // times, but feel very free to change this! - goTestCacheCmd := exec.Command("go", "clean", "-testcache") - goTestCacheCmd.Dir = detectDbChangesCodeDir - err = goTestCacheCmd.Run() - if err != nil { - panic(err) - } - } - - areDatabaseFormatChangesDocumented = checkIfDatabaseFormatChangesAreDocumented() - if areDatabaseFormatChangesDocumented { - // Dont bother doing anything if the changes are documented - return - } - - targetTestPackage := detectDbChangesCodeDir + "/tests/integration/" + testPackagePath - - _, err = os.Stat(targetTestPackage) - if os.IsNotExist(err) { - // This is a new test package, and thus the change detector is not applicable - // as the tests do not exist in the target branch. - skip = true - return - } else if err != nil { - panic(err) - } - - // If we are checking for database changes, and we are not seting up the database, - // then we must be in the main test process, and need to create a new process - // setting up the database for this test using the old branch We should not setup - // the database using the current branch/process - goTestCmd := exec.Command( - "go", - "test", - "./...", - "-v", - ) - - goTestCmd.Dir = targetTestPackage - goTestCmd.Env = os.Environ() - goTestCmd.Env = append( - goTestCmd.Env, - setupOnlyEnvName+"=true", - rootDBFilePathEnvName+"="+rootDatabaseDir, - ) - out, err := goTestCmd.Output() - if err != nil { - log.ErrorE(context.TODO(), string(out), err) - panic(err) - } -} - -// getTestPackagePath returns the path to the package currently under test, relative -// to `./tests/integration/`. Will return an empty string and false if the tests -// are not within that directory. -func getTestPackagePath() (string, bool) { - currentTestPackage, err := os.Getwd() - if err != nil { - panic(err) - } - - splitPath := strings.Split( - currentTestPackage, - "/tests/integration/", - ) - - if len(splitPath) != 2 { - return "", false - } - return splitPath[1], true -} - -func checkIfDatabaseFormatChangesAreDocumented() bool { - previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation( - detectDbChangesCodeDir, - false, - ) - if !targetDirFound { - panic("Documentation directory not found") - } - - previousDbChanges := make(map[string]struct{}, len(previousDbChangeFiles)) - for _, f := range previousDbChangeFiles { - // Note: we assume flat directory for now - sub directories are not expanded - previousDbChanges[f.Name()] = struct{}{} - } - - _, thisFilePath, _, _ := runtime.Caller(0) - currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(thisFilePath, true) - if !currentDirFound { - panic("Documentation directory not found") - } - - for _, f := range currentDbChanges { - if _, isChangeOld := previousDbChanges[f.Name()]; !isChangeOld { - // If there is a new file in the directory then the change - // has been documented and the test should pass - return true - } - } - - return false -} - -func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.DirEntry, bool) { - startInfo, err := os.Stat(startPath) - if err != nil { - panic(err) - } - - var currentDirectory string - if startInfo.IsDir() { - currentDirectory = startPath - } else { - currentDirectory = path.Dir(startPath) - } - - for { - directoryContents, err := os.ReadDir(currentDirectory) - if err != nil { - panic(err) - } - - for _, directoryItem := range directoryContents { - directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) - if directoryItem.Name() == documentationDirectoryName { - probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) - if err != nil { - panic(err) - } - for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { - if path.Ext(possibleDocumentationItem.Name()) == ".md" { - // If the directory's name matches the expected, and contains .md files - // we assume it is the documentation directory - return probableFormatChangeDirectoryContents, true - } - } - } else { - if directoryItem.IsDir() { - childContents, directoryFound := getDatabaseFormatDocumentation(directoryItemPath, false) - if directoryFound { - return childContents, true - } - } - } - } - - if allowDescend { - // If not found in this directory, continue down the path - currentDirectory = path.Dir(currentDirectory) - - if currentDirectory == "." || currentDirectory == "/" { - panic("Database documentation directory not found") - } - } else { - return []fs.DirEntry{}, false - } - } -} - -func getLatestCommit(repoName string, branchName string) string { - cmd := exec.Command("git", "ls-remote", repoName, "refs/heads/"+branchName) - result, err := cmd.Output() - if err != nil { - panic(err) - } - - // This is a tab, not a space! - seperator := "\t" - return strings.Split(string(result), seperator)[0] -} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index cc2a4c4749..f722516445 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -15,7 +15,6 @@ import ( "encoding/json" "fmt" "os" - "path" "reflect" "strconv" "strings" @@ -36,22 +35,17 @@ import ( "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/logging" "github.com/sourcenetwork/defradb/net" + changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" ) const ( - clientGoEnvName = "DEFRA_CLIENT_GO" - clientHttpEnvName = "DEFRA_CLIENT_HTTP" - memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" - fileBadgerEnvName = "DEFRA_BADGER_FILE" - fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" - rootDBFilePathEnvName = "DEFRA_TEST_ROOT" - inMemoryEnvName = "DEFRA_IN_MEMORY" - setupOnlyEnvName = "DEFRA_SETUP_ONLY" - detectDbChangesEnvName = "DEFRA_DETECT_DATABASE_CHANGES" - repositoryEnvName = "DEFRA_CODE_REPOSITORY" - targetBranchEnvName = "DEFRA_TARGET_BRANCH" - mutationTypeEnvName = "DEFRA_MUTATION_TYPE" - documentationDirectoryName = "data_format_changes" + clientGoEnvName = "DEFRA_CLIENT_GO" + clientHttpEnvName = "DEFRA_CLIENT_HTTP" + memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" + fileBadgerEnvName = "DEFRA_BADGER_FILE" + fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" + inMemoryEnvName = "DEFRA_IN_MEMORY" + mutationTypeEnvName = "DEFRA_MUTATION_TYPE" ) type DatabaseType string @@ -108,42 +102,16 @@ var ( httpClient bool goClient bool mutationType MutationType + databaseDir string ) -const subscriptionTimeout = 1 * time.Second - -// Instantiating lenses is expensive, and our tests do not benefit from a large number of them, -// so we explicitly set it to a low value. -const lensPoolSize = 2 - -var databaseDir string -var rootDatabaseDir string - -/* -If this is set to true the integration test suite will instead of its normal profile do -the following: - -On [package] Init: - - Get the (local) latest commit from the target/parent branch // code assumes - git fetch has been done - - Check to see if a clone of that commit/branch is available in the temp dir, and - if not clone the target branch - - Check to see if there are any new .md files in the current branch's data_format_changes - dir (vs the target branch) - -For each test: - - If new documentation detected, pass the test and exit - - Create a new (test/auto-deleted) temp dir for defra to live/run in - - Run the test setup (add initial schema, docs, updates) using the target branch (test is skipped - if test does not exist in target and is new to this branch) - - Run the test request and assert results (as per normal tests) using the current branch -*/ -var DetectDbChanges bool -var SetupOnly bool - -var detectDbChangesCodeDir string -var areDatabaseFormatChangesDocumented bool -var previousTestCaseTestName string +const ( + // subscriptionTimeout is the maximum time to wait for subscription results to be returned. + subscriptionTimeout = 1 * time.Second + // Instantiating lenses is expensive, and our tests do not benefit from a large number of them, + // so we explicitly set it to a low value. + lensPoolSize = 2 +) func init() { // We use environment variables instead of flags `go test ./...` throws for all packages @@ -153,22 +121,6 @@ func init() { badgerFile, _ = strconv.ParseBool(os.Getenv(fileBadgerEnvName)) badgerInMemory, _ = strconv.ParseBool(os.Getenv(memoryBadgerEnvName)) inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) - DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) - SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - - var repositoryValue string - if value, ok := os.LookupEnv(repositoryEnvName); ok { - repositoryValue = value - } else { - repositoryValue = "https://github.com/sourcenetwork/defradb.git" - } - - var targetBranchValue string - if value, ok := os.LookupEnv(targetBranchEnvName); ok { - targetBranchValue = value - } else { - targetBranchValue = "develop" - } if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) @@ -179,21 +131,22 @@ func init() { mutationType = CollectionSaveMutationType } - // Default is to test go client type. if !goClient && !httpClient { + // Default is to test go client type. goClient = true } - // Default is to test all but filesystem db types. - if !badgerInMemory && !badgerFile && !inMemoryStore && !DetectDbChanges { + if changeDetector.Enabled { + // Change detector only uses badger file db type. + badgerFile = true + badgerInMemory = false + inMemoryStore = false + } else if !badgerInMemory && !badgerFile && !inMemoryStore { + // Default is to test all but filesystem db types. badgerFile = false badgerInMemory = true inMemoryStore = true } - - if DetectDbChanges { - detectDbChangesInit(repositoryValue, targetBranchValue) - } } // AssertPanic asserts that the code inside the specified PanicTestFunc panics. @@ -203,7 +156,7 @@ func init() { // // Usage: AssertPanic(t, func() { executeTestCase(t, test) }) func AssertPanic(t *testing.T, f assert.PanicTestFunc) bool { - if IsDetectingDbChanges() { + if changeDetector.Enabled { // The `assert.Panics` call will falsely fail if this test is executed during // a detect changes test run. t.Skip("Assert panic with the change detector is not currently supported.") @@ -218,74 +171,76 @@ func AssertPanic(t *testing.T, f assert.PanicTestFunc) bool { } func NewBadgerMemoryDB(ctx context.Context, dbopts ...db.Option) (client.DB, error) { - opts := badgerds.Options{Options: badger.DefaultOptions("").WithInMemory(true)} + opts := badgerds.Options{ + Options: badger.DefaultOptions("").WithInMemory(true), + } rootstore, err := badgerds.NewDatastore("", &opts) if err != nil { return nil, err } - - dbopts = append(dbopts, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) - db, err := db.NewDB(ctx, rootstore, dbopts...) if err != nil { return nil, err } - return db, nil } -func NewInMemoryDB(ctx context.Context) (client.DB, error) { - rootstore := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, rootstore, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) +func NewInMemoryDB(ctx context.Context, dbopts ...db.Option) (client.DB, error) { + db, err := db.NewDB(ctx, memory.NewDatastore(ctx), dbopts...) if err != nil { return nil, err } - return db, nil } -func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, error) { +func NewBadgerFileDB(ctx context.Context, t testing.TB, dbopts ...db.Option) (client.DB, string, error) { var dbPath string - if databaseDir != "" { + switch { + case databaseDir != "": + // restarting database dbPath = databaseDir - } else if rootDatabaseDir != "" { - dbPath = path.Join(rootDatabaseDir, t.Name()) - } else { + + case changeDetector.Enabled: + // change detector + dbPath = changeDetector.DatabaseDir(t) + + default: + // default test case dbPath = t.TempDir() } - db, err := newBadgerFileDB(ctx, t, dbPath) - return db, dbPath, err -} - -func newBadgerFileDB(ctx context.Context, t testing.TB, path string) (client.DB, error) { - opts := badgerds.Options{Options: badger.DefaultOptions(path)} - rootstore, err := badgerds.NewDatastore(path, &opts) + opts := &badgerds.Options{ + Options: badger.DefaultOptions(dbPath), + } + rootstore, err := badgerds.NewDatastore(dbPath, opts) if err != nil { - return nil, err + return nil, "", err } - - db, err := db.NewDB(ctx, rootstore, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) + db, err := db.NewDB(ctx, rootstore, dbopts...) if err != nil { - return nil, err + return nil, "", err } - - return db, nil + return db, dbPath, err } // GetDatabase returns the database implementation for the current // testing state. The database type and client type on the test state // are used to select the datastore and client implementation to use. func GetDatabase(s *state) (cdb client.DB, path string, err error) { + dbopts := []db.Option{ + db.WithUpdateEvents(), + db.WithLensPoolSize(lensPoolSize), + } + switch s.dbt { case badgerIMType: - cdb, err = NewBadgerMemoryDB(s.ctx, db.WithUpdateEvents()) + cdb, err = NewBadgerMemoryDB(s.ctx, dbopts...) case badgerFileType: - cdb, path, err = NewBadgerFileDB(s.ctx, s.t) + cdb, path, err = NewBadgerFileDB(s.ctx, s.t, dbopts...) case defraIMType: - cdb, err = NewInMemoryDB(s.ctx) + cdb, err = NewInMemoryDB(s.ctx, dbopts...) default: err = fmt.Errorf("invalid database type: %v", s.dbt) @@ -323,11 +278,7 @@ func ExecuteTestCase( testCase TestCase, ) { collectionNames := getCollectionNames(testCase) - - if DetectDbChanges && DetectDbChangesPreTestChecks(t, collectionNames) { - return - } - + changeDetector.PreTestChecks(t, collectionNames) skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) var clients []ClientType @@ -370,7 +321,22 @@ func executeTestCase( dbt DatabaseType, clientType ClientType, ) { - log.Info(ctx, testCase.Description, logging.NewKV("Database", dbt)) + log.Info( + ctx, + testCase.Description, + logging.NewKV("badgerFile", badgerFile), + logging.NewKV("badgerInMemory", badgerInMemory), + logging.NewKV("inMemoryStore", inMemoryStore), + logging.NewKV("httpClient", httpClient), + logging.NewKV("goClient", goClient), + logging.NewKV("mutationType", mutationType), + logging.NewKV("databaseDir", databaseDir), + logging.NewKV("changeDetector.Enabled", changeDetector.Enabled), + logging.NewKV("changeDetector.SetupOnly", changeDetector.SetupOnly), + logging.NewKV("changeDetector.SourceBranch", changeDetector.SourceBranch), + logging.NewKV("changeDetector.TargetBranch", changeDetector.TargetBranch), + logging.NewKV("changeDetector.Repository", changeDetector.Repository), + ) flattenActions(&testCase) startActionIndex, endActionIndex := getActionRange(testCase) @@ -632,7 +598,7 @@ func getActionRange(testCase TestCase) (int, int) { startIndex := 0 endIndex := len(testCase.Actions) - 1 - if !DetectDbChanges { + if !changeDetector.Enabled { return startIndex, endIndex } @@ -656,7 +622,7 @@ ActionLoop: } } - if SetupOnly { + if changeDetector.SetupOnly { if setupCompleteIndex > -1 { endIndex = setupCompleteIndex } else if firstNonSetupIndex > -1 { @@ -825,7 +791,7 @@ func configureNode( s *state, action ConfigureNode, ) { - if DetectDbChanges { + if changeDetector.Enabled { // We do not yet support the change detector for tests running across multiple nodes. s.t.SkipNow() return