From 605f42805254613e4e59d6f554baef7c76ad7784 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 21:52:21 -0700 Subject: [PATCH 01/21] move change detector logic to tests/change_detector so it can run tests in parallel --- tests/change_detector/change_detector_test.go | 147 +++++++++ tests/integration/change_detector.go | 304 ------------------ tests/integration/utils2.go | 41 +-- 3 files changed, 156 insertions(+), 336 deletions(-) create mode 100644 tests/change_detector/change_detector_test.go delete mode 100644 tests/integration/change_detector.go diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go new file mode 100644 index 0000000000..9767a03796 --- /dev/null +++ b/tests/change_detector/change_detector_test.go @@ -0,0 +1,147 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package change_detector + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + repositoryEnvName = "DEFRA_CODE_REPOSITORY" + sourceBranchEnvName = "DEFRA_SOURCE_BRANCH" + targetBranchEnvName = "DEFRA_TARGET_BRANCH" +) + +func TestChanges(t *testing.T) { + var repository string + if value, ok := os.LookupEnv(repositoryEnvName); ok { + repository = value + } else { + repository = "https://github.com/sourcenetwork/defradb.git" + } + + var sourceBranch string + if value, ok := os.LookupEnv(sourceBranchEnvName); ok { + sourceBranch = value + } else { + sourceBranch = "develop" + } + + var targetBranch string + if value, ok := os.LookupEnv(targetBranchEnvName); ok { + targetBranch = value + } else { + targetBranch = "develop" + } + + sourceRepoDir := t.TempDir() + execClone(t, sourceRepoDir, repository, sourceBranch) + + targetRepoDir := t.TempDir() + execClone(t, targetRepoDir, repository, targetBranch) + + execMakeDeps(t, sourceRepoDir) + execMakeDeps(t, targetRepoDir) + + targetRepoTestDir := filepath.Join(targetRepoDir, "tests", "integration") + targetRepoPkgList := execList(t, targetRepoTestDir) + + sourceRepoTestDir := filepath.Join(sourceRepoDir, "tests", "integration") + sourceRepoPkgList := execList(t, sourceRepoTestDir) + + sourceRepoPkgMap := make(map[string]bool) + for _, pkg := range sourceRepoPkgList { + sourceRepoPkgMap[pkg] = true + } + + for _, pkg := range targetRepoPkgList { + if pkg == "" || !sourceRepoPkgMap[pkg] { + continue + } + pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") + + t.Run(pkgName, func(t *testing.T) { + t.Parallel() + dataDir := t.TempDir() + + fromTestPkg := filepath.Join(sourceRepoDir, pkgName) + execTest(t, dataDir, fromTestPkg, true) + + toTestPkg := filepath.Join(targetRepoDir, pkgName) + execTest(t, dataDir, toTestPkg, false) + }) + } +} + +func execList(t *testing.T, dir string) []string { + cmd := exec.Command( + "go", + "list", + "./...", + ) + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err, string(out)) + + return strings.Split(string(out), "\n") +} + +func execTest(t *testing.T, dir, pkg string, setupOnly bool) { + cmd := exec.Command( + "go", + "test", + ".", + "-v", + ) + cmd.Dir = pkg + cmd.Env = append(os.Environ(), "DEFRA_TEST_ROOT="+dir) + + if setupOnly { + cmd.Env = append(cmd.Env, "DEFRA_SETUP_ONLY=true") + } else { + cmd.Env = append(cmd.Env, "DEFRA_DETECT_DATABASE_CHANGES=true") + } + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +func execClone(t *testing.T, dir, url, branch string) { + cmd := exec.Command( + "git", + "clone", + "--branch", branch, + "--single-branch", + url, + dir, + ) + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} + +func execMakeDeps(t *testing.T, dir string) { + cmd := exec.Command( + "make", + "deps:lens", + ) + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err, string(out)) +} diff --git a/tests/integration/change_detector.go b/tests/integration/change_detector.go deleted file mode 100644 index 15f17fb16b..0000000000 --- a/tests/integration/change_detector.go +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package tests - -import ( - "context" - "fmt" - "io/fs" - "math/rand" - "os" - "os/exec" - "path" - "runtime" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -var skip bool - -func IsDetectingDbChanges() bool { - return DetectDbChanges -} - -// Returns true if test should pass early -func DetectDbChangesPreTestChecks( - t *testing.T, - collectionNames []string, -) bool { - if skip { - t.SkipNow() - } - - if previousTestCaseTestName == t.Name() { - // The database format changer currently only supports running the first test - // case, if a second case is detected we return early - return true - } - previousTestCaseTestName = t.Name() - - if areDatabaseFormatChangesDocumented { - // If we are checking that database formatting changes have been made and - // documented, and changes are documented, then the tests can all pass. - return true - } - - if len(collectionNames) == 0 { - // If the test doesn't specify any collections, then we can't use it to check - // the database format, so we skip it - t.SkipNow() - } - - if !SetupOnly { - dbDirectory := path.Join(rootDatabaseDir, t.Name()) - _, err := os.Stat(dbDirectory) - if os.IsNotExist(err) { - // This is a new test that does not exist in the target branch, we should - // skip it. - t.SkipNow() - } else { - require.NoError(t, err) - } - } - - return false -} - -func detectDbChangesInit(repository string, targetBranch string) { - badgerFile = true - badgerInMemory = false - - if SetupOnly { - // Only the primary test process should perform the setup below - return - } - - defraTempDir := path.Join(os.TempDir(), "defradb") - changeDetectorTempDir := path.Join(defraTempDir, "tests", "changeDetector") - - latestTargetCommitHash := getLatestCommit(repository, targetBranch) - detectDbChangesCodeDir = path.Join(changeDetectorTempDir, "code", latestTargetCommitHash) - r := rand.New(rand.NewSource(time.Now().Unix())) - randNumber := r.Int() - dbsDir := path.Join(changeDetectorTempDir, "dbs", fmt.Sprint(randNumber)) - - testPackagePath, isIntegrationTest := getTestPackagePath() - if !isIntegrationTest { - skip = true - return - } - rootDatabaseDir = path.Join(dbsDir, strings.ReplaceAll(testPackagePath, "/", "_")) - - _, err := os.Stat(detectDbChangesCodeDir) - // Warning - there is a race condition here, where if running multiple packages in - // parallel (as per default) against a new target commit multiple test pacakges will - // try and clone the target branch at the same time (and will fail). - // This could be solved by using a file lock or similar, however running the change - // detector in parallel is significantly slower than running it serially due to machine - // resource constraints, so I am leaving the race condition in and recommending running - // the change detector with the CLI args `-p 1` - if os.IsNotExist(err) { - cloneCmd := exec.Command( - "git", - "clone", - "-b", - targetBranch, - "--single-branch", - repository, - detectDbChangesCodeDir, - ) - cloneCmd.Stdout = os.Stdout - cloneCmd.Stderr = os.Stderr - err := cloneCmd.Run() - if err != nil { - panic(err) - } - } else if err != nil { - panic(err) - } else { - // Cache must be cleaned, or it might not run the test setup! - // Note: this also acts as a race condition if multiple build are running against the - // same target if this happens some tests might be silently skipped if the - // child-setup fails. Currently I think it is worth it for slightly faster build - // times, but feel very free to change this! - goTestCacheCmd := exec.Command("go", "clean", "-testcache") - goTestCacheCmd.Dir = detectDbChangesCodeDir - err = goTestCacheCmd.Run() - if err != nil { - panic(err) - } - } - - areDatabaseFormatChangesDocumented = checkIfDatabaseFormatChangesAreDocumented() - if areDatabaseFormatChangesDocumented { - // Dont bother doing anything if the changes are documented - return - } - - targetTestPackage := detectDbChangesCodeDir + "/tests/integration/" + testPackagePath - - _, err = os.Stat(targetTestPackage) - if os.IsNotExist(err) { - // This is a new test package, and thus the change detector is not applicable - // as the tests do not exist in the target branch. - skip = true - return - } else if err != nil { - panic(err) - } - - // If we are checking for database changes, and we are not seting up the database, - // then we must be in the main test process, and need to create a new process - // setting up the database for this test using the old branch We should not setup - // the database using the current branch/process - goTestCmd := exec.Command( - "go", - "test", - "./...", - "-v", - ) - - goTestCmd.Dir = targetTestPackage - goTestCmd.Env = os.Environ() - goTestCmd.Env = append( - goTestCmd.Env, - setupOnlyEnvName+"=true", - rootDBFilePathEnvName+"="+rootDatabaseDir, - ) - out, err := goTestCmd.Output() - if err != nil { - log.ErrorE(context.TODO(), string(out), err) - panic(err) - } -} - -// getTestPackagePath returns the path to the package currently under test, relative -// to `./tests/integration/`. Will return an empty string and false if the tests -// are not within that directory. -func getTestPackagePath() (string, bool) { - currentTestPackage, err := os.Getwd() - if err != nil { - panic(err) - } - - splitPath := strings.Split( - currentTestPackage, - "/tests/integration/", - ) - - if len(splitPath) != 2 { - return "", false - } - return splitPath[1], true -} - -func checkIfDatabaseFormatChangesAreDocumented() bool { - previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation( - detectDbChangesCodeDir, - false, - ) - if !targetDirFound { - panic("Documentation directory not found") - } - - previousDbChanges := make(map[string]struct{}, len(previousDbChangeFiles)) - for _, f := range previousDbChangeFiles { - // Note: we assume flat directory for now - sub directories are not expanded - previousDbChanges[f.Name()] = struct{}{} - } - - _, thisFilePath, _, _ := runtime.Caller(0) - currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(thisFilePath, true) - if !currentDirFound { - panic("Documentation directory not found") - } - - for _, f := range currentDbChanges { - if _, isChangeOld := previousDbChanges[f.Name()]; !isChangeOld { - // If there is a new file in the directory then the change - // has been documented and the test should pass - return true - } - } - - return false -} - -func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.DirEntry, bool) { - startInfo, err := os.Stat(startPath) - if err != nil { - panic(err) - } - - var currentDirectory string - if startInfo.IsDir() { - currentDirectory = startPath - } else { - currentDirectory = path.Dir(startPath) - } - - for { - directoryContents, err := os.ReadDir(currentDirectory) - if err != nil { - panic(err) - } - - for _, directoryItem := range directoryContents { - directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) - if directoryItem.Name() == documentationDirectoryName { - probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) - if err != nil { - panic(err) - } - for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { - if path.Ext(possibleDocumentationItem.Name()) == ".md" { - // If the directory's name matches the expected, and contains .md files - // we assume it is the documentation directory - return probableFormatChangeDirectoryContents, true - } - } - } else { - if directoryItem.IsDir() { - childContents, directoryFound := getDatabaseFormatDocumentation(directoryItemPath, false) - if directoryFound { - return childContents, true - } - } - } - } - - if allowDescend { - // If not found in this directory, continue down the path - currentDirectory = path.Dir(currentDirectory) - - if currentDirectory == "." || currentDirectory == "/" { - panic("Database documentation directory not found") - } - } else { - return []fs.DirEntry{}, false - } - } -} - -func getLatestCommit(repoName string, branchName string) string { - cmd := exec.Command("git", "ls-remote", repoName, "refs/heads/"+branchName) - result, err := cmd.Output() - if err != nil { - panic(err) - } - - // This is a tab, not a space! - seperator := "\t" - return strings.Split(string(result), seperator)[0] -} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index ab2bea2898..770418039a 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -48,8 +48,6 @@ const ( inMemoryEnvName = "DEFRA_IN_MEMORY" setupOnlyEnvName = "DEFRA_SETUP_ONLY" detectDbChangesEnvName = "DEFRA_DETECT_DATABASE_CHANGES" - repositoryEnvName = "DEFRA_CODE_REPOSITORY" - targetBranchEnvName = "DEFRA_TARGET_BRANCH" mutationTypeEnvName = "DEFRA_MUTATION_TYPE" documentationDirectoryName = "data_format_changes" ) @@ -141,10 +139,6 @@ For each test: var DetectDbChanges bool var SetupOnly bool -var detectDbChangesCodeDir string -var areDatabaseFormatChangesDocumented bool -var previousTestCaseTestName string - func init() { // We use environment variables instead of flags `go test ./...` throws for all packages // that don't have the flag defined @@ -156,20 +150,6 @@ func init() { DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - var repositoryValue string - if value, ok := os.LookupEnv(repositoryEnvName); ok { - repositoryValue = value - } else { - repositoryValue = "https://github.com/sourcenetwork/defradb.git" - } - - var targetBranchValue string - if value, ok := os.LookupEnv(targetBranchEnvName); ok { - targetBranchValue = value - } else { - targetBranchValue = "develop" - } - if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) } else { @@ -179,21 +159,22 @@ func init() { mutationType = CollectionSaveMutationType } - // Default is to test go client type. if !goClient && !httpClient { + // Default is to test go client type. goClient = true } - // Default is to test all but filesystem db types. - if !badgerInMemory && !badgerFile && !inMemoryStore && !DetectDbChanges { + if DetectDbChanges { + // Change detector only uses badger file db type. + badgerFile = true + badgerInMemory = false + inMemoryStore = false + } else if !badgerInMemory && !badgerFile && !inMemoryStore { + // Default is to test all but filesystem db types. badgerFile = false badgerInMemory = true inMemoryStore = true } - - if DetectDbChanges { - detectDbChangesInit(repositoryValue, targetBranchValue) - } } // AssertPanicAndSkipChangeDetection asserts that the code of function actually panics, @@ -202,7 +183,7 @@ func init() { // // Usage: AssertPanicAndSkipChangeDetection(t, func() { executeTestCase(t, test) }) func AssertPanicAndSkipChangeDetection(t *testing.T, f assert.PanicTestFunc) bool { - if IsDetectingDbChanges() { + if DetectDbChanges { // The `assert.Panics` call will falsely fail if this test is executed during // a detect changes test run t.Skip() @@ -317,10 +298,6 @@ func ExecuteTestCase( ) { collectionNames := getCollectionNames(testCase) - if DetectDbChanges && DetectDbChangesPreTestChecks(t, collectionNames) { - return - } - skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) var clients []ClientType From de1a1eb4d68aaca8570854ffa9d78d95fa72f4a7 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 21:59:19 -0700 Subject: [PATCH 02/21] set rootDatabaseDir from env variable --- tests/change_detector/change_detector_test.go | 6 +++--- tests/integration/utils2.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 9767a03796..95f2ca8d78 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -31,21 +31,21 @@ func TestChanges(t *testing.T) { if value, ok := os.LookupEnv(repositoryEnvName); ok { repository = value } else { - repository = "https://github.com/sourcenetwork/defradb.git" + repository = "https://github.com/nasdf/defradb.git" } var sourceBranch string if value, ok := os.LookupEnv(sourceBranchEnvName); ok { sourceBranch = value } else { - sourceBranch = "develop" + sourceBranch = "nasdf/test/parallel-change-detector" } var targetBranch string if value, ok := os.LookupEnv(targetBranchEnvName); ok { targetBranch = value } else { - targetBranch = "develop" + targetBranch = "nasdf/test/parallel-change-detector" } sourceRepoDir := t.TempDir() diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 770418039a..332f50de6a 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -149,6 +149,7 @@ func init() { inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) + rootDatabaseDir = os.Getenv(rootDBFilePathEnvName) if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) From 5b389cbb96ab90875d576f3e4b1f08d1958cec68 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 22:04:55 -0700 Subject: [PATCH 03/21] set databaseDir instead of rootDatabaseDir --- tests/integration/utils2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 332f50de6a..5c6e632cc5 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -149,7 +149,7 @@ func init() { inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - rootDatabaseDir = os.Getenv(rootDBFilePathEnvName) + databaseDir = os.Getenv(rootDBFilePathEnvName) if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) From f6bb25b1c0526ece8bef652e8224d32543d2aa40 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 22:25:56 -0700 Subject: [PATCH 04/21] cleanup database file path unused variables --- tests/change_detector/change_detector_test.go | 2 +- tests/integration/utils2.go | 40 +++++-------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 95f2ca8d78..5e5369825d 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -109,7 +109,7 @@ func execTest(t *testing.T, dir, pkg string, setupOnly bool) { "-v", ) cmd.Dir = pkg - cmd.Env = append(os.Environ(), "DEFRA_TEST_ROOT="+dir) + cmd.Env = append(os.Environ(), "DEFRA_BADGER_FILE_PATH="+dir) if setupOnly { cmd.Env = append(cmd.Env, "DEFRA_SETUP_ONLY=true") diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 5c6e632cc5..4b04a94e25 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -15,7 +15,6 @@ import ( "encoding/json" "fmt" "os" - "path" "reflect" "strconv" "strings" @@ -44,7 +43,6 @@ const ( memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" fileBadgerEnvName = "DEFRA_BADGER_FILE" fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" - rootDBFilePathEnvName = "DEFRA_TEST_ROOT" inMemoryEnvName = "DEFRA_IN_MEMORY" setupOnlyEnvName = "DEFRA_SETUP_ONLY" detectDbChangesEnvName = "DEFRA_DETECT_DATABASE_CHANGES" @@ -102,6 +100,7 @@ var ( log = logging.MustNewLogger("tests.integration") badgerInMemory bool badgerFile bool + badgerFilePath string inMemoryStore bool httpClient bool goClient bool @@ -114,9 +113,6 @@ const subscriptionTimeout = 1 * time.Second // so we explicitly set it to a low value. const lensPoolSize = 2 -var databaseDir string -var rootDatabaseDir string - /* If this is set to true the integration test suite will instead of its normal profile do the following: @@ -149,7 +145,7 @@ func init() { inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - databaseDir = os.Getenv(rootDBFilePathEnvName) + badgerFilePath = os.Getenv(fileBadgerPathEnvName) if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) @@ -220,32 +216,21 @@ func NewInMemoryDB(ctx context.Context) (client.DB, error) { } func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, error) { - var dbPath string - if databaseDir != "" { - dbPath = databaseDir - } else if rootDatabaseDir != "" { - dbPath = path.Join(rootDatabaseDir, t.Name()) - } else { - dbPath = t.TempDir() + if badgerFilePath != "" { + badgerFilePath = t.TempDir() } - - db, err := newBadgerFileDB(ctx, t, dbPath) - return db, dbPath, err -} - -func newBadgerFileDB(ctx context.Context, t testing.TB, path string) (client.DB, error) { - opts := badgerds.Options{Options: badger.DefaultOptions(path)} - rootstore, err := badgerds.NewDatastore(path, &opts) + opts := &badgerds.Options{ + Options: badger.DefaultOptions(badgerFilePath), + } + rootstore, err := badgerds.NewDatastore(badgerFilePath, opts) if err != nil { - return nil, err + return nil, "", err } - db, err := db.NewDB(ctx, rootstore, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) if err != nil { - return nil, err + return nil, "", err } - - return db, nil + return db, badgerFilePath, err } // GetDatabase returns the database implementation for the current @@ -688,11 +673,8 @@ func restartNodes( // We need to restart the nodes in reverse order, to avoid dial backoff issues. for i := len(s.nodes) - 1; i >= 0; i-- { - originalPath := databaseDir - databaseDir = s.dbPaths[i] db, _, err := GetDatabase(s) require.Nil(s.t, err) - databaseDir = originalPath if len(s.nodeConfigs) == 0 { // If there are no explicit node configuration actions the node will be From dc5aaf8c0d96a74a70cf8a3096c2dd5aea474742 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 22:28:27 -0700 Subject: [PATCH 05/21] set count flag in go test command --- tests/change_detector/change_detector_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 5e5369825d..6d60795a64 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -106,6 +106,7 @@ func execTest(t *testing.T, dir, pkg string, setupOnly bool) { "go", "test", ".", + "-count", "1", "-v", ) cmd.Dir = pkg From 6ab73712f84798d572d0638abc8048718674c6a0 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 11 Sep 2023 22:36:06 -0700 Subject: [PATCH 06/21] always set detect changes env --- tests/change_detector/change_detector_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 6d60795a64..75a3edd791 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -75,7 +75,7 @@ func TestChanges(t *testing.T) { pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") t.Run(pkgName, func(t *testing.T) { - t.Parallel() + //t.Parallel() dataDir := t.TempDir() fromTestPkg := filepath.Join(sourceRepoDir, pkgName) @@ -111,11 +111,10 @@ func execTest(t *testing.T, dir, pkg string, setupOnly bool) { ) cmd.Dir = pkg cmd.Env = append(os.Environ(), "DEFRA_BADGER_FILE_PATH="+dir) + cmd.Env = append(cmd.Env, "DEFRA_DETECT_DATABASE_CHANGES=true") if setupOnly { cmd.Env = append(cmd.Env, "DEFRA_SETUP_ONLY=true") - } else { - cmd.Env = append(cmd.Env, "DEFRA_DETECT_DATABASE_CHANGES=true") } out, err := cmd.Output() From 81952427c3c0d95f0540adf18ca7e2875ca607f9 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 07:55:20 -0700 Subject: [PATCH 07/21] fix bug in badger file path --- tests/integration/utils2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 4b04a94e25..9fbc606f2b 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -216,7 +216,7 @@ func NewInMemoryDB(ctx context.Context) (client.DB, error) { } func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, error) { - if badgerFilePath != "" { + if badgerFilePath == "" { badgerFilePath = t.TempDir() } opts := &badgerds.Options{ From 01afa55d863b40e63b18764781d1a70d74253536 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 08:30:07 -0700 Subject: [PATCH 08/21] restore preflight checks and db path logic --- tests/integration/change_detector.go | 39 ++++++++++++++++++++++++++++ tests/integration/utils2.go | 33 ++++++++++++++++++----- 2 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 tests/integration/change_detector.go diff --git a/tests/integration/change_detector.go b/tests/integration/change_detector.go new file mode 100644 index 0000000000..a2077fc273 --- /dev/null +++ b/tests/integration/change_detector.go @@ -0,0 +1,39 @@ +package tests + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func DetectDbChangesPreTestChecks( + t *testing.T, + collectionNames []string, +) { + if previousTestCaseTestName == t.Name() { + // The database format changer currently only supports running the first test + // case, if a second case is detected we return early + t.Skip() + } + previousTestCaseTestName = t.Name() + + if len(collectionNames) == 0 { + // If the test doesn't specify any collections, then we can't use it to check + // the database format, so we skip it + t.SkipNow() + } + + if !SetupOnly { + dbDirectory := path.Join(rootDatabaseDir, t.Name()) + _, err := os.Stat(dbDirectory) + if os.IsNotExist(err) { + // This is a new test that does not exist in the target branch, we should + // skip it. + t.SkipNow() + } else { + require.NoError(t, err) + } + } +} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 9fbc606f2b..8f29ab4d6d 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -15,6 +15,7 @@ import ( "encoding/json" "fmt" "os" + "path" "reflect" "strconv" "strings" @@ -43,6 +44,7 @@ const ( memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" fileBadgerEnvName = "DEFRA_BADGER_FILE" fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" + rootDBFilePathEnvName = "DEFRA_TEST_ROOT" inMemoryEnvName = "DEFRA_IN_MEMORY" setupOnlyEnvName = "DEFRA_SETUP_ONLY" detectDbChangesEnvName = "DEFRA_DETECT_DATABASE_CHANGES" @@ -100,11 +102,16 @@ var ( log = logging.MustNewLogger("tests.integration") badgerInMemory bool badgerFile bool - badgerFilePath string inMemoryStore bool httpClient bool goClient bool mutationType MutationType + // databaseDir is the directory of the badger file db. + databaseDir string + // rootDatabaseDir is the root directroy of the badger file db. + rootDatabaseDir string + // previousTestCaseName is the name of the previous test. + previousTestCaseTestName string ) const subscriptionTimeout = 1 * time.Second @@ -145,7 +152,7 @@ func init() { inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - badgerFilePath = os.Getenv(fileBadgerPathEnvName) + rootDatabaseDir = os.Getenv(rootDBFilePathEnvName) if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) @@ -216,13 +223,18 @@ func NewInMemoryDB(ctx context.Context) (client.DB, error) { } func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, error) { - if badgerFilePath == "" { - badgerFilePath = t.TempDir() + var dbPath string + if databaseDir != "" { + dbPath = databaseDir + } else if rootDatabaseDir != "" { + dbPath = path.Join(rootDatabaseDir, t.Name()) + } else { + dbPath = t.TempDir() } opts := &badgerds.Options{ - Options: badger.DefaultOptions(badgerFilePath), + Options: badger.DefaultOptions(dbPath), } - rootstore, err := badgerds.NewDatastore(badgerFilePath, opts) + rootstore, err := badgerds.NewDatastore(dbPath, opts) if err != nil { return nil, "", err } @@ -230,7 +242,7 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, erro if err != nil { return nil, "", err } - return db, badgerFilePath, err + return db, dbPath, err } // GetDatabase returns the database implementation for the current @@ -284,6 +296,10 @@ func ExecuteTestCase( ) { collectionNames := getCollectionNames(testCase) + if DetectDbChanges { + DetectDbChangesPreTestChecks(t, collectionNames) + } + skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) var clients []ClientType @@ -673,8 +689,11 @@ func restartNodes( // We need to restart the nodes in reverse order, to avoid dial backoff issues. for i := len(s.nodes) - 1; i >= 0; i-- { + originalPath := databaseDir + databaseDir = s.dbPaths[i] db, _, err := GetDatabase(s) require.Nil(s.t, err) + databaseDir = originalPath if len(s.nodeConfigs) == 0 { // If there are no explicit node configuration actions the node will be From 4b9c36ec477e7b500040bf4aa41c753d92c55b3b Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 08:37:35 -0700 Subject: [PATCH 09/21] enable parallel change detector --- tests/change_detector/change_detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 75a3edd791..1ece69561d 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -75,7 +75,7 @@ func TestChanges(t *testing.T) { pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") t.Run(pkgName, func(t *testing.T) { - //t.Parallel() + t.Parallel() dataDir := t.TempDir() fromTestPkg := filepath.Join(sourceRepoDir, pkgName) From 9ec616a36cf2a10be3aae8defbea2015d1dc0bc8 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 09:16:40 -0700 Subject: [PATCH 10/21] add more documentation --- tests/change_detector/change_detector_test.go | 39 ++++++++----------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 1ece69561d..9009531d60 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -70,7 +70,7 @@ func TestChanges(t *testing.T) { for _, pkg := range targetRepoPkgList { if pkg == "" || !sourceRepoPkgMap[pkg] { - continue + continue // ignore new packages } pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") @@ -79,20 +79,17 @@ func TestChanges(t *testing.T) { dataDir := t.TempDir() fromTestPkg := filepath.Join(sourceRepoDir, pkgName) - execTest(t, dataDir, fromTestPkg, true) + execTest(t, fromTestPkg, dataDir, true) toTestPkg := filepath.Join(targetRepoDir, pkgName) - execTest(t, dataDir, toTestPkg, false) + execTest(t, toTestPkg, dataDir, false) }) } } +// execList returns a list of all packages in the given directory. func execList(t *testing.T, dir string) []string { - cmd := exec.Command( - "go", - "list", - "./...", - ) + cmd := exec.Command("go", "list", "./...") cmd.Dir = dir out, err := cmd.Output() @@ -101,16 +98,12 @@ func execList(t *testing.T, dir string) []string { return strings.Split(string(out), "\n") } -func execTest(t *testing.T, dir, pkg string, setupOnly bool) { - cmd := exec.Command( - "go", - "test", - ".", - "-count", "1", - "-v", - ) - cmd.Dir = pkg - cmd.Env = append(os.Environ(), "DEFRA_BADGER_FILE_PATH="+dir) +// execTest runs the tests in the given directory and sets the data +// directory and setup only environment variables. +func execTest(t *testing.T, dir, dataDir string, setupOnly bool) { + cmd := exec.Command("go", "test", ".", "-count", "1", "-v") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "DEFRA_BADGER_FILE_PATH="+dataDir) cmd.Env = append(cmd.Env, "DEFRA_DETECT_DATABASE_CHANGES=true") if setupOnly { @@ -121,12 +114,14 @@ func execTest(t *testing.T, dir, pkg string, setupOnly bool) { require.NoError(t, err, string(out)) } +// execClone clones the repo from the given url and branch into the directory. func execClone(t *testing.T, dir, url, branch string) { cmd := exec.Command( "git", "clone", - "--branch", branch, "--single-branch", + "--branch", branch, + "--depth", "1", url, dir, ) @@ -135,11 +130,9 @@ func execClone(t *testing.T, dir, url, branch string) { require.NoError(t, err, string(out)) } +// execMakeDeps runs make:deps in the given directory. func execMakeDeps(t *testing.T, dir string) { - cmd := exec.Command( - "make", - "deps:lens", - ) + cmd := exec.Command("make", "deps:lens") cmd.Dir = dir out, err := cmd.Output() From 3b84d9fa0c65249730397962480050755872600e Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 09:27:47 -0700 Subject: [PATCH 11/21] update Makefile test targets --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6eb3456fcc..ca169902bc 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,8 @@ TEST_FLAGS=-race -shuffle=on -timeout 300s PLAYGROUND_DIRECTORY=playground LENS_TEST_DIRECTORY=tests/integration/schema/migrations CLI_TEST_DIRECTORY=tests/integration/cli -DEFAULT_TEST_DIRECTORIES=$$(go list ./... | grep -v -e $(LENS_TEST_DIRECTORY) -e $(CLI_TEST_DIRECTORY)) +CHANGE_DETECTOR_TEST_DIRECTORY=tests/change_detector +DEFAULT_TEST_DIRECTORIES=$$(go list ./... | grep -v -e $(LENS_TEST_DIRECTORY) -e $(CLI_TEST_DIRECTORY) -e $(CHANGE_DETECTOR_TEST_DIRECTORY)) default: @go run $(BUILD_FLAGS) cmd/defradb/main.go @@ -294,8 +295,7 @@ test\:coverage-html: .PHONY: test\:changes test\:changes: - @$(MAKE) deps:lens - env DEFRA_DETECT_DATABASE_CHANGES=true DEFRA_CLIENT_GO=true gotestsum -- ./... -shuffle=on -p 1 + gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... $(TEST_FLAGS) .PHONY: validate\:codecov validate\:codecov: From 536f6fee4217d612deece77f2f53af2f933b1591 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 11:26:36 -0700 Subject: [PATCH 12/21] move all change detector logic to new package. clean up env names and init logic. add data format documentation checks --- tests/change_detector/README.md | 15 ++ tests/change_detector/change_detector_test.go | 57 ++--- tests/change_detector/utils.go | 194 ++++++++++++++++++ tests/integration/change_detector.go | 39 ---- tests/integration/utils2.go | 124 +++++------ 5 files changed, 276 insertions(+), 153 deletions(-) create mode 100644 tests/change_detector/README.md create mode 100644 tests/change_detector/utils.go delete mode 100644 tests/integration/change_detector.go diff --git a/tests/change_detector/README.md b/tests/change_detector/README.md new file mode 100644 index 0000000000..4d824fb60f --- /dev/null +++ b/tests/change_detector/README.md @@ -0,0 +1,15 @@ +# Change Detector + +The change detector is used to detect data format changes between versions of DefraDB. + +## How it works + +The tests run using a `source` and `target` branch of DefraDB. Each branch is cloned into a temporary directory and dependencies are installed. + +The test runner executes all of the common test packages available in the `source` and `target` tests directory. + +For each test package execution the following steps occur: + +- Create a temporary data directory. This is used to share data between `source` and `target`. +- Run the `source` version in setup only mode. This creates test fixtures in the shared data directory. +- Run the `target` version in change detector mode. This skips the setup and executes the tests using the shared data directory. diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 9009531d60..61b6f58c46 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -11,6 +11,7 @@ package change_detector import ( + "fmt" "os" "os/exec" "path/filepath" @@ -20,37 +21,14 @@ import ( "github.com/stretchr/testify/require" ) -const ( - repositoryEnvName = "DEFRA_CODE_REPOSITORY" - sourceBranchEnvName = "DEFRA_SOURCE_BRANCH" - targetBranchEnvName = "DEFRA_TARGET_BRANCH" -) - func TestChanges(t *testing.T) { - var repository string - if value, ok := os.LookupEnv(repositoryEnvName); ok { - repository = value - } else { - repository = "https://github.com/nasdf/defradb.git" - } - - var sourceBranch string - if value, ok := os.LookupEnv(sourceBranchEnvName); ok { - sourceBranch = value - } else { - sourceBranch = "nasdf/test/parallel-change-detector" - } - - var targetBranch string - if value, ok := os.LookupEnv(targetBranchEnvName); ok { - targetBranch = value - } else { - targetBranch = "nasdf/test/parallel-change-detector" - } - sourceRepoDir := t.TempDir() execClone(t, sourceRepoDir, repository, sourceBranch) + if checkIfDatabaseFormatChangesAreDocumented(sourceRepoDir) { + t.Skip("skipping test with documented database format changes") + } + targetRepoDir := t.TempDir() execClone(t, targetRepoDir, repository, targetBranch) @@ -69,20 +47,20 @@ func TestChanges(t *testing.T) { } for _, pkg := range targetRepoPkgList { - if pkg == "" || !sourceRepoPkgMap[pkg] { - continue // ignore new packages - } pkgName := strings.TrimPrefix(pkg, "github.com/sourcenetwork/defradb/") - t.Run(pkgName, func(t *testing.T) { + if pkg == "" || !sourceRepoPkgMap[pkg] { + t.Skip("skipping unknown or new test package") + } + t.Parallel() dataDir := t.TempDir() - fromTestPkg := filepath.Join(sourceRepoDir, pkgName) - execTest(t, fromTestPkg, dataDir, true) + sourceTestPkg := filepath.Join(sourceRepoDir, pkgName) + execTest(t, sourceTestPkg, dataDir, true) - toTestPkg := filepath.Join(targetRepoDir, pkgName) - execTest(t, toTestPkg, dataDir, false) + targetTestPkg := filepath.Join(targetRepoDir, pkgName) + execTest(t, targetTestPkg, dataDir, false) }) } } @@ -103,11 +81,14 @@ func execList(t *testing.T, dir string) []string { func execTest(t *testing.T, dir, dataDir string, setupOnly bool) { cmd := exec.Command("go", "test", ".", "-count", "1", "-v") cmd.Dir = dir - cmd.Env = append(os.Environ(), "DEFRA_BADGER_FILE_PATH="+dataDir) - cmd.Env = append(cmd.Env, "DEFRA_DETECT_DATABASE_CHANGES=true") + cmd.Env = append( + os.Environ(), + fmt.Sprintf("%s=%s", enableEnvName, "true"), + fmt.Sprintf("%s=%s", rootDataDirEnvName, dataDir), + ) if setupOnly { - cmd.Env = append(cmd.Env, "DEFRA_SETUP_ONLY=true") + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", setupOnlyEnvName, "true")) } out, err := cmd.Output() diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go new file mode 100644 index 0000000000..a19438d57d --- /dev/null +++ b/tests/change_detector/utils.go @@ -0,0 +1,194 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package change_detector + +import ( + "io/fs" + "os" + "path" + "runtime" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +var ( + // Enabled is true when the change detector is running. + Enabled bool + // SetupOnly is true when the change detector is running in setup mode. + SetupOnly bool + // rootDatabaseDir is the shared database directory for running tests. + rootDatabaseDir string + // repository is the url of the repository to run change detector on. + repository string + // sourceBranch is the name of the source branch to run change detector on. + sourceBranch string + // targetBranch is the name of the target branch to run change detector on. + targetBranch string + // previousTestCaseTestName is the name of the previous test. + previousTestCaseTestName string +) + +const ( + repositoryEnvName = "DEFRA_CHANGE_DETECTOR_REPOSITORY" + sourceBranchEnvName = "DEFRA_CHANGE_DETECTOR_SOURCE_BRANCH" + targetBranchEnvName = "DEFRA_CHANGE_DETECTOR_TARGET_BRANCH" + setupOnlyEnvName = "DEFRA_CHANGE_DETECTOR_SETUP_ONLY" + rootDataDirEnvName = "DEFRA_CHANGE_DETECTOR_ROOT_DATA_DIR" + enableEnvName = "DEFRA_CHANGE_DETECTOR_ENABLE" +) + +const ( + defaultRepository = "https://github.com/nasdf/defradb.git" + defaultBranch = "nasdf/test/parallel-change-detector" + documentationDirectoryName = "data_format_changes" +) + +func init() { + Enabled, _ = strconv.ParseBool(os.Getenv(enableEnvName)) + SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) + rootDatabaseDir = os.Getenv(rootDataDirEnvName) + + if value, ok := os.LookupEnv(repositoryEnvName); ok { + repository = value + } else { + repository = defaultRepository + } + + if value, ok := os.LookupEnv(sourceBranchEnvName); ok { + sourceBranch = value + } else { + sourceBranch = defaultBranch + } + + if value, ok := os.LookupEnv(targetBranchEnvName); ok { + targetBranch = value + } else { + targetBranch = defaultBranch + } +} + +// DatabaseDir returns the database directory for change detector test. +func DatabaseDir(t testing.TB) string { + return path.Join(rootDatabaseDir, t.Name()) +} + +// PreTestChecks skips any test that can't be run by the change detector. +func PreTestChecks(t *testing.T, collectionNames []string) { + if !Enabled { + return + } + + if previousTestCaseTestName == t.Name() { + t.Skip("skipping duplicate test") + } + previousTestCaseTestName = t.Name() + + if len(collectionNames) == 0 { + t.Skip("skipping test with no collections") + } + + if SetupOnly { + return + } + + _, err := os.Stat(DatabaseDir(t)) + if os.IsNotExist(err) { + t.Skip("skipping new test package") + } + require.NoError(t, err) +} + +func checkIfDatabaseFormatChangesAreDocumented(codeDir string) bool { + previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation(codeDir, false) + if !targetDirFound { + panic("Documentation directory not found") + } + + previousDbChanges := make(map[string]struct{}, len(previousDbChangeFiles)) + for _, f := range previousDbChangeFiles { + // Note: we assume flat directory for now - sub directories are not expanded + previousDbChanges[f.Name()] = struct{}{} + } + + _, thisFilePath, _, _ := runtime.Caller(0) + currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(thisFilePath, true) + if !currentDirFound { + panic("Documentation directory not found") + } + + for _, f := range currentDbChanges { + if _, isChangeOld := previousDbChanges[f.Name()]; !isChangeOld { + // If there is a new file in the directory then the change + // has been documented and the test should pass + return true + } + } + + return false +} + +func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.DirEntry, bool) { + startInfo, err := os.Stat(startPath) + if err != nil { + panic(err) + } + + var currentDirectory string + if startInfo.IsDir() { + currentDirectory = startPath + } else { + currentDirectory = path.Dir(startPath) + } + + for { + directoryContents, err := os.ReadDir(currentDirectory) + if err != nil { + panic(err) + } + + for _, directoryItem := range directoryContents { + directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) + if directoryItem.Name() == documentationDirectoryName { + probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) + if err != nil { + panic(err) + } + for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { + if path.Ext(possibleDocumentationItem.Name()) == ".md" { + // If the directory's name matches the expected, and contains .md files + // we assume it is the documentation directory + return probableFormatChangeDirectoryContents, true + } + } + } else { + if directoryItem.IsDir() { + childContents, directoryFound := getDatabaseFormatDocumentation(directoryItemPath, false) + if directoryFound { + return childContents, true + } + } + } + } + + if allowDescend { + // If not found in this directory, continue down the path + currentDirectory = path.Dir(currentDirectory) + + if currentDirectory == "." || currentDirectory == "/" { + panic("Database documentation directory not found") + } + } else { + return []fs.DirEntry{}, false + } + } +} diff --git a/tests/integration/change_detector.go b/tests/integration/change_detector.go deleted file mode 100644 index a2077fc273..0000000000 --- a/tests/integration/change_detector.go +++ /dev/null @@ -1,39 +0,0 @@ -package tests - -import ( - "os" - "path" - "testing" - - "github.com/stretchr/testify/require" -) - -func DetectDbChangesPreTestChecks( - t *testing.T, - collectionNames []string, -) { - if previousTestCaseTestName == t.Name() { - // The database format changer currently only supports running the first test - // case, if a second case is detected we return early - t.Skip() - } - previousTestCaseTestName = t.Name() - - if len(collectionNames) == 0 { - // If the test doesn't specify any collections, then we can't use it to check - // the database format, so we skip it - t.SkipNow() - } - - if !SetupOnly { - dbDirectory := path.Join(rootDatabaseDir, t.Name()) - _, err := os.Stat(dbDirectory) - if os.IsNotExist(err) { - // This is a new test that does not exist in the target branch, we should - // skip it. - t.SkipNow() - } else { - require.NoError(t, err) - } - } -} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 8f29ab4d6d..a182539ddb 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -15,7 +15,6 @@ import ( "encoding/json" "fmt" "os" - "path" "reflect" "strconv" "strings" @@ -36,20 +35,17 @@ import ( "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/logging" "github.com/sourcenetwork/defradb/net" + "github.com/sourcenetwork/defradb/tests/change_detector" ) const ( - clientGoEnvName = "DEFRA_CLIENT_GO" - clientHttpEnvName = "DEFRA_CLIENT_HTTP" - memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" - fileBadgerEnvName = "DEFRA_BADGER_FILE" - fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" - rootDBFilePathEnvName = "DEFRA_TEST_ROOT" - inMemoryEnvName = "DEFRA_IN_MEMORY" - setupOnlyEnvName = "DEFRA_SETUP_ONLY" - detectDbChangesEnvName = "DEFRA_DETECT_DATABASE_CHANGES" - mutationTypeEnvName = "DEFRA_MUTATION_TYPE" - documentationDirectoryName = "data_format_changes" + clientGoEnvName = "DEFRA_CLIENT_GO" + clientHttpEnvName = "DEFRA_CLIENT_HTTP" + memoryBadgerEnvName = "DEFRA_BADGER_MEMORY" + fileBadgerEnvName = "DEFRA_BADGER_FILE" + fileBadgerPathEnvName = "DEFRA_BADGER_FILE_PATH" + inMemoryEnvName = "DEFRA_IN_MEMORY" + mutationTypeEnvName = "DEFRA_MUTATION_TYPE" ) type DatabaseType string @@ -106,41 +102,16 @@ var ( httpClient bool goClient bool mutationType MutationType - // databaseDir is the directory of the badger file db. - databaseDir string - // rootDatabaseDir is the root directroy of the badger file db. - rootDatabaseDir string - // previousTestCaseName is the name of the previous test. - previousTestCaseTestName string + databaseDir string ) -const subscriptionTimeout = 1 * time.Second - -// Instantiating lenses is expensive, and our tests do not benefit from a large number of them, -// so we explicitly set it to a low value. -const lensPoolSize = 2 - -/* -If this is set to true the integration test suite will instead of its normal profile do -the following: - -On [package] Init: - - Get the (local) latest commit from the target/parent branch // code assumes - git fetch has been done - - Check to see if a clone of that commit/branch is available in the temp dir, and - if not clone the target branch - - Check to see if there are any new .md files in the current branch's data_format_changes - dir (vs the target branch) - -For each test: - - If new documentation detected, pass the test and exit - - Create a new (test/auto-deleted) temp dir for defra to live/run in - - Run the test setup (add initial schema, docs, updates) using the target branch (test is skipped - if test does not exist in target and is new to this branch) - - Run the test request and assert results (as per normal tests) using the current branch -*/ -var DetectDbChanges bool -var SetupOnly bool +const ( + // subscriptionTimeout is the maximum time to wait for subscription results to be returned. + subscriptionTimeout = 1 * time.Second + // Instantiating lenses is expensive, and our tests do not benefit from a large number of them, + // so we explicitly set it to a low value. + lensPoolSize = 2 +) func init() { // We use environment variables instead of flags `go test ./...` throws for all packages @@ -150,9 +121,6 @@ func init() { badgerFile, _ = strconv.ParseBool(os.Getenv(fileBadgerEnvName)) badgerInMemory, _ = strconv.ParseBool(os.Getenv(memoryBadgerEnvName)) inMemoryStore, _ = strconv.ParseBool(os.Getenv(inMemoryEnvName)) - DetectDbChanges, _ = strconv.ParseBool(os.Getenv(detectDbChangesEnvName)) - SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) - rootDatabaseDir = os.Getenv(rootDBFilePathEnvName) if value, ok := os.LookupEnv(mutationTypeEnvName); ok { mutationType = MutationType(value) @@ -168,7 +136,7 @@ func init() { goClient = true } - if DetectDbChanges { + if change_detector.Enabled { // Change detector only uses badger file db type. badgerFile = true badgerInMemory = false @@ -187,7 +155,7 @@ func init() { // // Usage: AssertPanicAndSkipChangeDetection(t, func() { executeTestCase(t, test) }) func AssertPanicAndSkipChangeDetection(t *testing.T, f assert.PanicTestFunc) bool { - if DetectDbChanges { + if change_detector.Enabled { // The `assert.Panics` call will falsely fail if this test is executed during // a detect changes test run t.Skip() @@ -196,41 +164,44 @@ func AssertPanicAndSkipChangeDetection(t *testing.T, f assert.PanicTestFunc) boo } func NewBadgerMemoryDB(ctx context.Context, dbopts ...db.Option) (client.DB, error) { - opts := badgerds.Options{Options: badger.DefaultOptions("").WithInMemory(true)} + opts := badgerds.Options{ + Options: badger.DefaultOptions("").WithInMemory(true), + } rootstore, err := badgerds.NewDatastore("", &opts) if err != nil { return nil, err } - - dbopts = append(dbopts, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) - db, err := db.NewDB(ctx, rootstore, dbopts...) if err != nil { return nil, err } - return db, nil } -func NewInMemoryDB(ctx context.Context) (client.DB, error) { - rootstore := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, rootstore, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) +func NewInMemoryDB(ctx context.Context, dbopts ...db.Option) (client.DB, error) { + db, err := db.NewDB(ctx, memory.NewDatastore(ctx), dbopts...) if err != nil { return nil, err } - return db, nil } -func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, error) { +func NewBadgerFileDB(ctx context.Context, t testing.TB, dbopts ...db.Option) (client.DB, string, error) { var dbPath string - if databaseDir != "" { + switch { + case databaseDir != "": + // restarting database dbPath = databaseDir - } else if rootDatabaseDir != "" { - dbPath = path.Join(rootDatabaseDir, t.Name()) - } else { + + case change_detector.Enabled: + // change detector + dbPath = change_detector.DatabaseDir(t) + + default: + // default test case dbPath = t.TempDir() } + opts := &badgerds.Options{ Options: badger.DefaultOptions(dbPath), } @@ -238,7 +209,7 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, erro if err != nil { return nil, "", err } - db, err := db.NewDB(ctx, rootstore, db.WithUpdateEvents(), db.WithLensPoolSize(lensPoolSize)) + db, err := db.NewDB(ctx, rootstore, dbopts...) if err != nil { return nil, "", err } @@ -249,15 +220,20 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, string, erro // testing state. The database type and client type on the test state // are used to select the datastore and client implementation to use. func GetDatabase(s *state) (cdb client.DB, path string, err error) { + dbopts := []db.Option{ + db.WithUpdateEvents(), + db.WithLensPoolSize(lensPoolSize), + } + switch s.dbt { case badgerIMType: - cdb, err = NewBadgerMemoryDB(s.ctx, db.WithUpdateEvents()) + cdb, err = NewBadgerMemoryDB(s.ctx, dbopts...) case badgerFileType: - cdb, path, err = NewBadgerFileDB(s.ctx, s.t) + cdb, path, err = NewBadgerFileDB(s.ctx, s.t, dbopts...) case defraIMType: - cdb, err = NewInMemoryDB(s.ctx) + cdb, err = NewInMemoryDB(s.ctx, dbopts...) default: err = fmt.Errorf("invalid database type: %v", s.dbt) @@ -295,11 +271,7 @@ func ExecuteTestCase( testCase TestCase, ) { collectionNames := getCollectionNames(testCase) - - if DetectDbChanges { - DetectDbChangesPreTestChecks(t, collectionNames) - } - + change_detector.PreTestChecks(t, collectionNames) skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) var clients []ClientType @@ -604,7 +576,7 @@ func getActionRange(testCase TestCase) (int, int) { startIndex := 0 endIndex := len(testCase.Actions) - 1 - if !DetectDbChanges { + if !change_detector.Enabled { return startIndex, endIndex } @@ -628,7 +600,7 @@ ActionLoop: } } - if SetupOnly { + if change_detector.SetupOnly { if setupCompleteIndex > -1 { endIndex = setupCompleteIndex } else if firstNonSetupIndex > -1 { @@ -797,7 +769,7 @@ func configureNode( s *state, action ConfigureNode, ) { - if DetectDbChanges { + if change_detector.Enabled { // We do not yet support the change detector for tests running across multiple nodes. s.t.SkipNow() return From b312303b4e70992087b754a5ec275a65189b36b8 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 11:59:11 -0700 Subject: [PATCH 13/21] replace panic with test.Fail in checkIfDatabaseFormatChangesAreDocumented and getDatabaseFormatDocumentation --- tests/change_detector/change_detector_test.go | 2 +- tests/change_detector/utils.go | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 61b6f58c46..ed1a0de8a8 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -25,7 +25,7 @@ func TestChanges(t *testing.T) { sourceRepoDir := t.TempDir() execClone(t, sourceRepoDir, repository, sourceBranch) - if checkIfDatabaseFormatChangesAreDocumented(sourceRepoDir) { + if checkIfDatabaseFormatChangesAreDocumented(t, sourceRepoDir) { t.Skip("skipping test with documented database format changes") } diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go index a19438d57d..c5f6d8a79b 100644 --- a/tests/change_detector/utils.go +++ b/tests/change_detector/utils.go @@ -108,10 +108,10 @@ func PreTestChecks(t *testing.T, collectionNames []string) { require.NoError(t, err) } -func checkIfDatabaseFormatChangesAreDocumented(codeDir string) bool { - previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation(codeDir, false) +func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, codeDir string) bool { + previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation(t, codeDir, false) if !targetDirFound { - panic("Documentation directory not found") + t.Fatalf("Documentation directory not found") } previousDbChanges := make(map[string]struct{}, len(previousDbChangeFiles)) @@ -121,9 +121,9 @@ func checkIfDatabaseFormatChangesAreDocumented(codeDir string) bool { } _, thisFilePath, _, _ := runtime.Caller(0) - currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(thisFilePath, true) + currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(t, thisFilePath, true) if !currentDirFound { - panic("Documentation directory not found") + t.Fatalf("Documentation directory not found") } for _, f := range currentDbChanges { @@ -137,10 +137,10 @@ func checkIfDatabaseFormatChangesAreDocumented(codeDir string) bool { return false } -func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.DirEntry, bool) { +func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend bool) ([]fs.DirEntry, bool) { startInfo, err := os.Stat(startPath) if err != nil { - panic(err) + t.Fatal(err) } var currentDirectory string @@ -153,7 +153,7 @@ func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.D for { directoryContents, err := os.ReadDir(currentDirectory) if err != nil { - panic(err) + t.Fatal(err) } for _, directoryItem := range directoryContents { @@ -161,7 +161,7 @@ func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.D if directoryItem.Name() == documentationDirectoryName { probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) if err != nil { - panic(err) + t.Fatal(err) } for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { if path.Ext(possibleDocumentationItem.Name()) == ".md" { @@ -172,7 +172,7 @@ func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.D } } else { if directoryItem.IsDir() { - childContents, directoryFound := getDatabaseFormatDocumentation(directoryItemPath, false) + childContents, directoryFound := getDatabaseFormatDocumentation(t, directoryItemPath, false) if directoryFound { return childContents, true } @@ -185,7 +185,7 @@ func getDatabaseFormatDocumentation(startPath string, allowDescend bool) ([]fs.D currentDirectory = path.Dir(currentDirectory) if currentDirectory == "." || currentDirectory == "/" { - panic("Database documentation directory not found") + t.Fatal(err) } } else { return []fs.DirEntry{}, false From 6039650fc588df638d81a1a0c4f771819fda7a64 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 12 Sep 2023 12:01:03 -0700 Subject: [PATCH 14/21] update test:changes Makefile target test flags --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca169902bc..9bacb829ec 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,7 @@ test\:coverage-html: .PHONY: test\:changes test\:changes: - gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... $(TEST_FLAGS) + gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... -shuffle=on .PHONY: validate\:codecov validate\:codecov: From f6af08b997da7e1c4e289d17c8ccf5a0b93983e3 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 18 Sep 2023 11:30:29 -0700 Subject: [PATCH 15/21] add change_detector build flag. remove shuffle from change_detector Makefile target --- Makefile | 4 ++-- tests/change_detector/change_detector_test.go | 2 ++ tests/integration/utils2.go | 18 +++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 9bacb829ec..369d7b8245 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ PLAYGROUND_DIRECTORY=playground LENS_TEST_DIRECTORY=tests/integration/schema/migrations CLI_TEST_DIRECTORY=tests/integration/cli CHANGE_DETECTOR_TEST_DIRECTORY=tests/change_detector -DEFAULT_TEST_DIRECTORIES=$$(go list ./... | grep -v -e $(LENS_TEST_DIRECTORY) -e $(CLI_TEST_DIRECTORY) -e $(CHANGE_DETECTOR_TEST_DIRECTORY)) +DEFAULT_TEST_DIRECTORIES=$$(go list ./... | grep -v -e $(LENS_TEST_DIRECTORY) -e $(CLI_TEST_DIRECTORY)) default: @go run $(BUILD_FLAGS) cmd/defradb/main.go @@ -295,7 +295,7 @@ test\:coverage-html: .PHONY: test\:changes test\:changes: - gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... -shuffle=on + gotestsum --format testname -- ./$(CHANGE_DETECTOR_TEST_DIRECTORY)/... --tags change_detector .PHONY: validate\:codecov validate\:codecov: diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index ed1a0de8a8..bd5dc157fe 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -8,6 +8,8 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +//go:build change_detector + package change_detector import ( diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index a182539ddb..eb7e44e3ab 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -35,7 +35,7 @@ import ( "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/logging" "github.com/sourcenetwork/defradb/net" - "github.com/sourcenetwork/defradb/tests/change_detector" + changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" ) const ( @@ -136,7 +136,7 @@ func init() { goClient = true } - if change_detector.Enabled { + if changeDetector.Enabled { // Change detector only uses badger file db type. badgerFile = true badgerInMemory = false @@ -155,7 +155,7 @@ func init() { // // Usage: AssertPanicAndSkipChangeDetection(t, func() { executeTestCase(t, test) }) func AssertPanicAndSkipChangeDetection(t *testing.T, f assert.PanicTestFunc) bool { - if change_detector.Enabled { + if changeDetector.Enabled { // The `assert.Panics` call will falsely fail if this test is executed during // a detect changes test run t.Skip() @@ -193,9 +193,9 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB, dbopts ...db.Option) (cl // restarting database dbPath = databaseDir - case change_detector.Enabled: + case changeDetector.Enabled: // change detector - dbPath = change_detector.DatabaseDir(t) + dbPath = changeDetector.DatabaseDir(t) default: // default test case @@ -271,7 +271,7 @@ func ExecuteTestCase( testCase TestCase, ) { collectionNames := getCollectionNames(testCase) - change_detector.PreTestChecks(t, collectionNames) + changeDetector.PreTestChecks(t, collectionNames) skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) var clients []ClientType @@ -576,7 +576,7 @@ func getActionRange(testCase TestCase) (int, int) { startIndex := 0 endIndex := len(testCase.Actions) - 1 - if !change_detector.Enabled { + if !changeDetector.Enabled { return startIndex, endIndex } @@ -600,7 +600,7 @@ ActionLoop: } } - if change_detector.SetupOnly { + if changeDetector.SetupOnly { if setupCompleteIndex > -1 { endIndex = setupCompleteIndex } else if firstNonSetupIndex > -1 { @@ -769,7 +769,7 @@ func configureNode( s *state, action ConfigureNode, ) { - if change_detector.Enabled { + if changeDetector.Enabled { // We do not yet support the change detector for tests running across multiple nodes. s.t.SkipNow() return From eb80d804d7fcb12f3fa8a4c49ba0bb4a8cb7a840 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 18 Sep 2023 12:00:39 -0700 Subject: [PATCH 16/21] change detector target defaults to local repo --- tests/change_detector/change_detector_test.go | 15 ++++++++++++--- tests/change_detector/utils.go | 11 +++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index bd5dc157fe..d8b9b2aa97 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -31,11 +31,20 @@ func TestChanges(t *testing.T) { t.Skip("skipping test with documented database format changes") } - targetRepoDir := t.TempDir() - execClone(t, targetRepoDir, repository, targetBranch) + var targetRepoDir string + if targetBranch == "" { + // default to the local branch + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + require.NoError(t, err, string(out)) + targetRepoDir = strings.TrimSpace(string(out)) + } else { + // check out the target branch + targetRepoDir = t.TempDir() + execClone(t, targetRepoDir, repository, targetBranch) + execMakeDeps(t, targetRepoDir) + } execMakeDeps(t, sourceRepoDir) - execMakeDeps(t, targetRepoDir) targetRepoTestDir := filepath.Join(targetRepoDir, "tests", "integration") targetRepoPkgList := execList(t, targetRepoTestDir) diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go index c5f6d8a79b..d4092e4716 100644 --- a/tests/change_detector/utils.go +++ b/tests/change_detector/utils.go @@ -49,7 +49,7 @@ const ( const ( defaultRepository = "https://github.com/nasdf/defradb.git" - defaultBranch = "nasdf/test/parallel-change-detector" + defaultSourceBranch = "nasdf/test/parallel-change-detector" documentationDirectoryName = "data_format_changes" ) @@ -57,6 +57,7 @@ func init() { Enabled, _ = strconv.ParseBool(os.Getenv(enableEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) rootDatabaseDir = os.Getenv(rootDataDirEnvName) + targetBranch = os.Getenv(targetBranchEnvName) if value, ok := os.LookupEnv(repositoryEnvName); ok { repository = value @@ -67,13 +68,7 @@ func init() { if value, ok := os.LookupEnv(sourceBranchEnvName); ok { sourceBranch = value } else { - sourceBranch = defaultBranch - } - - if value, ok := os.LookupEnv(targetBranchEnvName); ok { - targetBranch = value - } else { - targetBranch = defaultBranch + sourceBranch = defaultSourceBranch } } From 97396de6b11fd8394548ba73135525f7231c8a48 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 18 Sep 2023 12:24:25 -0700 Subject: [PATCH 17/21] document change detector changes. set default branch and repo to upstream defradb repo. update checkIfDatabaseFormatChangesAreDocumented to use source and target directories --- .../i1436-no-change-tests-updated.md | 3 ++ tests/change_detector/change_detector_test.go | 9 ++-- tests/change_detector/utils.go | 48 +++++++------------ 3 files changed, 24 insertions(+), 36 deletions(-) create mode 100644 docs/data_format_changes/i1436-no-change-tests-updated.md diff --git a/docs/data_format_changes/i1436-no-change-tests-updated.md b/docs/data_format_changes/i1436-no-change-tests-updated.md new file mode 100644 index 0000000000..89f7305133 --- /dev/null +++ b/docs/data_format_changes/i1436-no-change-tests-updated.md @@ -0,0 +1,3 @@ +# Parallel change detector + +This is is not a breaking change. The change detector has been updated to allow for parallel test runs. There were changes to environment variables and test setup that makes the previous version of the change detector incompatible with this version. diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index d8b9b2aa97..dc79e21ed1 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -26,10 +26,7 @@ import ( func TestChanges(t *testing.T) { sourceRepoDir := t.TempDir() execClone(t, sourceRepoDir, repository, sourceBranch) - - if checkIfDatabaseFormatChangesAreDocumented(t, sourceRepoDir) { - t.Skip("skipping test with documented database format changes") - } + execMakeDeps(t, sourceRepoDir) var targetRepoDir string if targetBranch == "" { @@ -44,7 +41,9 @@ func TestChanges(t *testing.T) { execMakeDeps(t, targetRepoDir) } - execMakeDeps(t, sourceRepoDir) + if checkIfDatabaseFormatChangesAreDocumented(t, sourceRepoDir, targetRepoDir) { + t.Skip("skipping test with documented database format changes") + } targetRepoTestDir := filepath.Join(targetRepoDir, "tests", "integration") targetRepoPkgList := execList(t, targetRepoTestDir) diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go index d4092e4716..687874c3e1 100644 --- a/tests/change_detector/utils.go +++ b/tests/change_detector/utils.go @@ -14,7 +14,6 @@ import ( "io/fs" "os" "path" - "runtime" "strconv" "testing" @@ -48,8 +47,8 @@ const ( ) const ( - defaultRepository = "https://github.com/nasdf/defradb.git" - defaultSourceBranch = "nasdf/test/parallel-change-detector" + defaultRepository = "https://github.com/sourcenetwork/defradb.git" + defaultSourceBranch = "develop" documentationDirectoryName = "data_format_changes" ) @@ -103,26 +102,21 @@ func PreTestChecks(t *testing.T, collectionNames []string) { require.NoError(t, err) } -func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, codeDir string) bool { - previousDbChangeFiles, targetDirFound := getDatabaseFormatDocumentation(t, codeDir, false) - if !targetDirFound { - t.Fatalf("Documentation directory not found") - } +func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, sourceDir, targetDir string) bool { + sourceChanges, ok := getDatabaseFormatDocumentation(t, sourceDir, false) + require.True(t, ok, "Documentation directory not found") - previousDbChanges := make(map[string]struct{}, len(previousDbChangeFiles)) - for _, f := range previousDbChangeFiles { + changes := make(map[string]struct{}, len(sourceChanges)) + for _, f := range sourceChanges { // Note: we assume flat directory for now - sub directories are not expanded - previousDbChanges[f.Name()] = struct{}{} + changes[f.Name()] = struct{}{} } - _, thisFilePath, _, _ := runtime.Caller(0) - currentDbChanges, currentDirFound := getDatabaseFormatDocumentation(t, thisFilePath, true) - if !currentDirFound { - t.Fatalf("Documentation directory not found") - } + targetChanges, ok := getDatabaseFormatDocumentation(t, targetDir, true) + require.True(t, ok, "Documentation directory not found") - for _, f := range currentDbChanges { - if _, isChangeOld := previousDbChanges[f.Name()]; !isChangeOld { + for _, f := range targetChanges { + if _, isChangeOld := changes[f.Name()]; !isChangeOld { // If there is a new file in the directory then the change // has been documented and the test should pass return true @@ -134,9 +128,7 @@ func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, codeDir string) boo func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend bool) ([]fs.DirEntry, bool) { startInfo, err := os.Stat(startPath) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) var currentDirectory string if startInfo.IsDir() { @@ -147,17 +139,14 @@ func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend for { directoryContents, err := os.ReadDir(currentDirectory) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, directoryItem := range directoryContents { directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) if directoryItem.Name() == documentationDirectoryName { probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { if path.Ext(possibleDocumentationItem.Name()) == ".md" { // If the directory's name matches the expected, and contains .md files @@ -178,10 +167,7 @@ func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend if allowDescend { // If not found in this directory, continue down the path currentDirectory = path.Dir(currentDirectory) - - if currentDirectory == "." || currentDirectory == "/" { - t.Fatal(err) - } + require.True(t, currentDirectory != "." && currentDirectory != "/") } else { return []fs.DirEntry{}, false } From fc6cbcdb31b94163609a6d42308ad50596bb2e6d Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 18 Sep 2023 12:33:37 -0700 Subject: [PATCH 18/21] fix linter errors --- tests/change_detector/change_detector_test.go | 74 +++++++++++++++ tests/change_detector/utils.go | 95 +++---------------- 2 files changed, 85 insertions(+), 84 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index dc79e21ed1..39e89d5515 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -14,8 +14,10 @@ package change_detector import ( "fmt" + "io/fs" "os" "os/exec" + "path" "path/filepath" "strings" "testing" @@ -129,3 +131,75 @@ func execMakeDeps(t *testing.T, dir string) { out, err := cmd.Output() require.NoError(t, err, string(out)) } + +func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, sourceDir, targetDir string) bool { + sourceChanges, ok := getDatabaseFormatDocumentation(t, sourceDir, false) + require.True(t, ok, "Documentation directory not found") + + changes := make(map[string]struct{}, len(sourceChanges)) + for _, f := range sourceChanges { + // Note: we assume flat directory for now - sub directories are not expanded + changes[f.Name()] = struct{}{} + } + + targetChanges, ok := getDatabaseFormatDocumentation(t, targetDir, true) + require.True(t, ok, "Documentation directory not found") + + for _, f := range targetChanges { + if _, isChangeOld := changes[f.Name()]; !isChangeOld { + // If there is a new file in the directory then the change + // has been documented and the test should pass + return true + } + } + + return false +} + +func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend bool) ([]fs.DirEntry, bool) { + startInfo, err := os.Stat(startPath) + require.NoError(t, err) + + var currentDirectory string + if startInfo.IsDir() { + currentDirectory = startPath + } else { + currentDirectory = path.Dir(startPath) + } + + for { + directoryContents, err := os.ReadDir(currentDirectory) + require.NoError(t, err) + + for _, directoryItem := range directoryContents { + directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) + if directoryItem.Name() == documentationDirectoryName { + probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) + require.NoError(t, err) + + for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { + if path.Ext(possibleDocumentationItem.Name()) == ".md" { + // If the directory's name matches the expected, and contains .md files + // we assume it is the documentation directory + return probableFormatChangeDirectoryContents, true + } + } + } else { + if directoryItem.IsDir() { + childContents, directoryFound := getDatabaseFormatDocumentation(t, directoryItemPath, false) + if directoryFound { + return childContents, true + } + } + } + } + + if allowDescend { + // If not found in this directory, continue down the path + currentDirectory = path.Dir(currentDirectory) + require.True(t, currentDirectory != "." && currentDirectory != "/") + } else { + return []fs.DirEntry{}, false + } + } +} diff --git a/tests/change_detector/utils.go b/tests/change_detector/utils.go index 687874c3e1..4e6e938aa5 100644 --- a/tests/change_detector/utils.go +++ b/tests/change_detector/utils.go @@ -11,7 +11,6 @@ package change_detector import ( - "io/fs" "os" "path" "strconv" @@ -25,14 +24,14 @@ var ( Enabled bool // SetupOnly is true when the change detector is running in setup mode. SetupOnly bool + // Repository is the url of the repository to run change detector on. + Repository string + // SourceBranch is the name of the source branch to run change detector on. + SourceBranch string + // TargetBranch is the name of the target branch to run change detector on. + TargetBranch string // rootDatabaseDir is the shared database directory for running tests. rootDatabaseDir string - // repository is the url of the repository to run change detector on. - repository string - // sourceBranch is the name of the source branch to run change detector on. - sourceBranch string - // targetBranch is the name of the target branch to run change detector on. - targetBranch string // previousTestCaseTestName is the name of the previous test. previousTestCaseTestName string ) @@ -55,19 +54,19 @@ const ( func init() { Enabled, _ = strconv.ParseBool(os.Getenv(enableEnvName)) SetupOnly, _ = strconv.ParseBool(os.Getenv(setupOnlyEnvName)) + TargetBranch = os.Getenv(targetBranchEnvName) rootDatabaseDir = os.Getenv(rootDataDirEnvName) - targetBranch = os.Getenv(targetBranchEnvName) if value, ok := os.LookupEnv(repositoryEnvName); ok { - repository = value + Repository = value } else { - repository = defaultRepository + Repository = defaultRepository } if value, ok := os.LookupEnv(sourceBranchEnvName); ok { - sourceBranch = value + SourceBranch = value } else { - sourceBranch = defaultSourceBranch + SourceBranch = defaultSourceBranch } } @@ -101,75 +100,3 @@ func PreTestChecks(t *testing.T, collectionNames []string) { } require.NoError(t, err) } - -func checkIfDatabaseFormatChangesAreDocumented(t *testing.T, sourceDir, targetDir string) bool { - sourceChanges, ok := getDatabaseFormatDocumentation(t, sourceDir, false) - require.True(t, ok, "Documentation directory not found") - - changes := make(map[string]struct{}, len(sourceChanges)) - for _, f := range sourceChanges { - // Note: we assume flat directory for now - sub directories are not expanded - changes[f.Name()] = struct{}{} - } - - targetChanges, ok := getDatabaseFormatDocumentation(t, targetDir, true) - require.True(t, ok, "Documentation directory not found") - - for _, f := range targetChanges { - if _, isChangeOld := changes[f.Name()]; !isChangeOld { - // If there is a new file in the directory then the change - // has been documented and the test should pass - return true - } - } - - return false -} - -func getDatabaseFormatDocumentation(t *testing.T, startPath string, allowDescend bool) ([]fs.DirEntry, bool) { - startInfo, err := os.Stat(startPath) - require.NoError(t, err) - - var currentDirectory string - if startInfo.IsDir() { - currentDirectory = startPath - } else { - currentDirectory = path.Dir(startPath) - } - - for { - directoryContents, err := os.ReadDir(currentDirectory) - require.NoError(t, err) - - for _, directoryItem := range directoryContents { - directoryItemPath := path.Join(currentDirectory, directoryItem.Name()) - if directoryItem.Name() == documentationDirectoryName { - probableFormatChangeDirectoryContents, err := os.ReadDir(directoryItemPath) - require.NoError(t, err) - - for _, possibleDocumentationItem := range probableFormatChangeDirectoryContents { - if path.Ext(possibleDocumentationItem.Name()) == ".md" { - // If the directory's name matches the expected, and contains .md files - // we assume it is the documentation directory - return probableFormatChangeDirectoryContents, true - } - } - } else { - if directoryItem.IsDir() { - childContents, directoryFound := getDatabaseFormatDocumentation(t, directoryItemPath, false) - if directoryFound { - return childContents, true - } - } - } - } - - if allowDescend { - // If not found in this directory, continue down the path - currentDirectory = path.Dir(currentDirectory) - require.True(t, currentDirectory != "." && currentDirectory != "/") - } else { - return []fs.DirEntry{}, false - } - } -} From cfc1182c5717430eca514639062747545a0adf36 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 18 Sep 2023 12:36:22 -0700 Subject: [PATCH 19/21] fix build errors --- tests/change_detector/change_detector_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/change_detector/change_detector_test.go b/tests/change_detector/change_detector_test.go index 39e89d5515..ac9bc1a23f 100644 --- a/tests/change_detector/change_detector_test.go +++ b/tests/change_detector/change_detector_test.go @@ -27,11 +27,11 @@ import ( func TestChanges(t *testing.T) { sourceRepoDir := t.TempDir() - execClone(t, sourceRepoDir, repository, sourceBranch) + execClone(t, sourceRepoDir, Repository, SourceBranch) execMakeDeps(t, sourceRepoDir) var targetRepoDir string - if targetBranch == "" { + if TargetBranch == "" { // default to the local branch out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() require.NoError(t, err, string(out)) @@ -39,7 +39,7 @@ func TestChanges(t *testing.T) { } else { // check out the target branch targetRepoDir = t.TempDir() - execClone(t, targetRepoDir, repository, targetBranch) + execClone(t, targetRepoDir, Repository, TargetBranch) execMakeDeps(t, targetRepoDir) } From b01d3013d24deff815734f7ec38dea8a854b1854 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 19 Sep 2023 10:50:52 -0700 Subject: [PATCH 20/21] add config logging --- tests/integration/utils2.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index b662b7f27c..0821e95675 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -18,6 +18,7 @@ import ( "reflect" "strconv" "strings" + "sync" "testing" "time" @@ -103,6 +104,7 @@ var ( goClient bool mutationType MutationType databaseDir string + logConfigOnce sync.Once ) const ( @@ -277,6 +279,26 @@ func ExecuteTestCase( t *testing.T, testCase TestCase, ) { + logConfigOnce.Do(func() { + log.Info( + context.Background(), + "**** Running tests with config ****", + logging.NewKV("badgerFile", badgerFile), + logging.NewKV("badgerInMemoryStore", badgerInMemory), + logging.NewKV("inMemoryStore", inMemoryStore), + logging.NewKV("httpClient", httpClient), + logging.NewKV("httpClient", goClient), + logging.NewKV("mutationType", mutationType), + logging.NewKV("databaseDir", databaseDir), + logging.NewKV("changeDetector.Enabled", changeDetector.Enabled), + logging.NewKV("changeDetector.SetupOnly", changeDetector.SetupOnly), + logging.NewKV("changeDetector.SourceBranch", changeDetector.SourceBranch), + logging.NewKV("changeDetector.TargetBranch", changeDetector.TargetBranch), + logging.NewKV("changeDetector.Repository", changeDetector.Repository), + logging.NewKV("changeDetector.DatabaseDir", changeDetector.DatabaseDir(t)), + ) + }) + collectionNames := getCollectionNames(testCase) changeDetector.PreTestChecks(t, collectionNames) skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) From 362603747b99397a73df4a8f0f9a299d76ac4ec3 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 19 Sep 2023 12:02:51 -0700 Subject: [PATCH 21/21] move test logging to executeTestCase --- tests/integration/utils2.go | 39 +++++++++++++++---------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 0821e95675..f40a4cc447 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -18,7 +18,6 @@ import ( "reflect" "strconv" "strings" - "sync" "testing" "time" @@ -104,7 +103,6 @@ var ( goClient bool mutationType MutationType databaseDir string - logConfigOnce sync.Once ) const ( @@ -279,26 +277,6 @@ func ExecuteTestCase( t *testing.T, testCase TestCase, ) { - logConfigOnce.Do(func() { - log.Info( - context.Background(), - "**** Running tests with config ****", - logging.NewKV("badgerFile", badgerFile), - logging.NewKV("badgerInMemoryStore", badgerInMemory), - logging.NewKV("inMemoryStore", inMemoryStore), - logging.NewKV("httpClient", httpClient), - logging.NewKV("httpClient", goClient), - logging.NewKV("mutationType", mutationType), - logging.NewKV("databaseDir", databaseDir), - logging.NewKV("changeDetector.Enabled", changeDetector.Enabled), - logging.NewKV("changeDetector.SetupOnly", changeDetector.SetupOnly), - logging.NewKV("changeDetector.SourceBranch", changeDetector.SourceBranch), - logging.NewKV("changeDetector.TargetBranch", changeDetector.TargetBranch), - logging.NewKV("changeDetector.Repository", changeDetector.Repository), - logging.NewKV("changeDetector.DatabaseDir", changeDetector.DatabaseDir(t)), - ) - }) - collectionNames := getCollectionNames(testCase) changeDetector.PreTestChecks(t, collectionNames) skipIfMutationTypeUnsupported(t, testCase.SupportedMutationTypes) @@ -343,7 +321,22 @@ func executeTestCase( dbt DatabaseType, clientType ClientType, ) { - log.Info(ctx, testCase.Description, logging.NewKV("Database", dbt)) + log.Info( + ctx, + testCase.Description, + logging.NewKV("badgerFile", badgerFile), + logging.NewKV("badgerInMemory", badgerInMemory), + logging.NewKV("inMemoryStore", inMemoryStore), + logging.NewKV("httpClient", httpClient), + logging.NewKV("goClient", goClient), + logging.NewKV("mutationType", mutationType), + logging.NewKV("databaseDir", databaseDir), + logging.NewKV("changeDetector.Enabled", changeDetector.Enabled), + logging.NewKV("changeDetector.SetupOnly", changeDetector.SetupOnly), + logging.NewKV("changeDetector.SourceBranch", changeDetector.SourceBranch), + logging.NewKV("changeDetector.TargetBranch", changeDetector.TargetBranch), + logging.NewKV("changeDetector.Repository", changeDetector.Repository), + ) flattenActions(&testCase) startActionIndex, endActionIndex := getActionRange(testCase)