fix(db): add auto increment feature to environments table (#2166)

Ref: SRX-JXABF5
freiheit-com · Dec 11, 2024 · 1da7225 · 1da7225
1 parent 9e76991
commit 1da7225
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 60 deletions.
diff --git a/database/migrations/postgres/1733867056189286_environments_autoincrement.up.sql b/database/migrations/postgres/1733867056189286_environments_autoincrement.up.sql
@@ -0,0 +1,32 @@
+ALTER TABLE IF EXISTS environments ADD COLUMN IF NOT EXISTS row_version INTEGER;
+DO $$ 
+BEGIN
+    IF EXISTS (SELECT 1 
+               FROM information_schema.columns 
+               WHERE table_name = 'environments' 
+                 AND column_name = 'version') THEN
+        EXECUTE 'WITH ordered_rows AS (
+            SELECT version, name, ROW_NUMBER() OVER (ORDER BY version) AS row_num
+            FROM environments
+        )
+        UPDATE environments
+        SET row_version = ordered_rows.row_num
+        FROM ordered_rows
+        WHERE environments.version = ordered_rows.version AND environments.name = ordered_rows.name;';
+    END IF;
+END $$;
+
+DROP SEQUENCE IF EXISTS environments_version_seq CASCADE;
+CREATE SEQUENCE IF NOT EXISTS environments_version_seq OWNED BY environments.row_version;
+
+SELECT setval('environments_version_seq', coalesce(max(row_version), 0) + 1, false) FROM environments;
+
+ALTER TABLE IF EXISTS environments
+ALTER COLUMN row_version SET DEFAULT nextval('environments_version_seq');
+
+ALTER TABLE IF EXISTS environments DROP CONSTRAINT IF EXISTS environments_pkey;
+
+ALTER TABLE IF EXISTS environments ADD PRIMARY KEY (row_version, name);
+
+ALTER TABLE IF EXISTS environments DROP COLUMN IF EXISTS version;
+ALTER TABLE IF EXISTS environments RENAME COLUMN row_version TO version;
diff --git a/database/migrations/sqlite/1733867056189286_environments_autoincrement.up.sql b/database/migrations/sqlite/1733867056189286_environments_autoincrement.up.sql
@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS environments_new 
+(
+    version INTEGER PRIMARY KEY AUTOINCREMENT,
+    created TIMESTAMP,
+    name VARCHAR(255),
+    json VARCHAR,
+    deleted bool DEFAULT false NOT NULL,
+    applications VARCHAR
+);
+
+INSERT INTO environments_new (created, name, json, deleted, applications)
+SELECT created, name, json, deleted, applications
+FROM environments
+ORDER BY version;
+
+DROP TABLE IF EXISTS environments;
+ALTER TABLE environments_new RENAME TO environments;
diff --git a/pkg/db/db.go b/pkg/db/db.go
@@ -5316,15 +5316,13 @@ type DBAllEnvironmentsRow struct {
 
 type DBEnvironment struct {
 	Created      time.Time
-	Version      int64
 	Name         string
 	Config       config.EnvironmentConfig
 	Applications []string
 }
 
 type DBEnvironmentRow struct {
 	Created      time.Time
-	Version      int64
 	Name         string
 	Config       string
 	Applications string
@@ -5344,7 +5342,6 @@ func EnvironmentFromRow(_ context.Context, row *DBEnvironmentRow) (*DBEnvironmen
 	}
 	return &DBEnvironment{
 		Created:      row.Created,
-		Version:      row.Version,
 		Name:         row.Name,
 		Config:       parsedConfig,
 		Applications: applications,
@@ -5357,7 +5354,7 @@ func (h *DBHandler) DBSelectEnvironment(ctx context.Context, tx *sql.Tx, environ
 
 	selectQuery := h.AdaptQuery(
 		`
-SELECT created, version, name, json, applications
+SELECT created, name, json, applications
 FROM environments
 WHERE name=? AND deleted=false
 ORDER BY version DESC
@@ -5385,7 +5382,7 @@ func (h *DBHandler) DBSelectEnvironmentAtTimestamp(ctx context.Context, tx *sql.
 
 	selectQuery := h.AdaptQuery(
 		`
-SELECT created, version, name, json, applications
+SELECT created, name, json, applications
 FROM environments
 WHERE name=? AND deleted=false AND created <= ? 
 ORDER BY version DESC
@@ -5418,7 +5415,7 @@ func (h *DBHandler) processEnvironmentRow(ctx context.Context, rows *sql.Rows) (
 	if rows.Next() {
 		//exhaustruct:ignore
 		row := DBEnvironmentRow{}
-		err := rows.Scan(&row.Created, &row.Version, &row.Name, &row.Config, &row.Applications)
+		err := rows.Scan(&row.Created, &row.Name, &row.Config, &row.Applications)
 		if err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return nil, nil
@@ -5448,7 +5445,6 @@ func (h *DBHandler) DBSelectEnvironmentsBatch(ctx context.Context, tx *sql.Tx, e
 		`
 SELECT
   environments.created AS created,
-  environments.version AS version,
   environments.name AS name,
   environments.json AS json,
   environments.applications AS applications
@@ -5498,7 +5494,7 @@ LIMIT ?
 	for rows.Next() {
 		//exhaustruct:ignore
 		row := DBEnvironmentRow{}
-		err := rows.Scan(&row.Created, &row.Version, &row.Name, &row.Config, &row.Applications)
+		err := rows.Scan(&row.Created, &row.Name, &row.Config, &row.Applications)
 		if err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return nil, nil
@@ -5516,7 +5512,7 @@ LIMIT ?
 
 // DBWriteEnvironment writes the env to the db
 // if the previousVersion is not supplied, it will do another request to get the environment.
-func (h *DBHandler) DBWriteEnvironment(ctx context.Context, tx *sql.Tx, environmentName string, environmentConfig config.EnvironmentConfig, applications []string, previousVersion *int64) error {
+func (h *DBHandler) DBWriteEnvironment(ctx context.Context, tx *sql.Tx, environmentName string, environmentConfig config.EnvironmentConfig, applications []string) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "DBWriteEnvironment")
 	defer span.Finish()
 
@@ -5531,17 +5527,8 @@ func (h *DBHandler) DBWriteEnvironment(ctx context.Context, tx *sql.Tx, environm
 	if err != nil {
 		return fmt.Errorf("error while marshalling the environment config %v, error: %w", environmentConfig, err)
 	}
-	var existingEnvironmentVersion int64 = 0
-	if previousVersion == nil {
-		existingEnvironment, err := h.DBSelectEnvironment(ctx, tx, environmentName)
-		if err != nil {
-			return fmt.Errorf("error while selecting environment %s from database, error: %w", environmentName, err)
-		}
-		if existingEnvironment != nil {
-			existingEnvironmentVersion = existingEnvironment.Version
-		}
-	} else {
-		existingEnvironmentVersion = *previousVersion
+	if err != nil {
+		return fmt.Errorf("error while selecting environment %s from database, error: %w", environmentName, err)
 	}
 
 	slices.Sort(applications) // we don't really *need* the sorting, it's just for convenience
@@ -5551,7 +5538,7 @@ func (h *DBHandler) DBWriteEnvironment(ctx context.Context, tx *sql.Tx, environm
 	}
 
 	insertQuery := h.AdaptQuery(
-		"INSERT Into environments (created, version, name, json, applications, deleted) VALUES (?, ?, ?, ?, ?, ?);",
+		"INSERT Into environments (created, name, json, applications, deleted) VALUES (?, ?, ?, ?, ?);",
 	)
 	now, err := h.DBReadTransactionTimestamp(ctx, tx)
 	if err != nil {
@@ -5561,7 +5548,6 @@ func (h *DBHandler) DBWriteEnvironment(ctx context.Context, tx *sql.Tx, environm
 	_, err = tx.Exec(
 		insertQuery,
 		*now,
-		existingEnvironmentVersion+1,
 		environmentName,
 		jsonToInsert,
 		string(applicationsJson),
@@ -5759,7 +5745,7 @@ func (h *DBHandler) RunCustomMigrationEnvironments(ctx context.Context, getAllEn
 			if allEnvsApps[envName] == nil {
 				allEnvsApps[envName] = make([]string, 0)
 			}
-			err = h.DBWriteEnvironment(ctx, transaction, envName, config, allEnvsApps[envName], nil)
+			err = h.DBWriteEnvironment(ctx, transaction, envName, config, allEnvsApps[envName])
 			if err != nil {
 				return fmt.Errorf("unable to write manifest for environment %s to the database, error: %w", envName, err)
 			}
@@ -5813,7 +5799,7 @@ func (h *DBHandler) RunCustomMigrationEnvironmentApplications(ctx context.Contex
 				if allEnvsApps[envName] == nil {
 					allEnvsApps[envName] = make([]string, 0)
 				}
-				err = h.DBWriteEnvironment(ctx, transaction, envName, env.Config, allEnvsApps[envName], &env.Version)
+				err = h.DBWriteEnvironment(ctx, transaction, envName, env.Config, allEnvsApps[envName])
 				if err != nil {
 					return fmt.Errorf("unable to write manifest for environment %s to the database, error: %w", envName, err)
 				}

diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go
@@ -2612,7 +2612,6 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "development",
 			ExpectedEntry: &DBEnvironment{
-				Version:      1,
 				Name:         "development",
 				Config:       testutil.MakeEnvConfigLatest(nil),
 				Applications: []string{"app1", "app2", "app3"},
@@ -2629,7 +2628,6 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "development",
 			ExpectedEntry: &DBEnvironment{
-				Version:      1,
 				Name:         "development",
 				Config:       testutil.MakeEnvConfigLatestWithGroup(nil, conversion.FromString("development-group")),
 				Applications: []string{"app1"},
@@ -2649,9 +2647,8 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "development",
 			ExpectedEntry: &DBEnvironment{
-				Version: 2,
-				Name:    "development",
-				Config:  testutil.MakeEnvConfigLatestWithGroup(nil, conversion.FromString("development-group")),
+				Name:   "development",
+				Config: testutil.MakeEnvConfigLatestWithGroup(nil, conversion.FromString("development-group")),
 			},
 		},
 		{
@@ -2672,9 +2669,8 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "development",
 			ExpectedEntry: &DBEnvironment{
-				Version: 3,
-				Name:    "development",
-				Config:  testutil.MakeEnvConfigLatestWithGroup(nil, conversion.FromString("another-development-group")),
+				Name:   "development",
+				Config: testutil.MakeEnvConfigLatestWithGroup(nil, conversion.FromString("another-development-group")),
 			},
 		},
 		{
@@ -2691,9 +2687,8 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "staging",
 			ExpectedEntry: &DBEnvironment{
-				Version: 1,
-				Name:    "staging",
-				Config:  testutil.MakeEnvConfigUpstream("development", nil),
+				Name:   "staging",
+				Config: testutil.MakeEnvConfigUpstream("development", nil),
 			},
 		},
 		{
@@ -2722,7 +2717,6 @@ func TestReadWriteEnvironment(t *testing.T) {
 			},
 			EnvToQuery: "development",
 			ExpectedEntry: &DBEnvironment{
-				Version:      1,
 				Name:         "development",
 				Config:       testutil.MakeEnvConfigLatest(nil),
 				Applications: []string{"app1", "capp", "zapp"},
@@ -2738,7 +2732,7 @@ func TestReadWriteEnvironment(t *testing.T) {
 
 			for _, envToWrite := range tc.EnvsToWrite {
 				err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
-					err := dbHandler.DBWriteEnvironment(ctx, transaction, envToWrite.EnvironmentName, envToWrite.EnvironmentConfig, envToWrite.Applications, nil)
+					err := dbHandler.DBWriteEnvironment(ctx, transaction, envToWrite.EnvironmentName, envToWrite.EnvironmentConfig, envToWrite.Applications)
 					if err != nil {
 						return fmt.Errorf("error while writing environment, error: %w", err)
 					}
@@ -2802,13 +2796,11 @@ func TestReadEnvironmentBatch(t *testing.T) {
 			EnvsToQuery: []string{"development", "staging"},
 			ExpectedEnvs: &[]DBEnvironment{
 				{
-					Version:      1,
 					Name:         "development",
 					Config:       testutil.MakeEnvConfigLatest(nil),
 					Applications: []string{"app1", "app2", "app3"},
 				},
 				{
-					Version:      1,
 					Name:         "staging",
 					Config:       testutil.MakeEnvConfigLatest(nil),
 					Applications: []string{"app1", "app2", "app3"},
@@ -2837,13 +2829,11 @@ func TestReadEnvironmentBatch(t *testing.T) {
 			EnvsToQuery: []string{"development", "staging"},
 			ExpectedEnvs: &[]DBEnvironment{
 				{
-					Version:      2,
 					Name:         "development",
 					Config:       testutil.MakeEnvConfigLatest(nil),
 					Applications: []string{"app1", "app2"},
 				},
 				{
-					Version:      1,
 					Name:         "staging",
 					Config:       testutil.MakeEnvConfigLatest(nil),
 					Applications: []string{"app1", "app2", "app3"},
@@ -2860,7 +2850,7 @@ func TestReadEnvironmentBatch(t *testing.T) {
 
 			for _, envToWrite := range tc.EnvsToWrite {
 				err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
-					err := dbHandler.DBWriteEnvironment(ctx, transaction, envToWrite.EnvironmentName, envToWrite.EnvironmentConfig, envToWrite.Applications, nil)
+					err := dbHandler.DBWriteEnvironment(ctx, transaction, envToWrite.EnvironmentName, envToWrite.EnvironmentConfig, envToWrite.Applications)
 					if err != nil {
 						return fmt.Errorf("error while writing environment, error: %w", err)
 					}

diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go
@@ -2502,7 +2502,7 @@ func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction
 
 func (s *State) DBInsertEnvironmentWithOverview(ctx context.Context, tx *sql.Tx, environmentName string, environmentConfig config.EnvironmentConfig, applications []string) error {
 	h := s.DBHandler
-	err := h.DBWriteEnvironment(ctx, tx, environmentName, environmentConfig, applications, nil)
+	err := h.DBWriteEnvironment(ctx, tx, environmentName, environmentConfig, applications)
 	if err != nil {
 		return err
 	}

diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go
@@ -813,7 +813,7 @@ func (c *CreateApplicationVersion) Transform(
 				}
 			}
 			if envInfo != nil && !found {
-				err = state.DBHandler.DBWriteEnvironment(ctx, transaction, env, envInfo.Config, append(envInfo.Applications, c.Application), &envInfo.Version)
+				err = state.DBHandler.DBWriteEnvironment(ctx, transaction, env, envInfo.Config, append(envInfo.Applications, c.Application))
 				if err != nil {
 					return "", GetCreateReleaseGeneralFailure(err)
 				}
@@ -1701,7 +1701,7 @@ func (u *UndeployApplication) Transform(
 				}
 			}
 			t.AddAppEnv(u.Application, envName, dbApp.Metadata.Team)
-			err = state.DBHandler.DBWriteEnvironment(ctx, transaction, envName, env.Config, newEnvApps, &env.Version)
+			err = state.DBHandler.DBWriteEnvironment(ctx, transaction, envName, env.Config, newEnvApps)
 			if err != nil {
 				return "", fmt.Errorf("UndeployApplication: could not write environment: %v", err)
 			}
@@ -2808,19 +2808,13 @@ func (c *CreateEnvironment) Transform(
 	}
 	if state.DBHandler.ShouldUseOtherTables() {
 		// first read the env to see if it has applications:
-		existingEnvironment, err := state.DBHandler.DBSelectEnvironment(ctx, transaction, c.Environment)
 		if err != nil {
 			return "", fmt.Errorf("could not select environment %s from database, error: %w", c.Environment, err)
 		}
 
 		// write to environments table
 		environmentApplications := make([]string, 0)
-		var version *int64
-		if existingEnvironment != nil {
-			environmentApplications = existingEnvironment.Applications
-			version = &existingEnvironment.Version
-		}
-		err = state.DBHandler.DBWriteEnvironment(ctx, transaction, c.Environment, c.Config, environmentApplications, version)
+		err = state.DBHandler.DBWriteEnvironment(ctx, transaction, c.Environment, c.Config, environmentApplications)
 		if err != nil {
 			return "", fmt.Errorf("unable to write to the environment table, error: %w", err)
 		}

diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go
@@ -822,15 +822,15 @@ func TestReleaseTrain(t *testing.T) {
 						Environment: "staging",
 						Latest:      true,
 					},
-				}, []string{appName}, nil)
+				}, []string{appName})
 				if err != nil {
 					return err
 				}
 				err = dbHandler.DBWriteEnvironment(ctx, transaction, "production", config.EnvironmentConfig{
 					Upstream: &config.EnvironmentConfigUpstream{
 						Environment: "staging",
 					},
-				}, []string{appName}, nil)
+				}, []string{appName})
 				if err != nil {
 					return err
 				}
@@ -1710,7 +1710,7 @@ func TestCreateUndeployApplicationVersion(t *testing.T) {
 					ArgoCd:           nil,
 					EnvironmentGroup: nil,
 				}
-				err = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envConfig, []string{}, nil)
+				err = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envConfig, []string{})
 				if err != nil {
 					return err
 				}
@@ -2578,11 +2578,11 @@ func TestCreateUndeployLogic(t *testing.T) {
 					t.Fatal(err2)
 				}
 
-				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envAcceptanceConfig, []string{appName}, nil)
+				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envAcceptanceConfig, []string{appName})
 				if err2 != nil {
 					return err2
 				}
-				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, envAcceptance2Config, []string{appName}, nil)
+				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, envAcceptance2Config, []string{appName})
 				if err2 != nil {
 					return err2
 				}
@@ -2999,11 +2999,11 @@ func TestUndeployLogic(t *testing.T) {
 				if err2 != nil {
 					t.Fatal(err2)
 				}
-				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, environmentConfigAcceptance, []string{appName}, nil)
+				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, environmentConfigAcceptance, []string{appName})
 				if err2 != nil {
 					return err2
 				}
-				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, environmentConfigAcceptance2, []string{appName}, nil)
+				err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, environmentConfigAcceptance2, []string{appName})
 				if err2 != nil {
 					return err2
 				}