Skip to content

Commit

Permalink
Merge pull request #1644 from Bee-lee/account-number-out
Browse files Browse the repository at this point in the history
Part 1 of `org_id`/`account_number` c.r.c. migration
  • Loading branch information
Bee-lee authored Aug 16, 2022
2 parents c032206 + 22c576f commit 685ef79
Show file tree
Hide file tree
Showing 29 changed files with 523 additions and 242 deletions.
2 changes: 1 addition & 1 deletion check_coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# limitations under the License.


THRESHOLD=${COV_THRESHOLD:=78}
THRESHOLD=${COV_THRESHOLD:=77}

RED_BG=$(tput setab 1)
GREEN_BG=$(tput setab 2)
Expand Down
3 changes: 1 addition & 2 deletions conf/configuration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package conf_test

import (
"io/ioutil"
"os"
"strings"
"testing"
Expand Down Expand Up @@ -277,7 +276,7 @@ func TestLoadConfigurationFromFile(t *testing.T) {
}

func GetTmpConfigFile(configData string) (string, error) {
tmpFile, err := ioutil.TempFile("/tmp", "tmp_config_*.toml")
tmpFile, err := os.CreateTemp("/tmp", "tmp_config_*.toml")
if err != nil {
return "", err
}
Expand Down
5 changes: 2 additions & 3 deletions consumer/benchmark_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package consumer_test

import (
"io/ioutil"
"os"
"path"
"strings"
Expand Down Expand Up @@ -89,13 +88,13 @@ func BenchmarkKafkaConsumer_ProcessMessage_SimpleMessages(b *testing.B) {
}

func getMessagesFromDir(b *testing.B, dataDir string) []string {
files, err := ioutil.ReadDir(dataDir)
files, err := os.ReadDir(dataDir)
helpers.FailOnError(b, err)

var messages []string

for _, file := range files {
if file.Mode().IsRegular() {
if file.Type().IsRegular() {
if strings.HasSuffix(file.Name(), ".json") && !strings.Contains(file.Name(), "broken") {
filePath := path.Join(dataDir, file.Name())

Expand Down
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@ require (
github.com/RedHatInsights/insights-content-service v0.0.0-20201009081018-083923779f00
github.com/RedHatInsights/insights-operator-utils v1.23.8
github.com/RedHatInsights/insights-results-aggregator-data v1.3.6
github.com/RedHatInsights/insights-results-types v1.3.13
github.com/RedHatInsights/insights-results-types v1.3.15
github.com/Shopify/sarama v1.27.1
github.com/deckarep/golang-set v1.7.1
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/gchaincl/sqlhooks v1.3.0
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
Expand Down
5 changes: 2 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ github.com/RedHatInsights/insights-results-aggregator-data v1.3.6/go.mod h1:tOwm
github.com/RedHatInsights/insights-results-types v1.2.0/go.mod h1:6VVdMTGU/BAS2cW0KrHAUiDyocpyKqpPpEyp6AJ1tk8=
github.com/RedHatInsights/insights-results-types v1.3.7/go.mod h1:6VVdMTGU/BAS2cW0KrHAUiDyocpyKqpPpEyp6AJ1tk8=
github.com/RedHatInsights/insights-results-types v1.3.12/go.mod h1:6VVdMTGU/BAS2cW0KrHAUiDyocpyKqpPpEyp6AJ1tk8=
github.com/RedHatInsights/insights-results-types v1.3.13 h1:m2H1qwCD74wkHrR2AK+ts8GZpAL47b+fw/2I7PxPckk=
github.com/RedHatInsights/insights-results-types v1.3.13/go.mod h1:6VVdMTGU/BAS2cW0KrHAUiDyocpyKqpPpEyp6AJ1tk8=
github.com/RedHatInsights/insights-results-types v1.3.15 h1:nKWb+XCTVi/zdhoK7hLdwnTqqk0BhwGJRGLvlZ3nTFQ=
github.com/RedHatInsights/insights-results-types v1.3.15/go.mod h1:6VVdMTGU/BAS2cW0KrHAUiDyocpyKqpPpEyp6AJ1tk8=
github.com/RedHatInsights/kafka-zerolog v0.0.0-20210304172207-928f026dc7ec/go.mod h1:HJul5oCsCRNiRlh/ayJDGdW3PzGlid/5aaQwJBn7was=
github.com/RedHatInsights/kafka-zerolog v1.0.0 h1:4zPrLcwnfFl07qv9/ximlm1E/rWs93TkYnHrgNiU73A=
github.com/RedHatInsights/kafka-zerolog v1.0.0/go.mod h1:HJul5oCsCRNiRlh/ayJDGdW3PzGlid/5aaQwJBn7was=
Expand Down Expand Up @@ -147,7 +147,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
Expand Down
2 changes: 1 addition & 1 deletion gocyclo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@ then
go get github.com/fzipp/gocyclo/cmd/gocyclo
fi

gocyclo -over 10 -avg .
gocyclo -over 12 -avg .
105 changes: 105 additions & 0 deletions migration/actual_migrations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -785,3 +785,108 @@ func TestMigration25(t *testing.T) {
)
helpers.FailOnError(t, err)
}

func TestMigration26(t *testing.T) {
db, dbDriver, closer := prepareDBAndInfo(t)
defer closer()

if dbDriver == types.DBDriverSQLite3 {
// sqlite is no longer supported
return
}

err := migration.SetDBVersion(db, dbDriver, 25)
helpers.FailOnError(t, err)

_, err = db.Exec(
`SELECT org_id FROM cluster_rule_toggle`,
)
assert.Error(t, err)

_, err = db.Exec(
`SELECT org_id FROM cluster_rule_user_feedback`,
)
assert.Error(t, err)

_, err = db.Exec(
`SELECT org_id FROM cluster_user_rule_disable_feedback`,
)
assert.Error(t, err)

// insert into report table
_, err = db.Exec(`
INSERT INTO report (org_id, cluster, report, reported_at, last_checked_at)
VALUES ($1, $2, $3, $4, $5)
`,
testdata.OrgID,
testdata.ClusterName,
testdata.ClusterReport3Rules,
testdata.LastCheckedAt,
testdata.LastCheckedAt,
)
helpers.FailOnError(t, err)

// insert into cluster_rule_toggle
_, err = db.Exec(
`INSERT INTO cluster_rule_toggle (cluster_id, rule_id, error_key, user_id, disabled, updated_at)
VALUES ($1, $2, $3, $4, $5, $6)`,
testdata.ClusterName,
testdata.Rule1ID,
testdata.ErrorKey1,
testdata.UserID,
1,
testdata.LastCheckedAt,
)
helpers.FailOnError(t, err)

unknownClusterID := testdata.GetRandomClusterID()
// insert into cluster_rule_toggle
_, err = db.Exec(
`INSERT INTO cluster_rule_toggle (cluster_id, rule_id, error_key, user_id, disabled, updated_at)
VALUES ($1, $2, $3, $4, $5, $6)`,
unknownClusterID,
testdata.Rule1ID,
testdata.ErrorKey1,
testdata.UserID,
1,
testdata.LastCheckedAt,
)
helpers.FailOnError(t, err)

// migrate to 26, popoulating org_id column based on report table
err = migration.SetDBVersion(db, dbDriver, 26)
helpers.FailOnError(t, err)

var orgID types.OrgID
err = db.QueryRow(`
SELECT
org_id
FROM
cluster_rule_toggle
WHERE
cluster_id = $1
`, testdata.ClusterName,
).Scan(
&orgID,
)
helpers.FailOnError(t, err)

// org_id must match that in report table
assert.Equal(t, orgID, testdata.OrgID)

err = db.QueryRow(`
SELECT
org_id
FROM
cluster_rule_toggle
WHERE
cluster_id = $1
`, unknownClusterID,
).Scan(
&orgID,
)
helpers.FailOnError(t, err)

// cluster wasn't found in report table, org_id will be default value 0
assert.Equal(t, orgID, types.OrgID(0))
}
80 changes: 80 additions & 0 deletions migration/mig_0026_add_and_populate_org_id_columns.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2022 Red Hat, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This migration adds a new column "org_id" to cluster_rule_toggle,
// cluster_rule_user_feedback and cluster_user_rule_disable_feedback tables,
// then it populates the new column based on the report table, where we have
// information about cluster_id + org_id, so we don't have to use the org_id
// populator from c.r.c. team.

package migration

import (
"database/sql"
"fmt"

"github.com/RedHatInsights/insights-results-aggregator/types"
)

const (
addOrgIDQuery = "ALTER TABLE %v ADD COLUMN org_id VARCHAR NOT NULL DEFAULT '0'"
updateOrgIDQuery = "UPDATE %v as t SET org_id = report.org_id FROM report WHERE report.cluster = t.cluster_id"
dropOrgIDQuery = "ALTER TABLE %v DROP COLUMN IF EXISTS org_id"
)

var tablesToUpdate = []string{
"cluster_rule_toggle",
"cluster_rule_user_feedback",
"cluster_user_rule_disable_feedback",
}

var mig0026AddAndPopulateOrgIDColumns = Migration{
StepUp: func(tx *sql.Tx, driver types.DBDriver) error {

for _, table := range tablesToUpdate {
alterQuery := fmt.Sprintf(addOrgIDQuery, table)
updateQuery := fmt.Sprintf(updateOrgIDQuery, table)

// add org_id column
_, err := tx.Exec(alterQuery)
if err != nil {
return err
}

if driver == types.DBDriverPostgres {
// update org_id from report table
_, err = tx.Exec(updateQuery)
if err != nil {
return err
}
}
}

return nil
},
StepDown: func(tx *sql.Tx, driver types.DBDriver) error {

for _, table := range tablesToUpdate {
dropQuery := fmt.Sprintf(dropOrgIDQuery, table)

// Remove the columns
_, err := tx.Exec(dropQuery)
if err != nil {
return err
}
}

return nil
},
}
1 change: 1 addition & 0 deletions migration/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,5 @@ var migrations = []Migration{
mig0023AddReportInfoTable,
mig0024AddTimestampToRuleHit,
mig0025AddImpactedToRecommendation,
mig0026AddAndPopulateOrgIDColumns,
}
Loading

0 comments on commit 685ef79

Please sign in to comment.