Skip to content

Commit

Permalink
feat: switch to Lagoon API DB for project group membership
Browse files Browse the repository at this point in the history
  • Loading branch information
@smlx
smlx committed Apr 19, 2024
1 parent f04bce9 commit 95bb341
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 118 deletions.
52 changes: 39 additions & 13 deletions internal/sync/indexpatterns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,29 @@ func calculateIndexPatternDiff(log *zap.Logger,

// generateIndexPatternsForGroup returns a slice of index patterns for all the
// projects associated with the given group.
func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
projectNames map[int]string, legacyDelimiter bool) ([]string, error) {
pids, err := projectIDsForGroup(group)
if err != nil {
return nil, fmt.Errorf("couldn't get project IDs for group: %v", err)
func generateIndexPatternsForGroup(
log *zap.Logger,
group keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
legacyDelimiter bool,
) ([]string, error) {
pids, ok := groupProjectsMap[group.ID]
if !ok {
return nil, fmt.Errorf("missing project group ID %s in groupProjectsMap",
group.ID)
}
var indexPatterns []string
for _, pid := range pids {
name, ok := projectNames[pid]
if !ok {
log.Debug("invalid project ID in lagoon-projects group attribute",
// If you see this warning it means that a project ID appears in the
// kc_group_projects table that does not appear in the projects table in
// the Lagoon API DB.
// This is likely a bug in Lagoon causing loss of referential integrity,
// as there is no foreign key constraint to enforce valid project IDs in
// the group mapping.
log.Warn("invalid project ID when generating index patterns",
zap.Int("projectID", pid))
continue
}
Expand All @@ -156,8 +168,13 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
//
// Only regular Lagoon groups are associated with a tenant (which is where
// index patterns are placed), so project groups are ignored.
func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,
projectNames map[int]string, legacyDelimiter bool) map[string]map[string]bool {
func generateIndexPatterns(
log *zap.Logger,
groups []keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
legacyDelimiter bool,
) map[string]map[string]bool {
indexPatterns := map[string]map[string]bool{}
var patterns []string
var err error
Expand All @@ -166,7 +183,7 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,
continue
}
patterns, err = generateIndexPatternsForGroup(log, group, projectNames,
legacyDelimiter)
groupProjectsMap, legacyDelimiter)
if err != nil {
log.Warn("couldn't generate index patterns for group",
zap.String("group", group.Name), zap.Error(err))
Expand All @@ -191,17 +208,26 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,

// syncIndexPatterns reconciles Opensearch Dashboards index patterns with
// Lagoon logging requirements.
func syncIndexPatterns(ctx context.Context, log *zap.Logger,
groups []keycloak.Group, projectNames map[int]string, o OpensearchService,
d DashboardsService, dryRun, legacyDelimiter bool) {
func syncIndexPatterns(
ctx context.Context,
log *zap.Logger,
groups []keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
o OpensearchService,
d DashboardsService,
dryRun,
legacyDelimiter bool,
) {
// get index patterns from Opensearch
existing, err := o.IndexPatterns(ctx)
if err != nil {
log.Error("couldn't get index patterns from Opensearch", zap.Error(err))
return
}
// generate the index patterns required by Lagoon
required := generateIndexPatterns(log, groups, projectNames, legacyDelimiter)
required := generateIndexPatterns(log, groups, projectNames,
groupProjectsMap, legacyDelimiter)
// calculate index templates to add/remove
toCreate, toDelete := calculateIndexPatternDiff(log, existing, required)
for tenant, patternIDMap := range toDelete {
Expand Down
54 changes: 25 additions & 29 deletions internal/sync/indexpatterns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ import (
)

type generateIndexPatternsForGroupInput struct {
group keycloak.Group
projectNames map[int]string
group keycloak.Group
projectNames map[int]string
groupProjectsMap map[string][]int
}

type generateIndexPatternsForGroupOutput struct {
Expand All @@ -31,10 +32,6 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
ID: "f6697da3-016a-43cd-ba9f-3f5b91b45302",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "drupal-example",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"drupal-example":[31,34,35]}`},
"lagoon-projects": {`31,34,35`},
},
},
},
projectNames: map[int]string{
Expand All @@ -43,6 +40,9 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
35: "drupal10-prerelease",
36: "delta-backend",
},
groupProjectsMap: map[string][]int{
"f6697da3-016a-43cd-ba9f-3f5b91b45302": {31, 34, 35},
},
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
Expand Down Expand Up @@ -71,10 +71,6 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "drupal-example2",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"drupal-example":[31,35,44]}`},
"lagoon-projects": {`31,35,44`},
},
},
},
projectNames: map[int]string{
Expand All @@ -83,6 +79,9 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
35: "drupal10-prerelease",
36: "delta-backend",
},
groupProjectsMap: map[string][]int{
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee": {31, 35, 44},
},
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
Expand All @@ -106,7 +105,7 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns, err := sync.GenerateIndexPatternsForGroup(log, tc.input.group,
tc.input.projectNames, false)
tc.input.projectNames, tc.input.groupProjectsMap, false)
if (err == nil && tc.expect.err != nil) ||
(err != nil && tc.expect.err == nil) {
tt.Fatalf("got err:\n%v\nexpected err:\n%v\n", err, tc.expect.err)
Expand Down Expand Up @@ -376,9 +375,10 @@ func TestCalculateIndexPatternDiff(t *testing.T) {

func TestGenerateIndexPatterns(t *testing.T) {
type generateIndexPatternsInput struct {
groups []keycloak.Group
projectNames map[int]string
legacyDelimiter bool
groups []keycloak.Group
projectNames map[int]string
groupProjectsMap map[string][]int
legacyDelimiter bool
}
var testCases = map[string]struct {
input generateIndexPatternsInput
Expand All @@ -391,27 +391,25 @@ func TestGenerateIndexPatterns(t *testing.T) {
ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "foocorp",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`},
"lagoon-projects": {`3133,34435`},
},
},
},
{
ID: "9f92af94-a7ee-4759-83bb-2b983bd30142",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "project-drupal12-base",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`},
"lagoon-projects": {`34435`},
"type": {`project-default-group`},
"type": {`project-default-group`},
},
},
},
},
projectNames: map[int]string{
34435: "drupal12-base",
},
groupProjectsMap: map[string][]int{
"08fef83d-cde7-43a5-8bd2-a18cf440214a": {3133, 34435},
"9f92af94-a7ee-4759-83bb-2b983bd30142": {34435},
},
legacyDelimiter: false,
},
expect: map[string]map[string]bool{
Expand Down Expand Up @@ -446,27 +444,25 @@ func TestGenerateIndexPatterns(t *testing.T) {
ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "foocorp",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`},
"lagoon-projects": {`3133,34435`},
},
},
},
{
ID: "9f92af94-a7ee-4759-83bb-2b983bd30142",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "project-drupal12-base",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`},
"lagoon-projects": {`34435`},
"type": {`project-default-group`},
"type": {`project-default-group`},
},
},
},
},
projectNames: map[int]string{
34435: "drupal12-base",
},
groupProjectsMap: map[string][]int{
"08fef83d-cde7-43a5-8bd2-a18cf440214a": {3133, 34435},
"9f92af94-a7ee-4759-83bb-2b983bd30142": {34435},
},
legacyDelimiter: true,
},
expect: map[string]map[string]bool{
Expand Down Expand Up @@ -499,7 +495,7 @@ func TestGenerateIndexPatterns(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns := sync.GenerateIndexPatterns(
log, tc.input.groups, tc.input.projectNames,
log, tc.input.groups, tc.input.projectNames, tc.input.groupProjectsMap,
tc.input.legacyDelimiter)
if !reflect.DeepEqual(indexPatterns, tc.expect) {
tt.Fatalf("got:\n%v\nexpected:\n%v\n", indexPatterns, tc.expect)
Expand Down
Loading

0 comments on commit 95bb341

Please sign in to comment.