From 23b5e0d00ee1c369e89d013c9fc2b3c247be4861 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Wed, 4 Sep 2024 10:28:55 -0500 Subject: [PATCH 1/8] Testing --- internal/validator/handler.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/validator/handler.go b/internal/validator/handler.go index 8fee79ac..57de7927 100644 --- a/internal/validator/handler.go +++ b/internal/validator/handler.go @@ -163,12 +163,20 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da events = &messageModel.ValidatedMessages{} events.PlaybookType = requestType + truncateData := len(data) >= 1*1024*1024 + lines := strings.Split(string(data), "\n") - for _, line := range lines { + for i, line := range lines { if len(strings.TrimSpace(line)) == 0 { continue } + + if truncateData && i > 2 && i < len(lines)-2 { + fmt.Println("Discarding line of output data...") + continue + } + if requestType == playbookSatPayloadHeaderValue { err = validateWithSchema(ctx, this.schemas[1], true, line, events) if err == nil { From c794588f1c0791d7dcb657dd01cfe00010777dc2 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Wed, 4 Sep 2024 10:52:24 -0500 Subject: [PATCH 2/8] Linter seems broken, try to upgrade it?? --- .github/workflows/golangci-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d2455795..64ed0621 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -16,12 +16,12 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.18 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: latest From f198f3c86cee74edf8f11fb4bc92f1c4dfc7858e Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Wed, 4 Sep 2024 10:57:32 -0500 Subject: [PATCH 3/8] Try this --- .github/workflows/pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5033fc18..c4683d95 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -11,7 +11,7 @@ jobs: container: node:12-slim steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: npx @stoplight/spectral@6.0.0 lint schema/public.openapi.yaml - run: npx @stoplight/spectral@6.0.0 lint schema/private.openapi.yaml @@ -32,7 +32,7 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-go@v1 with: @@ -55,7 +55,7 @@ jobs: java: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Cache Maven dependencies uses: actions/cache@v1 From a94ccc163365632481bea460cc07d04c497f6dd1 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Wed, 4 Sep 2024 11:02:24 -0500 Subject: [PATCH 4/8] Try removing broken things?? --- .github/workflows/pr.yml | 73 ---------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml deleted file mode 100644 index c4683d95..00000000 --- a/.github/workflows/pr.yml +++ /dev/null @@ -1,73 +0,0 @@ - ---- -name: Validation - -on: -- pull_request - -jobs: - lint: - runs-on: ubuntu-latest - container: node:12-slim - - steps: - - uses: actions/checkout@v4 - - run: npx @stoplight/spectral@6.0.0 lint schema/public.openapi.yaml - - run: npx @stoplight/spectral@6.0.0 lint schema/private.openapi.yaml - - golang: - runs-on: ubuntu-20.04 - services: - postgres: - image: postgres - env: - POSTGRES_DB: insights - POSTGRES_USER: insights - POSTGRES_PASSWORD: insights - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v1 - with: - go-version: 1.18 - - - run: make build - - run: go vet . - - - run: CLOWDER_ENABLED=false make test - - - name: test down migrations - run: make migrate-db-undo-all - - # Check that the code generated from the OpenAPI schema is up-to-date (i.e. make generate-api has been run after changing the openapi schema) - - name: test openapi schema up to date - run: make init && cp internal/api/controllers/private/spec.gen.go internal/api/controllers/private/spec.gen.go.bak && make generate-api && diff internal/api/controllers/private/spec.gen.go internal/api/controllers/private/spec.gen.go.bak - env: - GOPATH: /home/runner/go - - java: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Cache Maven dependencies - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - - name: Build with Maven - run: mvn -f event-streams/pom.xml clean verify From d6d9aeda6f3f0561e21992db3137554903932a5b Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Wed, 4 Sep 2024 23:28:54 -0500 Subject: [PATCH 5/8] Very ugly --- internal/validator/handler.go | 136 +++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/internal/validator/handler.go b/internal/validator/handler.go index 57de7927..ad7bc443 100644 --- a/internal/validator/handler.go +++ b/internal/validator/handler.go @@ -164,30 +164,96 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da events.PlaybookType = requestType truncateData := len(data) >= 1*1024*1024 + if truncateData { + utils.GetLogFromContext(ctx).Debugw("Payload too big. Truncating payload.") + } + + eventTypeCount := make(map[string]int) + hostRunningPlaybook := make(map[string]int) lines := strings.Split(string(data), "\n") - for i, line := range lines { + for _, line := range lines { if len(strings.TrimSpace(line)) == 0 { continue } - if truncateData && i > 2 && i < len(lines)-2 { - fmt.Println("Discarding line of output data...") - continue - } - if requestType == playbookSatPayloadHeaderValue { - err = validateWithSchema(ctx, this.schemas[1], true, line, events) + validatedEvent, err := validateSatRunResponseWithSchema(ctx, this.schemas[1], line) + if err == nil { - err = validateSatHostUUID(line) + err = validateSatHostUUID(validatedEvent) + } + + if err != nil { + return nil, err } + + var storeEvent bool = true + + if truncateData { + + storeEvent = false + /* + EventSatPlaybookFinished = "playbook_run_finished" + EventSatPlaybookCompleted = "playbook_run_completed" + */ + + if validatedEvent.Type == "playbook_run_completed" || validatedEvent.Type == "playbook_run_finished" { + storeEvent = true + fmt.Println("storing run complete/finished for host " /*, *validatedEvent.Host*/) + } + + /* + , "version": 3, "correlation_id": "00000000-0000-0000-0000-000000000000", "host": "aec36412-5918-45f1-a66b-5560a265bfdd", "status": "success", "connection_code": 0, "execution_code": 0} + */ + + if validatedEvent.Type == "playbook_run_update" { + + if validatedEvent.Host != nil { + if _, ok := hostRunningPlaybook[*validatedEvent.Host]; !ok { + storeEvent = true + fmt.Println("storing run update for host ", *validatedEvent.Host) + } else { + fmt.Println("discarding run update for host ", *validatedEvent.Host) + } + + hostRunningPlaybook[*validatedEvent.Host]++ + } + } + /* + {"type": "playbook_run_update", "version": 3, "correlation_id": "00000000-0000-0000-0000-000000000000", "sequence": 1, "host": "aec36412-5918-45f1-a66b-5560a265bfdd", "console": "aec36412-5918-45f1-a66b-5560a265bfdd | SUCCESS => {\n \"changed\": false,\n \"ping\": \"pong\"\n}"} + */ + + } + + if storeEvent { + events.PlaybookSat = append(events.PlaybookSat, *validatedEvent) + } + } else { - err = validateWithSchema(ctx, this.schemas[0], false, line, events) - } + validatedEvent, err := validateRunResponseWithSchema(ctx, this.schemas[0], line) + if err != nil { + return nil, err + } - if err != nil { - return nil, err + var storeEvent bool = true + + if truncateData { + storeEvent = false + fmt.Println("validateEvent.Event: ", validatedEvent.Event) + + // FIXME: hardcoded :( + if validatedEvent.Event == "executor_on_start" || validatedEvent.Event == "playbook_on_stats" || validatedEvent.Event == "runner_on_failed" || validatedEvent.Event == "executor_on_failed" { + eventTypeCount[validatedEvent.Event]++ + storeEvent = eventTypeCount[validatedEvent.Event] <= 1 + fmt.Println("storeEvent: ", storeEvent) + } + } + + if storeEvent { + events.Playbook = append(events.Playbook, *validatedEvent) + } } } @@ -199,14 +265,7 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da return events, nil } -func validateSatHostUUID(line string) (err error) { - event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{} - err = json.Unmarshal([]byte(line), &event) - - if err != nil { - return err - } - +func validateSatHostUUID(event *messageModel.PlaybookSatRunResponseMessageYamlEventsElem) (err error) { if event.Host != nil { _, err = uuid.Parse(*event.Host) if err != nil { @@ -216,33 +275,40 @@ func validateSatHostUUID(line string) (err error) { return nil } -func validateWithSchema(ctx context.Context, schema *jsonschema.Schema, rhcsatRequest bool, line string, events *messageModel.ValidatedMessages) (err error) { +func validateRunResponseWithSchema(ctx context.Context, schema *jsonschema.Schema, line string) (validatedEvent *messageModel.PlaybookRunResponseMessageYamlEventsElem, err error) { + errors, parserError := schema.ValidateBytes(ctx, []byte(line)) if parserError != nil { - return parserError + return nil, parserError } else if len(errors) > 0 { - return errors[0] + return nil, errors[0] } - if rhcsatRequest { - event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{} - err = json.Unmarshal([]byte(line), &event) - if err != nil { - return err - } + event := &messageModel.PlaybookRunResponseMessageYamlEventsElem{} + err = json.Unmarshal([]byte(line), &event) + if err != nil { + return nil, err + } - events.PlaybookSat = append(events.PlaybookSat, *event) - return + return event, nil +} + +func validateSatRunResponseWithSchema(ctx context.Context, schema *jsonschema.Schema, line string) (validatedEvent *messageModel.PlaybookSatRunResponseMessageYamlEventsElem, err error) { + + errors, parserError := schema.ValidateBytes(ctx, []byte(line)) + if parserError != nil { + return nil, parserError + } else if len(errors) > 0 { + return nil, errors[0] } - event := &messageModel.PlaybookRunResponseMessageYamlEventsElem{} + event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{} err = json.Unmarshal([]byte(line), &event) if err != nil { - return err + return nil, err } - events.Playbook = append(events.Playbook, *event) - return + return event, nil } func (this *handler) validationFailed(ctx context.Context, err error, requestType string, request *messageModel.IngressValidationRequest) { From 80f0a579d4dfb075feb439cde2f91959d7dbf4f5 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Thu, 5 Sep 2024 12:36:12 -0500 Subject: [PATCH 6/8] Minor clean...needs more cleanup/refactoring...try it on real data first?? --- internal/validator/handler.go | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/internal/validator/handler.go b/internal/validator/handler.go index ad7bc443..63d7798d 100644 --- a/internal/validator/handler.go +++ b/internal/validator/handler.go @@ -163,9 +163,12 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da events = &messageModel.ValidatedMessages{} events.PlaybookType = requestType + log := utils.GetLogFromContext(ctx) + + // FIXME: make this configurable truncateData := len(data) >= 1*1024*1024 if truncateData { - utils.GetLogFromContext(ctx).Debugw("Payload too big. Truncating payload.") + log.Debugw("Payload too big. Truncating payload.") } eventTypeCount := make(map[string]int) @@ -194,40 +197,27 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da if truncateData { storeEvent = false - /* - EventSatPlaybookFinished = "playbook_run_finished" - EventSatPlaybookCompleted = "playbook_run_completed" - */ if validatedEvent.Type == "playbook_run_completed" || validatedEvent.Type == "playbook_run_finished" { storeEvent = true - fmt.Println("storing run complete/finished for host " /*, *validatedEvent.Host*/) + log.Debugw("storing run complete/finished for host " /*, *validatedEvent.Host*/) } - /* - , "version": 3, "correlation_id": "00000000-0000-0000-0000-000000000000", "host": "aec36412-5918-45f1-a66b-5560a265bfdd", "status": "success", "connection_code": 0, "execution_code": 0} - */ - if validatedEvent.Type == "playbook_run_update" { if validatedEvent.Host != nil { if _, ok := hostRunningPlaybook[*validatedEvent.Host]; !ok { storeEvent = true - fmt.Println("storing run update for host ", *validatedEvent.Host) - } else { - fmt.Println("discarding run update for host ", *validatedEvent.Host) + log.Debugw("storing run update for host ", *validatedEvent.Host) } hostRunningPlaybook[*validatedEvent.Host]++ } } - /* - {"type": "playbook_run_update", "version": 3, "correlation_id": "00000000-0000-0000-0000-000000000000", "sequence": 1, "host": "aec36412-5918-45f1-a66b-5560a265bfdd", "console": "aec36412-5918-45f1-a66b-5560a265bfdd | SUCCESS => {\n \"changed\": false,\n \"ping\": \"pong\"\n}"} - */ - } if storeEvent { + log.Debugw("storing event ", validatedEvent.Type) events.PlaybookSat = append(events.PlaybookSat, *validatedEvent) } @@ -241,17 +231,16 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da if truncateData { storeEvent = false - fmt.Println("validateEvent.Event: ", validatedEvent.Event) // FIXME: hardcoded :( if validatedEvent.Event == "executor_on_start" || validatedEvent.Event == "playbook_on_stats" || validatedEvent.Event == "runner_on_failed" || validatedEvent.Event == "executor_on_failed" { eventTypeCount[validatedEvent.Event]++ storeEvent = eventTypeCount[validatedEvent.Event] <= 1 - fmt.Println("storeEvent: ", storeEvent) } } if storeEvent { + log.Debugw("storing event ", validatedEvent.Event) events.Playbook = append(events.Playbook, *validatedEvent) } } From b73ae4bf8dbf66c70949bfa9627390fb2dacf40f Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Thu, 5 Sep 2024 12:57:08 -0500 Subject: [PATCH 7/8] I think the STORAGE_MAX_CONCURRENCY setting is in the wrong place. The api doesn't use it, the validator does. No biggie, just move it, fix it, etc Increase the max artifact size --- deploy/clowdapp.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 4b300f3e..e83b155b 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -150,9 +150,6 @@ objects: - name: RBAC_HOST value: ${RBAC_HOST} - - name: STORAGE_MAX_CONCURRENCY - value: ${STORAGE_MAX_CONCURRENCY} - - name: TENANT_TRANSLATOR_IMPL value: ${TENANT_TRANSLATOR_IMPL} - name: TENANT_TRANSLATOR_HOST @@ -265,6 +262,10 @@ objects: value: ${LOG_LEVEL} - name: DB_SSLMODE value: ${DB_SSLMODE} + - name: STORAGE_MAX_CONCURRENCY + value: ${STORAGE_MAX_CONCURRENCY} + - name: ARTIFACT_MAX_SIZE + value: ${ARTIFACT_MAX_SIZE} resources: limits: cpu: ${CPU_LIMIT} @@ -341,6 +342,8 @@ parameters: - name: STORAGE_MAX_CONCURRENCY value: "5" +- name: ARTIFACT_MAX_SIZE + value: '3145728' - name: RETURN_URL value: TBD From 59792efa8e2efefb588f599c398c688274b0c101 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Thu, 5 Sep 2024 13:05:40 -0500 Subject: [PATCH 8/8] Ignore the github workflow changes for now --- .github/workflows/golangci-lint.yml | 6 +-- .github/workflows/pr.yml | 73 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 64ed0621..d2455795 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -16,12 +16,12 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v3 with: go-version: 1.18 - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: latest diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..5033fc18 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,73 @@ + +--- +name: Validation + +on: +- pull_request + +jobs: + lint: + runs-on: ubuntu-latest + container: node:12-slim + + steps: + - uses: actions/checkout@v2 + - run: npx @stoplight/spectral@6.0.0 lint schema/public.openapi.yaml + - run: npx @stoplight/spectral@6.0.0 lint schema/private.openapi.yaml + + golang: + runs-on: ubuntu-20.04 + services: + postgres: + image: postgres + env: + POSTGRES_DB: insights + POSTGRES_USER: insights + POSTGRES_PASSWORD: insights + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v1 + with: + go-version: 1.18 + + - run: make build + - run: go vet . + + - run: CLOWDER_ENABLED=false make test + + - name: test down migrations + run: make migrate-db-undo-all + + # Check that the code generated from the OpenAPI schema is up-to-date (i.e. make generate-api has been run after changing the openapi schema) + - name: test openapi schema up to date + run: make init && cp internal/api/controllers/private/spec.gen.go internal/api/controllers/private/spec.gen.go.bak && make generate-api && diff internal/api/controllers/private/spec.gen.go internal/api/controllers/private/spec.gen.go.bak + env: + GOPATH: /home/runner/go + + java: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Cache Maven dependencies + uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Build with Maven + run: mvn -f event-streams/pom.xml clean verify