Merge branch 'master' into master

deploy/clowdapp.yaml

@@ -150,9 +150,6 @@ objects:
           - name: RBAC_HOST
             value: ${RBAC_HOST}
 
-          - name: STORAGE_MAX_CONCURRENCY
-            value: ${STORAGE_MAX_CONCURRENCY}
-
           - name: TENANT_TRANSLATOR_IMPL
             value: ${TENANT_TRANSLATOR_IMPL}
           - name: TENANT_TRANSLATOR_HOST
@@ -265,6 +262,10 @@ objects:
             value: ${LOG_LEVEL}
           - name: DB_SSLMODE
             value: ${DB_SSLMODE}
+          - name: STORAGE_MAX_CONCURRENCY
+            value: ${STORAGE_MAX_CONCURRENCY}
+          - name: ARTIFACT_MAX_SIZE
+            value: ${ARTIFACT_MAX_SIZE}
         resources:
           limits:
             cpu: ${CPU_LIMIT}
@@ -341,6 +342,8 @@ parameters:
 
 - name: STORAGE_MAX_CONCURRENCY
   value: "5"
+- name: ARTIFACT_MAX_SIZE
+  value: '3145728'
 
 - name: RETURN_URL
   value: TBD

internal/validator/handler.go

@@ -163,23 +163,86 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da
 	events = &messageModel.ValidatedMessages{}
 	events.PlaybookType = requestType
 
+	log := utils.GetLogFromContext(ctx)
+
+	// FIXME:  make this configurable
+	truncateData := len(data) >= 1*1024*1024
+	if truncateData {
+		log.Debugw("Payload too big.  Truncating payload.")
+	}
+
+	eventTypeCount := make(map[string]int)
+	hostRunningPlaybook := make(map[string]int)
+
 	lines := strings.Split(string(data), "\n")
 
 	for _, line := range lines {
 		if len(strings.TrimSpace(line)) == 0 {
 			continue
 		}
+
 		if requestType == playbookSatPayloadHeaderValue {
-			err = validateWithSchema(ctx, this.schemas[1], true, line, events)
+			validatedEvent, err := validateSatRunResponseWithSchema(ctx, this.schemas[1], line)
+
 			if err == nil {
-				err = validateSatHostUUID(line)
+				err = validateSatHostUUID(validatedEvent)
+			}
+
+			if err != nil {
+				return nil, err
+			}
+
+			var storeEvent bool = true
+
+			if truncateData {
+
+				storeEvent = false
+
+				if validatedEvent.Type == "playbook_run_completed" || validatedEvent.Type == "playbook_run_finished" {
+					storeEvent = true
+					log.Debugw("storing run complete/finished for host " /*, *validatedEvent.Host*/)
+				}
+
+				if validatedEvent.Type == "playbook_run_update" {
+
+					if validatedEvent.Host != nil {
+						if _, ok := hostRunningPlaybook[*validatedEvent.Host]; !ok {
+							storeEvent = true
+							log.Debugw("storing run update for host ", *validatedEvent.Host)
+						}
+
+						hostRunningPlaybook[*validatedEvent.Host]++
+					}
+				}
 			}
+
+			if storeEvent {
+				log.Debugw("storing event ", validatedEvent.Type)
+				events.PlaybookSat = append(events.PlaybookSat, *validatedEvent)
+			}
+
 		} else {
-			err = validateWithSchema(ctx, this.schemas[0], false, line, events)
-		}
+			validatedEvent, err := validateRunResponseWithSchema(ctx, this.schemas[0], line)
+			if err != nil {
+				return nil, err
+			}
 
-		if err != nil {
-			return nil, err
+			var storeEvent bool = true
+
+			if truncateData {
+				storeEvent = false
+
+				// FIXME:  hardcoded  :(
+				if validatedEvent.Event == "executor_on_start" || validatedEvent.Event == "playbook_on_stats" || validatedEvent.Event == "runner_on_failed" || validatedEvent.Event == "executor_on_failed" {
+					eventTypeCount[validatedEvent.Event]++
+					storeEvent = eventTypeCount[validatedEvent.Event] <= 1
+				}
+			}
+
+			if storeEvent {
+				log.Debugw("storing event ", validatedEvent.Event)
+				events.Playbook = append(events.Playbook, *validatedEvent)
+			}
 		}
 
 	}
@@ -191,14 +254,7 @@ func (this *handler) validateContent(ctx context.Context, requestType string, da
 	return events, nil
 }
 
-func validateSatHostUUID(line string) (err error) {
-	event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{}
-	err = json.Unmarshal([]byte(line), &event)
-
-	if err != nil {
-		return err
-	}
-
+func validateSatHostUUID(event *messageModel.PlaybookSatRunResponseMessageYamlEventsElem) (err error) {
 	if event.Host != nil {
 		_, err = uuid.Parse(*event.Host)
 		if err != nil {
@@ -208,33 +264,40 @@ func validateSatHostUUID(line string) (err error) {
 	return nil
 }
 
-func validateWithSchema(ctx context.Context, schema *jsonschema.Schema, rhcsatRequest bool, line string, events *messageModel.ValidatedMessages) (err error) {
+func validateRunResponseWithSchema(ctx context.Context, schema *jsonschema.Schema, line string) (validatedEvent *messageModel.PlaybookRunResponseMessageYamlEventsElem, err error) {
+
 	errors, parserError := schema.ValidateBytes(ctx, []byte(line))
 	if parserError != nil {
-		return parserError
+		return nil, parserError
 	} else if len(errors) > 0 {
-		return errors[0]
+		return nil, errors[0]
 	}
 
-	if rhcsatRequest {
-		event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{}
-		err = json.Unmarshal([]byte(line), &event)
-		if err != nil {
-			return err
-		}
+	event := &messageModel.PlaybookRunResponseMessageYamlEventsElem{}
+	err = json.Unmarshal([]byte(line), &event)
+	if err != nil {
+		return nil, err
+	}
 
-		events.PlaybookSat = append(events.PlaybookSat, *event)
-		return
+	return event, nil
+}
+
+func validateSatRunResponseWithSchema(ctx context.Context, schema *jsonschema.Schema, line string) (validatedEvent *messageModel.PlaybookSatRunResponseMessageYamlEventsElem, err error) {
+
+	errors, parserError := schema.ValidateBytes(ctx, []byte(line))
+	if parserError != nil {
+		return nil, parserError
+	} else if len(errors) > 0 {
+		return nil, errors[0]
 	}
 
-	event := &messageModel.PlaybookRunResponseMessageYamlEventsElem{}
+	event := &messageModel.PlaybookSatRunResponseMessageYamlEventsElem{}
 	err = json.Unmarshal([]byte(line), &event)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	events.Playbook = append(events.Playbook, *event)
-	return
+	return event, nil
 }
 
 func (this *handler) validationFailed(ctx context.Context, err error, requestType string, request *messageModel.IngressValidationRequest) {