From 1771a8e10da399010f040b828df4028c2374ec92 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Tue, 14 Jan 2025 17:26:33 +0530 Subject: [PATCH] chore: refactor types --- .../reporting_dropped_events_test.go | 5 +- .../reporting_error_index_test.go | 5 +- integration_test/tracing/tracing_test.go | 9 +- .../transformer_contract_test.go | 8 +- integration_test/warehouse/warehouse_test.go | 30 +- internal/enricher/enricher.go | 2 +- internal/enricher/geolocation.go | 2 +- internal/enricher/geolocation_test.go | 2 +- .../processor/transformer/mock_transformer.go | 14 +- processor/consent.go | 2 +- processor/consent_test.go | 6 +- processor/delayed/stats.go | 5 +- processor/delayed/stats_test.go | 9 +- processor/eventfilter/eventfilter.go | 19 +- processor/eventfilter/eventfilter_test.go | 20 +- processor/events_response.go | 10 +- processor/integrations/integrations.go | 4 +- .../destination_transformer.go | 12 +- .../trackingplan_validation.go | 9 +- .../user_transformer/user_transformer.go | 17 +- processor/processor.go | 285 ++++++++------- processor/processorBenchmark_test.go | 3 +- processor/processor_test.go | 345 +++++++++--------- processor/trackingplan.go | 33 +- processor/trackingplan_test.go | 50 +-- processor/transformer/transformer.go | 126 ++----- processor/transformer/transformer_test.go | 141 ++++--- processor/types/types.go | 122 +++++++ .../transformationStatusUploader.go | 15 +- .../transformationStatusUploader_test.go | 19 +- testhelper/transformertest/builder.go | 7 +- testhelper/transformertest/handler_funcs.go | 36 +- utils/misc/misc.go | 10 - utils/types/types.go | 24 -- warehouse/integrations/testhelper/staging.go | 6 +- 35 files changed, 725 insertions(+), 687 deletions(-) create mode 100644 processor/types/types.go diff --git a/integration_test/reporting_dropped_events/reporting_dropped_events_test.go b/integration_test/reporting_dropped_events/reporting_dropped_events_test.go index 4554c2dfb8..72b23d47ee 100644 --- a/integration_test/reporting_dropped_events/reporting_dropped_events_test.go +++ b/integration_test/reporting_dropped_events/reporting_dropped_events_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -25,7 +27,6 @@ import ( kithelper "github.com/rudderlabs/rudder-go-kit/testhelper" "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres" "github.com/rudderlabs/rudder-go-kit/testhelper/rand" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -120,7 +121,7 @@ func TestReportingDroppedEvents(t *testing.T) { transformertest.ViolationErrorTransformerHandler( http.StatusBadRequest, "tracking plan validation failed", - []transformer.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, + []types.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, ), ). Build() diff --git a/integration_test/reporting_error_index/reporting_error_index_test.go b/integration_test/reporting_error_index/reporting_error_index_test.go index 46249289d6..6e3eda931b 100644 --- a/integration_test/reporting_error_index/reporting_error_index_test.go +++ b/integration_test/reporting_error_index/reporting_error_index_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + _ "github.com/marcboeker/go-duckdb" "github.com/ory/dockertest/v3" "github.com/samber/lo" @@ -28,7 +30,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres" "github.com/rudderlabs/rudder-go-kit/testhelper/rand" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -62,7 +63,7 @@ func TestReportingErrorIndex(t *testing.T) { transformertest.ViolationErrorTransformerHandler( http.StatusBadRequest, "tracking plan validation failed", - []transformer.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, + []types.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, ), ). Build() diff --git a/integration_test/tracing/tracing_test.go b/integration_test/tracing/tracing_test.go index ed580b488a..dbbfe78144 100644 --- a/integration_test/tracing/tracing_test.go +++ b/integration_test/tracing/tracing_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + _ "github.com/marcboeker/go-duckdb" "github.com/ory/dockertest/v3" "github.com/samber/lo" @@ -33,7 +35,6 @@ import ( "github.com/rudderlabs/rudder-server/app" "github.com/rudderlabs/rudder-server/gateway/response" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -359,15 +360,15 @@ func TestTracing(t *testing.T) { defer bcServer.Close() trServer := transformertest.NewBuilder(). - WithUserTransformHandler(func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + WithUserTransformHandler(func(request []types.TransformerEvent) (response []types.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, }) - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, diff --git a/integration_test/transformer_contract/transformer_contract_test.go b/integration_test/transformer_contract/transformer_contract_test.go index b15474fabf..e9fb53ed71 100644 --- a/integration_test/transformer_contract/transformer_contract_test.go +++ b/integration_test/transformer_contract/transformer_contract_test.go @@ -28,7 +28,7 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/health" "github.com/rudderlabs/rudder-server/testhelper/transformertest" @@ -107,7 +107,7 @@ func TestTransformerContract(t *testing.T) { trServer := transformertest.NewBuilder(). WithUserTransformHandler( - func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + func(request []types.TransformerEvent) (response []types.TransformerResponse) { for i := range request { req := request[i] @@ -123,7 +123,7 @@ func TestTransformerContract(t *testing.T) { require.Equal(t, req.Metadata.TransformationID, "transformation-1") require.Equal(t, req.Metadata.TransformationVersionID, "version-1") require.Equal(t, req.Metadata.EventType, "identify") - require.Equal(t, req.Credentials, []transformer.Credential{ + require.Equal(t, req.Credentials, []types.Credential{ { ID: "credential-1", Key: "key-1", @@ -131,7 +131,7 @@ func TestTransformerContract(t *testing.T) { IsSecret: false, }, }) - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, diff --git a/integration_test/warehouse/warehouse_test.go b/integration_test/warehouse/warehouse_test.go index 8043b76f4c..7eedbe7cc4 100644 --- a/integration_test/warehouse/warehouse_test.go +++ b/integration_test/warehouse/warehouse_test.go @@ -17,6 +17,8 @@ import ( "text/template" "time" + "github.com/rudderlabs/rudder-server/processor/types" + transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" "github.com/google/uuid" @@ -1589,14 +1591,14 @@ func TestDestinationTransformation(t *testing.T) { testcases := []struct { name string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "with allowUsersContextTraits=true", configOverride: map[string]any{ "allowUsersContextTraits": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1625,7 +1627,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1652,7 +1654,7 @@ func TestDestinationTransformation(t *testing.T) { { name: "without allowUsersContextTraits", configOverride: map[string]any{}, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1719,7 +1721,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) @@ -1735,14 +1737,14 @@ func TestDestinationTransformation(t *testing.T) { testcases := []struct { name string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "with underscoreDivideNumbers=true", configOverride: map[string]any{ "underscoreDivideNumbers": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1767,7 +1769,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "underscoreDivideNumbers": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1790,7 +1792,7 @@ func TestDestinationTransformation(t *testing.T) { { name: "without underscoreDivideNumbers", configOverride: map[string]any{}, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1850,7 +1852,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) @@ -1867,7 +1869,7 @@ func TestDestinationTransformation(t *testing.T) { name string destType string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "for non-datalake destinations should be present", @@ -1875,7 +1877,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1905,7 +1907,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1975,7 +1977,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) diff --git a/internal/enricher/enricher.go b/internal/enricher/enricher.go index dd16eae785..fbe3177a93 100644 --- a/internal/enricher/enricher.go +++ b/internal/enricher/enricher.go @@ -2,7 +2,7 @@ package enricher import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/processor/types" ) // PipelineEnricher is a new paradigm under which the gateway events in diff --git a/internal/enricher/geolocation.go b/internal/enricher/geolocation.go index cece86354f..49954c8d4a 100644 --- a/internal/enricher/geolocation.go +++ b/internal/enricher/geolocation.go @@ -14,8 +14,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/services/geolocation" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( diff --git a/internal/enricher/geolocation_test.go b/internal/enricher/geolocation_test.go index 66b2fd46d5..af7762dec5 100644 --- a/internal/enricher/geolocation_test.go +++ b/internal/enricher/geolocation_test.go @@ -17,8 +17,8 @@ import ( svcMetric "github.com/rudderlabs/rudder-go-kit/stats/metric" miniodocker "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/minio" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/services/geolocation" - "github.com/rudderlabs/rudder-server/utils/types" ) func TestGeolocationEnrichment_Setup(t *testing.T) { diff --git a/mocks/processor/transformer/mock_transformer.go b/mocks/processor/transformer/mock_transformer.go index 2bbf4758b4..bd4251f0a7 100644 --- a/mocks/processor/transformer/mock_transformer.go +++ b/mocks/processor/transformer/mock_transformer.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - transformer "github.com/rudderlabs/rudder-server/processor/transformer" + types "github.com/rudderlabs/rudder-server/processor/types" gomock "go.uber.org/mock/gomock" ) @@ -42,10 +42,10 @@ func (m *MockTransformer) EXPECT() *MockTransformerMockRecorder { } // Transform mocks base method. -func (m *MockTransformer) Transform(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Transform", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } @@ -56,10 +56,10 @@ func (mr *MockTransformerMockRecorder) Transform(ctx, clientEvents, batchSize an } // UserTransform mocks base method. -func (m *MockTransformer) UserTransform(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UserTransform", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } @@ -70,10 +70,10 @@ func (mr *MockTransformerMockRecorder) UserTransform(ctx, clientEvents, batchSiz } // Validate mocks base method. -func (m *MockTransformer) Validate(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Validate", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } diff --git a/processor/consent.go b/processor/consent.go index 1d5af5bf5c..671e77b5ac 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -6,8 +6,8 @@ import ( "github.com/samber/lo" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) type ConsentManagementInfo struct { diff --git a/processor/consent_test.go b/processor/consent_test.go index 11c9fc0942..af65d3feb3 100644 --- a/processor/consent_test.go +++ b/processor/consent_test.go @@ -4,13 +4,11 @@ import ( "testing" "github.com/samber/lo" + "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - - "github.com/stretchr/testify/require" - - "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/processor/types" ) func TestGetOneTrustConsentCategories(t *testing.T) { diff --git a/processor/delayed/stats.go b/processor/delayed/stats.go index 62f8f427bd..31447eeeb5 100644 --- a/processor/delayed/stats.go +++ b/processor/delayed/stats.go @@ -4,10 +4,11 @@ import ( "strings" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/utils/misc" ) @@ -25,7 +26,7 @@ func NewEventStats(stats stats.Stats, config *config.Config) *eventStats { } } -func (s *eventStats) ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) { +func (s *eventStats) ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) { statusCount := map[string]map[string]int{ "missing_original_timestamp": {}, "missing_sent_at": {}, diff --git a/processor/delayed/stats_test.go b/processor/delayed/stats_test.go index 9ec6174482..2137bc4636 100644 --- a/processor/delayed/stats_test.go +++ b/processor/delayed/stats_test.go @@ -3,6 +3,8 @@ package delayed_test import ( "testing" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/stats" @@ -11,7 +13,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-server/processor/delayed" - "github.com/rudderlabs/rudder-server/processor/transformer" ) func TestEventStats(t *testing.T) { @@ -36,7 +37,7 @@ func TestEventStats(t *testing.T) { sdkLibrary := "rudder-go" sdkVersion := "1.0.0" - es.ObserveSourceEvents(source, fromSDK(sdkLibrary, sdkVersion, []transformer.TransformerEvent{ + es.ObserveSourceEvents(source, fromSDK(sdkLibrary, sdkVersion, []types.TransformerEvent{ // missing originalTimestamp { Message: map[string]interface{}{ @@ -228,7 +229,7 @@ func TestEventStats(t *testing.T) { t.Run(tc.name, func(t *testing.T) { tc.message["originalTimestamp"] = "2020-01-01T00:00:00.000Z" tc.message["sentAt"] = "2020-01-01T00:10:00.000Z" - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { Message: tc.message, }, @@ -259,7 +260,7 @@ func TestEventStats(t *testing.T) { }) } -func fromSDK(lib, version string, events []transformer.TransformerEvent) []transformer.TransformerEvent { +func fromSDK(lib, version string, events []types.TransformerEvent) []types.TransformerEvent { for i := range events { events[i].Message["context"] = map[string]interface{}{ "library": map[string]interface{}{ diff --git a/processor/eventfilter/eventfilter.go b/processor/eventfilter/eventfilter.go index a57eff9d3e..4b434b4725 100644 --- a/processor/eventfilter/eventfilter.go +++ b/processor/eventfilter/eventfilter.go @@ -4,11 +4,12 @@ import ( "slices" "strings" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( @@ -84,7 +85,7 @@ func GetSupportedMessageEvents(destination *backendconfig.DestinationT) ([]strin } type AllowTransformerEventParams struct { - TransformerEvent *transformer.TransformerEvent + TransformerEvent *types.TransformerEvent SupportedMessageTypes []string } @@ -122,12 +123,12 @@ Currently this method supports below validations(executed in the same order): 2. Validate if the event is sendable to destination based on connectionMode, sourceType & messageType */ -func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEvent, supportedMsgTypes []string) (bool, *transformer.TransformerResponse) { +func AllowEventToDestTransformation(transformerEvent *types.TransformerEvent, supportedMsgTypes []string) (bool, *types.TransformerResponse) { // MessageType filtering -- STARTS messageType := strings.TrimSpace(strings.ToLower(getMessageType(&transformerEvent.Message))) if messageType == "" { // We will abort the event - return false, &transformer.TransformerResponse{ + return false, &types.TransformerResponse{ Output: transformerEvent.Message, StatusCode: 400, Metadata: transformerEvent.Metadata, Error: "Invalid message type. Type assertion failed", @@ -140,8 +141,8 @@ func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEve "supportedMsgTypes", supportedMsgTypes, "messageType", messageType, ) // We will not allow the event - return false, &transformer.TransformerResponse{ - Output: transformerEvent.Message, StatusCode: types.FilterEventCode, + return false, &types.TransformerResponse{ + Output: transformerEvent.Message, StatusCode: reportingTypes.FilterEventCode, Metadata: transformerEvent.Metadata, Error: "Message type not supported", } @@ -164,8 +165,8 @@ func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEve }) if !allow { - return allow, &transformer.TransformerResponse{ - Output: transformerEvent.Message, StatusCode: types.FilterEventCode, + return allow, &types.TransformerResponse{ + Output: transformerEvent.Message, StatusCode: reportingTypes.FilterEventCode, Metadata: transformerEvent.Metadata, Error: "Filtering event based on hybridModeFilter", } diff --git a/processor/eventfilter/eventfilter_test.go b/processor/eventfilter/eventfilter_test.go index be6d7f1145..d47d34861b 100644 --- a/processor/eventfilter/eventfilter_test.go +++ b/processor/eventfilter/eventfilter_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" ) func TestFilterEventsForHybridMode(t *testing.T) { @@ -458,19 +458,19 @@ func TestConvertToArrayOfType(t *testing.T) { func TestAllowEventToDestTransformation(t *testing.T) { type testCaseT struct { caseName string - transformerEvent *transformer.TransformerEvent + transformerEvent *types.TransformerEvent expected bool supportedMsgTypes []string - expectedResp *transformer.TransformerResponse + expectedResp *types.TransformerResponse } testCases := []testCaseT{ { caseName: "if message type is invalid, return false with statusCode 400", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": ""}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": ""}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": ""}, StatusCode: 400, Error: "Invalid message type. Type assertion failed", @@ -478,10 +478,10 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if message type is unsupported, return false with statusCode 298", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "identify"}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "identify"}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": "identify"}, StatusCode: 298, Error: "Message type not supported", @@ -489,10 +489,10 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if event is filtered due to FilterEventsForHybridMode, return statusCode 298", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Metadata: transformer.Metadata{}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Metadata: types.Metadata{}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": "track"}, StatusCode: 298, Error: "Filtering event based on hybridModeFilter", @@ -500,7 +500,7 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if event is legit, return true with nil response", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Destination: backendconfig.DestinationT{IsProcessorEnabled: true}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Destination: backendconfig.DestinationT{IsProcessorEnabled: true}}, expected: true, supportedMsgTypes: []string{"track"}, expectedResp: nil, diff --git a/processor/events_response.go b/processor/events_response.go index 95686dfdac..5612451305 100644 --- a/processor/events_response.go +++ b/processor/events_response.go @@ -6,24 +6,24 @@ import ( "github.com/samber/lo" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" ) -func (proc *Handle) getDroppedJobs(response transformer.Response, eventsToTransform []transformer.TransformerEvent) []*jobsdb.JobT { +func (proc *Handle) getDroppedJobs(response types.Response, eventsToTransform []types.TransformerEvent) []*jobsdb.JobT { // each messageID is one event when sending to the transformer - inputMessageIDs := lo.Map(eventsToTransform, func(e transformer.TransformerEvent, _ int) string { + inputMessageIDs := lo.Map(eventsToTransform, func(e types.TransformerEvent, _ int) string { return e.Metadata.MessageID }) // in transformer response, multiple messageIDs could be batched together successFullMessageIDs := make([]string, 0) - lo.ForEach(response.Events, func(e transformer.TransformerResponse, _ int) { + lo.ForEach(response.Events, func(e types.TransformerResponse, _ int) { successFullMessageIDs = append(successFullMessageIDs, e.Metadata.GetMessagesIDs()...) }) // for failed as well failedMessageIDs := make([]string, 0) - lo.ForEach(response.FailedEvents, func(e transformer.TransformerResponse, _ int) { + lo.ForEach(response.FailedEvents, func(e types.TransformerResponse, _ int) { failedMessageIDs = append(failedMessageIDs, e.Metadata.GetMessagesIDs()...) }) // the remainder of the messageIDs are those that are dropped diff --git a/processor/integrations/integrations.go b/processor/integrations/integrations.go index f9eaf9e736..9b64b74939 100644 --- a/processor/integrations/integrations.go +++ b/processor/integrations/integrations.go @@ -10,8 +10,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) var ( @@ -94,7 +94,7 @@ func ValidatePostInfo(transformRawParams PostParametersT) error { // FilterClientIntegrations parses the destination names from the // input JSON, matches them with enabled destinations from controle plane and returns the IDSs func FilterClientIntegrations(clientEvent types.SingularEventT, destNameIDMap map[string]backendconfig.DestinationDefinitionT) (retVal []string) { - clientIntgs, ok := misc.GetRudderEventVal("integrations", clientEvent) + clientIntgs, ok := types.GetRudderEventVal("integrations", clientEvent) if !ok { clientIntgs = make(map[string]interface{}) } diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 7aa276c691..885bc31b32 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -3,6 +3,7 @@ package destination_transformer import ( "fmt" "strings" + "time" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" @@ -14,7 +15,10 @@ import ( type DestTransformer struct { config struct { - destTransformationURL string + destTransformationURL string + maxRetry config.ValueLoader[int] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -34,8 +38,10 @@ func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.log = log handle.stat = stat handle.client = http_client.NewHTTPClient(conf) - - handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") + handle.config.destTransformationURL = handle.conf.GetString("DEST_TRANSFORM_URL", "http://localhost:9090") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index 56caba741c..e38f87db85 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -2,6 +2,7 @@ package trackingplan_validation import ( "fmt" + "time" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" @@ -12,7 +13,10 @@ import ( type TPValidator struct { config struct { - destTransformationURL string + destTransformationURL string + maxRetry config.ValueLoader[int] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -33,6 +37,9 @@ func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *T handle.stat = stat handle.client = http_client.NewHTTPClient(conf) handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index ef7ac3fadd..3849a2c392 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -1,18 +1,25 @@ package user_transformer import ( + "context" "fmt" + "time" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/types" ) type UserTransformer struct { config struct { - userTransformationURL string + userTransformationURL string + maxRetry config.ValueLoader[int] + failOnUserTransformTimeout config.ValueLoader[bool] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -20,7 +27,7 @@ type UserTransformer struct { client http_client.HTTPDoer } -func (u *UserTransformer) SendRequest(data interface{}) (interface{}, error) { +func (u *UserTransformer) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) (interface{}, error) { fmt.Println("Sending request to Service A") // Add service-specific logic return "Response from Service A", nil @@ -32,7 +39,11 @@ func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.log = log handle.stat = stat handle.client = http_client.NewHTTPClient(conf) - handle.config.userTransformationURL = handle.conf.GetString("Warehouse.userTransformationURL", "http://localhost:9090") + handle.config.userTransformationURL = handle.conf.GetString("USER_TRANSFORM_URL", "http://localhost:9090") + handle.config.failOnUserTransformTimeout = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnUserTransformTimeout") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/processor.go b/processor/processor.go index f7d5d2f4c8..2de2bd2650 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -40,6 +40,7 @@ import ( "github.com/rudderlabs/rudder-server/processor/isolation" "github.com/rudderlabs/rudder-server/processor/stash" "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/router/batchrouter" "github.com/rudderlabs/rudder-server/rruntime" destinationdebugger "github.com/rudderlabs/rudder-server/services/debugger/destination" @@ -54,7 +55,7 @@ import ( "github.com/rudderlabs/rudder-server/utils/crash" "github.com/rudderlabs/rudder-server/utils/misc" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck - "github.com/rudderlabs/rudder-server/utils/types" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" "github.com/rudderlabs/rudder-server/utils/workerpool" ) @@ -75,7 +76,7 @@ func NewHandle(c *config.Config, transformer transformer.Transformer) *Handle { } type sourceObserver interface { - ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) + ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) } type trackedUsersReporter interface { @@ -101,7 +102,7 @@ type Handle struct { logger logger.Logger enrichers []enricher.PipelineEnricher dedup dedup.Dedup - reporting types.Reporting + reporting reportingTypes.Reporting reportingEnabled bool backgroundWait func() error backgroundCancel context.CancelFunc @@ -157,7 +158,7 @@ type Handle struct { enableParallelScan bool archivalEnabled config.ValueLoader[bool] eventAuditEnabled map[string]bool - credentialsMap map[string][]transformer.Credential + credentialsMap map[string][]types.Credential nonEventStreamSources map[string]bool } @@ -260,10 +261,10 @@ type MetricMetadata struct { type NonSuccessfulTransformationMetrics struct { failedJobs []*jobsdb.JobT - failedMetrics []*types.PUReportedMetric + failedMetrics []*reportingTypes.PUReportedMetric failedCountMap map[string]int64 filteredJobs []*jobsdb.JobT - filteredMetrics []*types.PUReportedMetric + filteredMetrics []*reportingTypes.PUReportedMetric filteredCountMap map[string]int64 } @@ -376,7 +377,7 @@ func (proc *Handle) Setup( gatewayDB, routerDB, batchRouterDB, readErrorDB, writeErrorDB, eventSchemaDB, archivalDB jobsdb.JobsDB, - reporting types.Reporting, + reporting reportingTypes.Reporting, transientSources transientsource.Service, fileuploader fileuploader.Provider, rsourcesService rsources.JobService, @@ -389,7 +390,7 @@ func (proc *Handle) Setup( proc.reporting = reporting proc.destDebugger = destDebugger proc.transDebugger = transDebugger - proc.reportingEnabled = config.GetBoolVar(types.DefaultReportingEnabled, "Reporting.enabled") + proc.reportingEnabled = config.GetBoolVar(reportingTypes.DefaultReportingEnabled, "Reporting.enabled") if proc.conf == nil { proc.conf = config.Default } @@ -844,7 +845,7 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) { sourceIdDestinationMap = make(map[string][]backendconfig.DestinationT) sourceIdSourceMap = make(map[string]backendconfig.SourceT) eventAuditEnabled = make(map[string]bool) - credentialsMap = make(map[string][]transformer.Credential) + credentialsMap = make(map[string][]types.Credential) nonEventStreamSources = make(map[string]bool) connectionConfigMap = make(map[connection]backendconfig.Connection) ) @@ -875,8 +876,8 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) { } workspaceLibrariesMap[workspaceID] = wConfig.Libraries eventAuditEnabled[workspaceID] = wConfig.Settings.EventAuditEnabled - credentialsMap[workspaceID] = lo.MapToSlice(wConfig.Credentials, func(key string, value backendconfig.Credential) transformer.Credential { - return transformer.Credential{ + credentialsMap[workspaceID] = lo.MapToSlice(wConfig.Credentials, func(key string, value backendconfig.Credential) types.Credential { + return types.Credential{ ID: key, Key: value.Key, Value: value.Value, @@ -968,7 +969,7 @@ func getTimestampFromEvent(event types.SingularEventT, field string, defaultTime return timestamp } -func enhanceWithTimeFields(event *transformer.TransformerEvent, singularEvent types.SingularEventT, receivedAt time.Time) { +func enhanceWithTimeFields(event *types.TransformerEvent, singularEvent types.SingularEventT, receivedAt time.Time) { // set timestamp skew based on timestamp fields from SDKs originalTimestamp := getTimestampFromEvent(singularEvent, "originalTimestamp", receivedAt) sentAt := getTimestampFromEvent(singularEvent, "sentAt", receivedAt) @@ -989,8 +990,8 @@ func enhanceWithTimeFields(event *transformer.TransformerEvent, singularEvent ty event.Message["timestamp"] = timestamp.Format(misc.RFC3339Milli) } -func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.SingularEventT, userID string, jobId int64, receivedAt time.Time, source *backendconfig.SourceT, eventParams types.EventParams) *transformer.Metadata { - commonMetadata := transformer.Metadata{} +func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.SingularEventT, userID string, jobId int64, receivedAt time.Time, source *backendconfig.SourceT, eventParams types.EventParams) *types.Metadata { + commonMetadata := types.Metadata{} commonMetadata.SourceID = source.ID commonMetadata.SourceName = source.Name commonMetadata.OriginalSourceID = source.OriginalID @@ -1020,8 +1021,8 @@ func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.Sing } // add metadata to each singularEvent which will be returned by transformer in response -func enhanceWithMetadata(commonMetadata *transformer.Metadata, event *transformer.TransformerEvent, destination *backendconfig.DestinationT) { - metadata := transformer.Metadata{} +func enhanceWithMetadata(commonMetadata *types.Metadata, event *types.TransformerEvent, destination *backendconfig.DestinationT) { + metadata := types.Metadata{} metadata.SourceType = commonMetadata.SourceType metadata.SourceCategory = commonMetadata.SourceCategory metadata.SourceID = commonMetadata.SourceID @@ -1113,24 +1114,24 @@ func (proc *Handle) recordEventDeliveryStatus(jobsByDestID map[string][]*jobsdb. } func (proc *Handle) getTransformerEvents( - response transformer.Response, - commonMetaData *transformer.Metadata, + response types.Response, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, destination *backendconfig.DestinationT, connection backendconfig.Connection, inPU, pu string, ) ( - []transformer.TransformerEvent, - []*types.PUReportedMetric, + []types.TransformerEvent, + []*reportingTypes.PUReportedMetric, map[string]int64, map[string]MetricMetadata, ) { - successMetrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + successMetrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) successCountMap := make(map[string]int64) successCountMetadataMap := make(map[string]MetricMetadata) - var eventsToTransform []transformer.TransformerEvent + var eventsToTransform []types.TransformerEvent for i := range response.Events { // Update metrics maps userTransformedEvent := &response.Events[i] @@ -1143,7 +1144,7 @@ func (proc *Handle) getTransformerEvents( for _, message := range messages { proc.updateMetricMaps(successCountMetadataMap, successCountMap, connectionDetailsMap, statusDetailsMap, userTransformedEvent, jobsdb.Succeeded.State, pu, func() json.RawMessage { - if pu != types.TRACKINGPLAN_VALIDATOR { + if pu != reportingTypes.TRACKINGPLAN_VALIDATOR { return []byte(`{}`) } if proc.transientSources.Apply(commonMetaData.SourceID) { @@ -1176,7 +1177,7 @@ func (proc *Handle) getTransformerEvents( eventMetadata.DestinationDefinitionID = userTransformedEvent.Metadata.DestinationDefinitionID eventMetadata.SourceCategory = userTransformedEvent.Metadata.SourceCategory eventMetadata.TraceParent = userTransformedEvent.Metadata.TraceParent - updatedEvent := transformer.TransformerEvent{ + updatedEvent := types.TransformerEvent{ Message: userTransformedEvent.Output, Metadata: *eventMetadata, Destination: *destination, @@ -1188,13 +1189,13 @@ func (proc *Handle) getTransformerEvents( // REPORTING - START if proc.isReportingEnabled() { - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(inPU, pu, false, false), + PUDetails: *reportingTypes.CreatePUDetails(inPU, pu, false, false), StatusDetail: sd, } successMetrics = append(successMetrics, m) @@ -1209,9 +1210,9 @@ func (proc *Handle) getTransformerEvents( func (proc *Handle) updateMetricMaps( countMetadataMap map[string]MetricMetadata, countMap map[string]int64, - connectionDetailsMap map[string]*types.ConnectionDetails, - statusDetailsMap map[string]map[string]*types.StatusDetail, - event *transformer.TransformerResponse, + connectionDetailsMap map[string]*reportingTypes.ConnectionDetails, + statusDetailsMap map[string]map[string]*reportingTypes.StatusDetail, + event *types.TransformerResponse, status, stage string, payload func() json.RawMessage, eventsByMessageID map[string]types.SingularEventWithReceivedAt, @@ -1264,7 +1265,7 @@ func (proc *Handle) updateMetricMaps( ) if _, ok := connectionDetailsMap[key]; !ok { - connectionDetailsMap[key] = &types.ConnectionDetails{ + connectionDetailsMap[key] = &reportingTypes.ConnectionDetails{ SourceID: event.Metadata.SourceID, SourceTaskRunID: event.Metadata.SourceTaskRunID, SourceJobID: event.Metadata.SourceJobID, @@ -1281,16 +1282,16 @@ func (proc *Handle) updateMetricMaps( } if _, ok := statusDetailsMap[key]; !ok { - statusDetailsMap[key] = make(map[string]*types.StatusDetail) + statusDetailsMap[key] = make(map[string]*reportingTypes.StatusDetail) } // create status details for each validation error // single event can have multiple validation errors of same type veCount := len(event.ValidationErrors) - if stage == types.TRACKINGPLAN_VALIDATOR && status == jobsdb.Succeeded.State { + if stage == reportingTypes.TRACKINGPLAN_VALIDATOR && status == jobsdb.Succeeded.State { if veCount > 0 { - status = types.SUCCEEDED_WITH_VIOLATIONS + status = reportingTypes.SUCCEEDED_WITH_VIOLATIONS } else { - status = types.SUCCEEDED_WITHOUT_VIOLATIONS + status = reportingTypes.SUCCEEDED_WITHOUT_VIOLATIONS } } sdkeySet := map[string]struct{}{} @@ -1300,7 +1301,7 @@ func (proc *Handle) updateMetricMaps( sd, ok := statusDetailsMap[key][sdkey] if !ok { - sd = &types.StatusDetail{ + sd = &reportingTypes.StatusDetail{ Status: status, StatusCode: event.StatusCode, SampleResponse: event.Error, @@ -1322,7 +1323,7 @@ func (proc *Handle) updateMetricMaps( sdkey := fmt.Sprintf("%s:%d:%s:%s:%s", status, event.StatusCode, eventName, eventType, "") sd, ok := statusDetailsMap[key][sdkey] if !ok { - sd = &types.StatusDetail{ + sd = &reportingTypes.StatusDetail{ Status: status, StatusCode: event.StatusCode, SampleResponse: event.Error, @@ -1339,15 +1340,15 @@ func (proc *Handle) updateMetricMaps( messageIDs := lo.Uniq(event.Metadata.GetMessagesIDs()) // this is called defensive programming... :( for _, messageID := range messageIDs { receivedAt := eventsByMessageID[messageID].ReceivedAt - sd.FailedMessages = append(sd.FailedMessages, &types.FailedMessage{MessageID: messageID, ReceivedAt: receivedAt}) + sd.FailedMessages = append(sd.FailedMessages, &reportingTypes.FailedMessage{MessageID: messageID, ReceivedAt: receivedAt}) } } sd.ViolationCount += int64(veCount) } func (proc *Handle) getNonSuccessfulMetrics( - response transformer.Response, - commonMetaData *transformer.Metadata, + response types.Response, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, inPU, pu string, ) *NonSuccessfulTransformationMetrics { @@ -1355,8 +1356,8 @@ func (proc *Handle) getNonSuccessfulMetrics( grouped := lo.GroupBy( response.FailedEvents, - func(event transformer.TransformerResponse) bool { - return event.StatusCode == types.FilterEventCode + func(event types.TransformerResponse) bool { + return event.StatusCode == reportingTypes.FilterEventCode }, ) filtered, failed := grouped[true], grouped[false] @@ -1407,15 +1408,15 @@ func procErrorCountsStat(destType, pu, statusCode string) { } func (proc *Handle) getTransformationMetrics( - transformerResponses []transformer.TransformerResponse, + transformerResponses []types.TransformerResponse, state string, - commonMetaData *transformer.Metadata, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, inPU, pu string, -) ([]*jobsdb.JobT, []*types.PUReportedMetric, map[string]int64) { - metrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) +) ([]*jobsdb.JobT, []*reportingTypes.PUReportedMetric, map[string]int64) { + metrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) countMap := make(map[string]int64) var jobs []*jobsdb.JobT statFunc := procErrorCountsStat @@ -1501,12 +1502,12 @@ func (proc *Handle) getTransformationMetrics( // REPORTING - START if proc.isReportingEnabled() { - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(inPU, pu, false, false), + PUDetails: *reportingTypes.CreatePUDetails(inPU, pu, false, false), StatusDetail: sd, } metrics = append(metrics, m) @@ -1566,10 +1567,10 @@ func getDiffMetrics( inCountMetadataMap map[string]MetricMetadata, inCountMap, successCountMap, failedCountMap, filteredCountMap map[string]int64, statFactory stats.Stats, -) []*types.PUReportedMetric { +) []*reportingTypes.PUReportedMetric { // Calculate diff and append to reportMetrics // diff = successCount + abortCount - inCount - diffMetrics := make([]*types.PUReportedMetric, 0) + diffMetrics := make([]*reportingTypes.PUReportedMetric, 0) for key, inCount := range inCountMap { var eventName, eventType string splitKey := strings.Split(key, MetricKeyDelimiter) @@ -1586,8 +1587,8 @@ func getDiffMetrics( diff := successCount + failedCount + filteredCount - inCount if diff != 0 { metricMetadata := inCountMetadataMap[key] - metric := &types.PUReportedMetric{ - ConnectionDetails: types.ConnectionDetails{ + metric := &reportingTypes.PUReportedMetric{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: metricMetadata.sourceID, DestinationID: metricMetadata.destinationID, SourceTaskRunID: metricMetadata.sourceTaskRunID, @@ -1601,12 +1602,12 @@ func getDiffMetrics( TrackingPlanID: metricMetadata.trackingPlanID, TrackingPlanVersion: metricMetadata.trackingPlanVersion, }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: inPU, PU: pu, }, - StatusDetail: &types.StatusDetail{ - Status: types.DiffStatus, + StatusDetail: &reportingTypes.StatusDetail{ + Status: reportingTypes.DiffStatus, Count: diff, SampleEvent: []byte(`{}`), EventName: eventName, @@ -1644,20 +1645,20 @@ type preTransformationMessage struct { subJobs subJob eventSchemaJobs []*jobsdb.JobT archivalJobs []*jobsdb.JobT - connectionDetailsMap map[string]*types.ConnectionDetails - statusDetailsMap map[string]map[string]*types.StatusDetail - reportMetrics []*types.PUReportedMetric - destFilterStatusDetailMap map[string]map[string]*types.StatusDetail + connectionDetailsMap map[string]*reportingTypes.ConnectionDetails + statusDetailsMap map[string]map[string]*reportingTypes.StatusDetail + reportMetrics []*reportingTypes.PUReportedMetric + destFilterStatusDetailMap map[string]map[string]*reportingTypes.StatusDetail inCountMetadataMap map[string]MetricMetadata inCountMap map[string]int64 outCountMap map[string]int64 totalEvents int marshalStart time.Time - groupedEventsBySourceId map[SourceIDT][]transformer.TransformerEvent + groupedEventsBySourceId map[SourceIDT][]types.TransformerEvent eventsByMessageID map[string]types.SingularEventWithReceivedAt procErrorJobs []*jobsdb.JobT jobIDToSpecificDestMapOnly map[int64]string - groupedEvents map[string][]transformer.TransformerEvent + groupedEvents map[string][]types.TransformerEvent uniqueMessageIdsBySrcDestKey map[string]map[string]struct{} statusList []*jobsdb.JobStatusT jobList []*jobsdb.JobT @@ -1677,8 +1678,8 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra proc.stats.statNumRequests(partition).Count(len(jobList)) var statusList []*jobsdb.JobStatusT - groupedEvents := make(map[string][]transformer.TransformerEvent) - groupedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) + groupedEvents := make(map[string][]types.TransformerEvent) + groupedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) eventsByMessageID := make(map[string]types.SingularEventWithReceivedAt) var procErrorJobs []*jobsdb.JobT eventSchemaJobs := make([]*jobsdb.JobT, 0) @@ -1701,14 +1702,14 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra uniqueMessageIdsBySrcDestKey := make(map[string]map[string]struct{}) sourceDupStats := make(map[dupStatKey]int) - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) inCountMap := make(map[string]int64) inCountMetadataMap := make(map[string]MetricMetadata) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) outCountMap := make(map[string]int64) // destinations enabled - destFilterStatusDetailMap := make(map[string]map[string]*types.StatusDetail) + destFilterStatusDetailMap := make(map[string]map[string]*reportingTypes.StatusDetail) // map of jobID to destinationID: for messages that needs to be delivered to a specific destinations only jobIDToSpecificDestMapOnly := make(map[int64]string) @@ -1922,7 +1923,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra } // REPORTING - GATEWAY metrics - START // dummy event for metrics purposes only - transformerEvent := &transformer.TransformerResponse{} + transformerEvent := &types.TransformerResponse{} if proc.isReportingEnabled() { transformerEvent.Metadata = *commonMetadataFromSingularEvent proc.updateMetricMaps( @@ -1932,7 +1933,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra statusDetailsMap, transformerEvent, jobsdb.Succeeded.State, - types.GATEWAY, + reportingTypes.GATEWAY, func() json.RawMessage { if sourceIsTransient { return []byte(`{}`) @@ -1955,9 +1956,9 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra } if _, ok := groupedEventsBySourceId[SourceIDT(sourceId)]; !ok { - groupedEventsBySourceId[SourceIDT(sourceId)] = make([]transformer.TransformerEvent, 0) + groupedEventsBySourceId[SourceIDT(sourceId)] = make([]types.TransformerEvent, 0) } - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Message = event.singularEvent shallowEventCopy.Message["request_ip"] = event.requestIP enhanceWithTimeFields(&shallowEventCopy, event.singularEvent, event.recievedAt) @@ -1983,7 +1984,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra groupedEventsBySourceId[SourceIDT(sourceId)] = append(groupedEventsBySourceId[SourceIDT(sourceId)], shallowEventCopy) if proc.isReportingEnabled() { - proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, transformerEvent, jobsdb.Succeeded.State, types.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, transformerEvent, jobsdb.Succeeded.State, reportingTypes.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) } } @@ -2075,21 +2076,21 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // REPORTING - GATEWAY metrics - START if proc.isReportingEnabled() { - types.AssertSameKeys(preTrans.connectionDetailsMap, preTrans.statusDetailsMap) + reportingTypes.AssertSameKeys(preTrans.connectionDetailsMap, preTrans.statusDetailsMap) for k, cd := range preTrans.connectionDetailsMap { for _, sd := range preTrans.statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails("", types.GATEWAY, false, true), + PUDetails: *reportingTypes.CreatePUDetails("", reportingTypes.GATEWAY, false, true), StatusDetail: sd, } preTrans.reportMetrics = append(preTrans.reportMetrics, m) } for _, dsd := range preTrans.destFilterStatusDetailMap[k] { - destFilterMetric := &types.PUReportedMetric{ + destFilterMetric := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(types.GATEWAY, types.DESTINATION_FILTER, false, false), + PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.GATEWAY, reportingTypes.DESTINATION_FILTER, false, false), StatusDetail: dsd, } preTrans.reportMetrics = append(preTrans.reportMetrics, destFilterMetric) @@ -2098,8 +2099,8 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // empty failedCountMap because no failures, // events are just dropped at this point if no destination is found to route the events diffMetrics := getDiffMetrics( - types.GATEWAY, - types.DESTINATION_FILTER, + reportingTypes.GATEWAY, + reportingTypes.DESTINATION_FILTER, preTrans.inCountMetadataMap, preTrans.inCountMap, preTrans.outCountMap, @@ -2171,7 +2172,7 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // Adding a singular event multiple times if there are multiple destinations of same type for idx := range enabledDestinationsList { destination := &enabledDestinationsList[idx] - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Connection = proc.getConnectionConfig(connection{sourceID: sourceId, destinationID: destination.ID}) shallowEventCopy.Message = singularEvent shallowEventCopy.Destination = *destination @@ -2197,7 +2198,7 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // We have at-least one event so marking it good _, ok := preTrans.groupedEvents[srcAndDestKey] if !ok { - preTrans.groupedEvents[srcAndDestKey] = make([]transformer.TransformerEvent, 0) + preTrans.groupedEvents[srcAndDestKey] = make([]types.TransformerEvent, 0) } preTrans.groupedEvents[srcAndDestKey] = append(preTrans.groupedEvents[srcAndDestKey], shallowEventCopy) @@ -2249,8 +2250,8 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans proc.stats.statNumRequests(partition).Count(len(jobList)) var statusList []*jobsdb.JobStatusT - groupedEvents := make(map[string][]transformer.TransformerEvent) - groupedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) + groupedEvents := make(map[string][]types.TransformerEvent) + groupedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) eventsByMessageID := make(map[string]types.SingularEventWithReceivedAt) var procErrorJobs []*jobsdb.JobT eventSchemaJobs := make([]*jobsdb.JobT, 0) @@ -2273,14 +2274,14 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans uniqueMessageIdsBySrcDestKey := make(map[string]map[string]struct{}) sourceDupStats := make(map[dupStatKey]int) - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) inCountMap := make(map[string]int64) inCountMetadataMap := make(map[string]MetricMetadata) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) outCountMap := make(map[string]int64) // destinations enabled - destFilterStatusDetailMap := make(map[string]map[string]*types.StatusDetail) + destFilterStatusDetailMap := make(map[string]map[string]*reportingTypes.StatusDetail) // map of jobID to destinationID: for messages that needs to be delivered to a specific destinations only jobIDToSpecificDestMapOnly := make(map[int64]string) @@ -2448,7 +2449,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans // REPORTING - GATEWAY metrics - START // dummy event for metrics purposes only - event := &transformer.TransformerResponse{} + event := &types.TransformerResponse{} if proc.isReportingEnabled() { event.Metadata = *commonMetadataFromSingularEvent proc.updateMetricMaps( @@ -2458,7 +2459,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans statusDetailsMap, event, jobsdb.Succeeded.State, - types.GATEWAY, + reportingTypes.GATEWAY, func() json.RawMessage { if sourceIsTransient { return []byte(`{}`) @@ -2482,9 +2483,9 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans } if _, ok := groupedEventsBySourceId[SourceIDT(sourceID)]; !ok { - groupedEventsBySourceId[SourceIDT(sourceID)] = make([]transformer.TransformerEvent, 0) + groupedEventsBySourceId[SourceIDT(sourceID)] = make([]types.TransformerEvent, 0) } - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Message = singularEvent shallowEventCopy.Message["request_ip"] = requestIP enhanceWithTimeFields(&shallowEventCopy, singularEvent, receivedAt) @@ -2511,7 +2512,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans groupedEventsBySourceId[SourceIDT(sourceID)] = append(groupedEventsBySourceId[SourceIDT(sourceID)], shallowEventCopy) if proc.isReportingEnabled() { - proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, event, jobsdb.Succeeded.State, types.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, event, jobsdb.Succeeded.State, reportingTypes.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) } } } @@ -2550,12 +2551,12 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans } type transformationMessage struct { - groupedEvents map[string][]transformer.TransformerEvent + groupedEvents map[string][]types.TransformerEvent trackingPlanEnabledMap map[SourceIDT]bool eventsByMessageID map[string]types.SingularEventWithReceivedAt uniqueMessageIdsBySrcDestKey map[string]map[string]struct{} - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric statusList []*jobsdb.JobStatusT procErrorJobs []*jobsdb.JobT sourceDupStats map[dupStatKey]int @@ -2694,7 +2695,7 @@ type storeMessage struct { procErrorJobs []*jobsdb.JobT routerDestIDs []string - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric sourceDupStats map[dupStatKey]int dedupKeys map[string]struct{} @@ -2950,7 +2951,7 @@ func getJobCountsByWorkspaceDestType(jobs []*jobsdb.JobT) map[string]map[string] } type transformSrcDestOutput struct { - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric destJobs []*jobsdb.JobT batchDestJobs []*jobsdb.JobT errorsPerDestID map[string][]*jobsdb.JobT @@ -2962,7 +2963,7 @@ func (proc *Handle) transformSrcDest( ctx context.Context, partition string, // main inputs - srcAndDestKey string, eventList []transformer.TransformerEvent, + srcAndDestKey string, eventList []types.TransformerEvent, // helpers trackingPlanEnabledMap map[SourceIDT]bool, @@ -2977,7 +2978,7 @@ func (proc *Handle) transformSrcDest( connection := eventList[0].Connection workspaceID := eventList[0].Metadata.WorkspaceID destType := destination.DestinationDefinition.Name - commonMetaData := &transformer.Metadata{ + commonMetaData := &types.Metadata{ SourceID: sourceID, SourceName: sourceName, OriginalSourceID: eventList[0].Metadata.OriginalSourceID, @@ -2991,7 +2992,7 @@ func (proc *Handle) transformSrcDest( SourceDefinitionType: eventList[0].Metadata.SourceDefinitionType, } - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) batchDestJobs := make([]*jobsdb.JobT, 0) destJobs := make([]*jobsdb.JobT, 0) routerDestIDs := make(map[string]struct{}) @@ -3045,13 +3046,13 @@ func (proc *Handle) transformSrcDest( } // REPORTING - END - var response transformer.Response - var eventsToTransform []transformer.TransformerEvent + var response types.Response + var eventsToTransform []types.TransformerEvent var inPU string if trackingPlanEnabled { - inPU = types.TRACKINGPLAN_VALIDATOR + inPU = reportingTypes.TRACKINGPLAN_VALIDATOR } else { - inPU = types.DESTINATION_FILTER + inPU = reportingTypes.DESTINATION_FILTER } // Send to custom transformer only if the destination has a transformer enabled if transformationEnabled { @@ -3065,11 +3066,11 @@ func (proc *Handle) transformSrcDest( d := time.Since(startedAt) userTransformationStat.transformTime.SendTiming(d) - var successMetrics []*types.PUReportedMetric + var successMetrics []*reportingTypes.PUReportedMetric var successCountMap map[string]int64 var successCountMetadataMap map[string]MetricMetadata - eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, types.USER_TRANSFORMER) - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, types.USER_TRANSFORMER) + eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.USER_TRANSFORMER) + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.USER_TRANSFORMER) droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventList), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) if _, ok := procErrorJobsByDestID[destID]; !ok { procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) @@ -3086,8 +3087,8 @@ func (proc *Handle) transformSrcDest( // REPORTING - START if proc.isReportingEnabled() { diffMetrics := getDiffMetrics( - types.DESTINATION_FILTER, - types.USER_TRANSFORMER, + reportingTypes.DESTINATION_FILTER, + reportingTypes.USER_TRANSFORMER, inCountMetadataMap, inCountMap, successCountMap, @@ -3105,7 +3106,7 @@ func (proc *Handle) transformSrcDest( inCountMetadataMap = successCountMetadataMap } // REPORTING - END - inPU = types.USER_TRANSFORMER // for the next step in the pipeline + inPU = reportingTypes.USER_TRANSFORMER // for the next step in the pipeline }) } else { proc.logger.Debug("No custom transformation") @@ -3141,7 +3142,7 @@ func (proc *Handle) transformSrcDest( response = ConvertToFilteredTransformerResponse( eventsToTransform, transformAt != "none", - func(event transformer.TransformerEvent) (bool, string) { + func(event types.TransformerEvent) (bool, string) { if event.Metadata.SourceJobRunID != "" && slices.Contains( proc.drainConfig.jobRunIDs.Load(), @@ -3156,23 +3157,23 @@ func (proc *Handle) transformSrcDest( return false, "" }, ) - var successMetrics []*types.PUReportedMetric + var successMetrics []*reportingTypes.PUReportedMetric var successCountMap map[string]int64 var successCountMetadataMap map[string]MetricMetadata - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, types.EVENT_FILTER) + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.EVENT_FILTER) droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventsToTransform), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) if _, ok := procErrorJobsByDestID[destID]; !ok { procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) } procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) - eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, types.EVENT_FILTER) + eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.EVENT_FILTER) proc.logger.Debug("Supported messages filtering output size", len(eventsToTransform)) // REPORTING - START if proc.isReportingEnabled() { diffMetrics := getDiffMetrics( inPU, - types.EVENT_FILTER, + reportingTypes.EVENT_FILTER, inCountMetadataMap, inCountMap, successCountMap, @@ -3231,7 +3232,7 @@ func (proc *Handle) transformSrcDest( nonSuccessMetrics := proc.getNonSuccessfulMetrics( response, commonMetaData, eventsByMessageID, - types.EVENT_FILTER, types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, ) destTransformationStat.numEvents.Count(len(eventsToTransform)) destTransformationStat.numOutputSuccessEvents.Count(len(response.Events)) @@ -3246,21 +3247,21 @@ func (proc *Handle) transformSrcDest( // REPORTING - PROCESSOR metrics - START if proc.isReportingEnabled() { - successMetrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + successMetrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) successCountMap := make(map[string]int64) for i := range response.Events { // Update metrics maps - proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, types.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, reportingTypes.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) } - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(types.EVENT_FILTER, types.DEST_TRANSFORMER, false, false), + PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, false, false), StatusDetail: sd, } successMetrics = append(successMetrics, m) @@ -3268,8 +3269,8 @@ func (proc *Handle) transformSrcDest( } diffMetrics := getDiffMetrics( - types.EVENT_FILTER, - types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, + reportingTypes.DEST_TRANSFORMER, inCountMetadataMap, inCountMap, successCountMap, @@ -3395,12 +3396,12 @@ func (proc *Handle) saveDroppedJobs(ctx context.Context, droppedJobs []*jobsdb.J } func ConvertToFilteredTransformerResponse( - events []transformer.TransformerEvent, + events []types.TransformerEvent, filter bool, - drainFunc func(transformer.TransformerEvent) (bool, string), -) transformer.Response { - var responses []transformer.TransformerResponse - var failedEvents []transformer.TransformerResponse + drainFunc func(types.TransformerEvent) (bool, string), +) types.Response { + var responses []types.TransformerResponse + var failedEvents []types.TransformerResponse type cacheValue struct { values []string @@ -3417,9 +3418,9 @@ func ConvertToFilteredTransformerResponse( if drain, reason := drainFunc(*event); drain { failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, - StatusCode: types.DrainEventCode, + StatusCode: reportingTypes.DrainEventCode, Metadata: event.Metadata, Error: reason, }, @@ -3458,7 +3459,7 @@ func ConvertToFilteredTransformerResponse( // add to FailedEvents failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, StatusCode: 400, Metadata: event.Metadata, @@ -3470,9 +3471,9 @@ func ConvertToFilteredTransformerResponse( if !slices.Contains(supportedEvents.values, messageEvent) { failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, - StatusCode: types.FilterEventCode, + StatusCode: reportingTypes.FilterEventCode, Metadata: event.Metadata, Error: "Event not supported", }, @@ -3485,7 +3486,7 @@ func ConvertToFilteredTransformerResponse( // allow event responses = append( responses, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, StatusCode: 200, Metadata: event.Metadata, @@ -3493,7 +3494,7 @@ func ConvertToFilteredTransformerResponse( ) } - return transformer.Response{Events: responses, FailedEvents: failedEvents} + return types.Response{Events: responses, FailedEvents: failedEvents} } func (proc *Handle) getJobs(partition string) jobsdb.JobsResult { @@ -3680,7 +3681,7 @@ func (proc *Handle) updateRudderSourcesStats(ctx context.Context, tx jobsdb.Stor return err } -func filterConfig(eventCopy *transformer.TransformerEvent) { +func filterConfig(eventCopy *types.TransformerEvent) { if configsToFilterI, ok := eventCopy.Destination.DestinationDefinition.Config["configFilters"]; ok { if configsToFilter, ok := configsToFilterI.([]interface{}); ok { omitKeys := lo.FilterMap(configsToFilter, func(configKey interface{}, _ int) (string, bool) { diff --git a/processor/processorBenchmark_test.go b/processor/processorBenchmark_test.go index e3c7bcdd14..9a614ec695 100644 --- a/processor/processorBenchmark_test.go +++ b/processor/processorBenchmark_test.go @@ -6,10 +6,9 @@ import ( "github.com/google/uuid" - "github.com/rudderlabs/rudder-server/utils/types" - backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" + "github.com/rudderlabs/rudder-server/processor/types" ) func Benchmark_makeCommonMetadataFromSingularEvent(b *testing.B) { diff --git a/processor/processor_test.go b/processor/processor_test.go index 5aca374841..c11e5cebff 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,7 +41,7 @@ import ( mock_features "github.com/rudderlabs/rudder-server/mocks/services/transformer" mockReportingTypes "github.com/rudderlabs/rudder-server/mocks/utils/types" "github.com/rudderlabs/rudder-server/processor/isolation" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" destinationdebugger "github.com/rudderlabs/rudder-server/services/debugger/destination" transformationdebugger "github.com/rudderlabs/rudder-server/services/debugger/transformation" dedupTypes "github.com/rudderlabs/rudder-server/services/dedup/types" @@ -51,20 +53,19 @@ import ( "github.com/rudderlabs/rudder-server/utils/pubsub" testutils "github.com/rudderlabs/rudder-server/utils/tests" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck - "github.com/rudderlabs/rudder-server/utils/types" ) type mockObserver struct { calls []struct { source *backendconfig.SourceT - events []transformer.TransformerEvent + events []types.TransformerEvent } } -func (m *mockObserver) ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) { +func (m *mockObserver) ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) { m.calls = append(m.calls, struct { source *backendconfig.SourceT - events []transformer.TransformerEvent + events []types.TransformerEvent }{source: source, events: events}) } @@ -955,7 +956,7 @@ var _ = Describe("Tracking Plan Validation", Ordered, func() { Context("RudderTyper", func() { It("Tracking plan id and version from DgSourceTrackingPlanConfig", func() { mockTransformer := mocksTransformer.NewMockTransformer(c.mockCtrl) - mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(transformer.Response{}) + mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(types.Response{}) isolationStrategy, err := isolation.GetStrategy(isolation.ModeNone) Expect(err).To(BeNil()) @@ -1030,7 +1031,7 @@ var _ = Describe("Tracking Plan Validation", Ordered, func() { }) It("Tracking plan version override from context.ruddertyper", func() { mockTransformer := mocksTransformer.NewMockTransformer(c.mockCtrl) - mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(transformer.Response{}) + mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(types.Response{}) isolationStrategy, err := isolation.GetStrategy(isolation.ModeNone) Expect(err).To(BeNil()) @@ -3104,19 +3105,19 @@ var _ = Describe("Processor", Ordered, func() { // We expect one call to user transform for destination B callUserTransform := mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).After(callUnprocessed). - DoAndReturn(func(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { + DoAndReturn(func(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { defer GinkgoRecover() - outputEvents := make([]transformer.TransformerResponse, 0) + outputEvents := make([]types.TransformerResponse, 0) for _, event := range clientEvents { event.Message["user-transform"] = "value" - outputEvents = append(outputEvents, transformer.TransformerResponse{ + outputEvents = append(outputEvents, types.TransformerResponse{ Output: event.Message, }) } - return transformer.Response{ + return types.Response{ Events: outputEvents, } }) @@ -3809,19 +3810,19 @@ var _ = Describe("Processor", Ordered, func() { // We expect one call to user transform for destination B callUserTransform := mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).After(callUnprocessed). - DoAndReturn(func(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { + DoAndReturn(func(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { defer GinkgoRecover() - outputEvents := make([]transformer.TransformerResponse, 0) + outputEvents := make([]types.TransformerResponse, 0) for _, event := range clientEvents { event.Message["user-transform"] = "value" - outputEvents = append(outputEvents, transformer.TransformerResponse{ + outputEvents = append(outputEvents, types.TransformerResponse{ Output: event.Message, }) } - return transformer.Response{ + return types.Response{ Events: outputEvents, } }) @@ -3937,16 +3938,16 @@ var _ = Describe("Processor", Ordered, func() { }, } - transformerResponses := []transformer.TransformerResponse{ + transformerResponses := []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, StatusCode: 400, Error: "error-1", }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, StatusCode: 400, @@ -3996,8 +3997,8 @@ var _ = Describe("Processor", Ordered, func() { }).Return(jobsdb.JobsResult{Jobs: unprocessedJobsList}, nil).Times(1) // Test transformer failure mockTransformer.EXPECT().Transform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1). - Return(transformer.Response{ - Events: []transformer.TransformerResponse{}, + Return(types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: transformerResponses, }) @@ -4075,9 +4076,9 @@ var _ = Describe("Processor", Ordered, func() { }, } - transformerResponses := []transformer.TransformerResponse{ + transformerResponses := []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageIDs: []string{"message-1", "message-2"}, }, StatusCode: 400, @@ -4134,8 +4135,8 @@ var _ = Describe("Processor", Ordered, func() { // Test transformer failure mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1). - Return(transformer.Response{ - Events: []transformer.TransformerResponse{}, + Return(types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: transformerResponses, }) @@ -4895,7 +4896,7 @@ var _ = Describe("Processor", Ordered, func() { defer cancel() Expect(processor.config.asyncInit.WaitContext(ctx)).To(BeNil()) - commonMetadata := transformer.Metadata{SourceID: SourceIDEnabled, DestinationID: DestinationIDEnabledA} + commonMetadata := types.Metadata{SourceID: SourceIDEnabled, DestinationID: DestinationIDEnabledA} singularEventWithReceivedAt1 := types.SingularEventWithReceivedAt{ SingularEvent: event1, ReceivedAt: time.Now(), @@ -4920,22 +4921,22 @@ var _ = Describe("Processor", Ordered, func() { metadata3 := commonMetadata metadata3.MessageID = "msg3" - FailedEvents := []transformer.TransformerResponse{ + FailedEvents := []types.TransformerResponse{ {StatusCode: 400, Metadata: metadata1, Output: event1}, {StatusCode: 298, Metadata: metadata2, Output: event2}, {StatusCode: 299, Metadata: metadata3, Output: event2}, } - transformerResponse := transformer.Response{ - Events: []transformer.TransformerResponse{}, + transformerResponse := types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: FailedEvents, } m := processor.getNonSuccessfulMetrics(transformerResponse, &commonMetadata, eventsByMessageID, - types.EVENT_FILTER, - types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, + reportingTypes.DEST_TRANSFORMER, ) key := strings.Join([]string{ @@ -4948,7 +4949,7 @@ var _ = Describe("Processor", Ordered, func() { Expect(len(m.failedJobs)).To(Equal(2)) Expect(len(m.failedMetrics)).To(Equal(2)) - slices.SortFunc(m.failedMetrics, func(a, b *types.PUReportedMetric) int { + slices.SortFunc(m.failedMetrics, func(a, b *reportingTypes.PUReportedMetric) int { return a.StatusDetail.StatusCode - b.StatusDetail.StatusCode }) Expect(m.failedMetrics[0].StatusDetail.StatusCode).To(Equal(299)) @@ -4967,9 +4968,9 @@ var _ = Describe("Static Function Tests", func() { Context("TransformerFormatResponse Tests", func() { It("Should match ConvertToTransformerResponse without filtering", func() { - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -4977,7 +4978,7 @@ var _ = Describe("Static Function Tests", func() { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -4985,14 +4986,14 @@ var _ = Describe("Static Function Tests", func() { }, }, } - expectedResponses := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponses := types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "some-key-1": "some-value-1", }, StatusCode: 200, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, }, @@ -5001,13 +5002,13 @@ var _ = Describe("Static Function Tests", func() { "some-key-2": "some-value-2", }, StatusCode: 200, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, }, }, } - response := ConvertToFilteredTransformerResponse(events, false, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, false, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response.Events[0].StatusCode).To(Equal(expectedResponses.Events[0].StatusCode)) Expect(response.Events[0].Metadata.MessageID).To(Equal(expectedResponses.Events[0].Metadata.MessageID)) Expect(response.Events[0].Output["some-key-1"]).To(Equal(expectedResponses.Events[0].Output["some-key-1"])) @@ -5067,22 +5068,22 @@ var _ = Describe("Static Function Tests", func() { "some-key-2": 3, } - expectedResponse := []*types.PUReportedMetric{ + expectedResponse := []*reportingTypes.PUReportedMetric{ { - ConnectionDetails: types.ConnectionDetails{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: "some-source-id-1", DestinationID: "some-destination-id-1", SourceTaskRunID: "some-source-task-run-id-1", SourceJobID: "some-source-job-id-1", SourceJobRunID: "some-source-job-run-id-1", }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: "some-string-1", PU: "some-string-2", TerminalPU: false, InitialPU: false, }, - StatusDetail: &types.StatusDetail{ + StatusDetail: &reportingTypes.StatusDetail{ Status: "diff", Count: 5, StatusCode: 0, @@ -5091,20 +5092,20 @@ var _ = Describe("Static Function Tests", func() { }, }, { - ConnectionDetails: types.ConnectionDetails{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: "some-source-id-2", DestinationID: "some-destination-id-2", SourceTaskRunID: "some-source-task-run-id-2", SourceJobID: "some-source-job-id-2", SourceJobRunID: "some-source-job-run-id-2", }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: "some-string-1", PU: "some-string-2", TerminalPU: false, InitialPU: false, }, - StatusDetail: &types.StatusDetail{ + StatusDetail: &reportingTypes.StatusDetail{ Status: "diff", Count: 7, StatusCode: 0, @@ -5146,8 +5147,8 @@ var _ = Describe("Static Function Tests", func() { proc.reportingEnabled = true proc.reporting = &mockReportingTypes.MockReporting{} - inputEvent := &transformer.TransformerResponse{ - Metadata: transformer.Metadata{ + inputEvent := &types.TransformerResponse{ + Metadata: types.Metadata{ SourceID: "source-id-1", DestinationID: "destination-id-1", TransformationID: "transformation-id-1", @@ -5157,7 +5158,7 @@ var _ = Describe("Static Function Tests", func() { }, StatusCode: 200, Error: "", - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "type-1", Message: "message-1", @@ -5186,7 +5187,7 @@ var _ = Describe("Static Function Tests", func() { trackingPlanID: inputEvent.Metadata.TrackingPlanID, trackingPlanVersion: inputEvent.Metadata.TrackingPlanVersion, } - expectedConnectionDetails := &types.ConnectionDetails{ + expectedConnectionDetails := &reportingTypes.ConnectionDetails{ SourceID: inputEvent.Metadata.SourceID, DestinationID: inputEvent.Metadata.DestinationID, SourceJobRunID: inputEvent.Metadata.SourceJobRunID, @@ -5200,12 +5201,12 @@ var _ = Describe("Static Function Tests", func() { TrackingPlanID: inputEvent.Metadata.TrackingPlanID, TrackingPlanVersion: inputEvent.Metadata.TrackingPlanVersion, } - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) countMap := make(map[string]int64) countMetadataMap := make(map[string]MetricMetadata) // update metric maps - proc.updateMetricMaps(countMetadataMap, countMap, connectionDetailsMap, statusDetailsMap, inputEvent, jobsdb.Succeeded.State, types.TRACKINGPLAN_VALIDATOR, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(countMetadataMap, countMap, connectionDetailsMap, statusDetailsMap, inputEvent, jobsdb.Succeeded.State, reportingTypes.TRACKINGPLAN_VALIDATOR, func() json.RawMessage { return []byte(`{}`) }, nil) Expect(len(countMetadataMap)).To(Equal(1)) Expect(len(countMap)).To(Equal(1)) @@ -5234,9 +5235,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5246,7 +5247,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5256,7 +5257,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5266,15 +5267,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, Metadata: events[0].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 298, @@ -5289,7 +5290,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(_ transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(_ types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5307,9 +5308,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5319,7 +5320,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5329,7 +5330,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5339,15 +5340,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 200, Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5362,7 +5363,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5373,9 +5374,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5385,7 +5386,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5395,7 +5396,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5405,8 +5406,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5425,7 +5426,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) It("Should allow all events when supportedMessageTypes is not an array", func() { @@ -5437,9 +5438,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5449,7 +5450,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5459,7 +5460,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5469,8 +5470,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5489,7 +5490,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5505,9 +5506,9 @@ var _ = Describe("Static Function Tests", func() { DestinationDefinition: backendconfig.DestinationDefinitionT{}, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5517,7 +5518,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5527,7 +5528,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5537,15 +5538,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 200, Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5560,7 +5561,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5578,9 +5579,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5590,7 +5591,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5600,7 +5601,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5610,8 +5611,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5630,7 +5631,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5668,9 +5669,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5681,7 +5682,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5692,7 +5693,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5703,15 +5704,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, Metadata: events[0].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 298, @@ -5726,7 +5727,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5764,9 +5765,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5777,7 +5778,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5788,7 +5789,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5799,9 +5800,9 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ + expectedResponse := types.Response{ Events: nil, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5822,7 +5823,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5860,9 +5861,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5873,7 +5874,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5884,7 +5885,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5895,8 +5896,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5908,7 +5909,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -5917,7 +5918,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5950,9 +5951,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5963,7 +5964,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5974,7 +5975,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5985,8 +5986,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5998,7 +5999,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6007,7 +6008,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -6043,9 +6044,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -6056,7 +6057,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -6067,7 +6068,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -6078,8 +6079,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -6091,7 +6092,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6100,7 +6101,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -6139,9 +6140,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "", }, @@ -6152,7 +6153,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "", }, @@ -6163,7 +6164,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "", }, @@ -6174,8 +6175,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -6187,7 +6188,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6196,7 +6197,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) }) @@ -6275,7 +6276,7 @@ func assertJobStatus(job *jobsdb.JobT, status *jobsdb.JobStatusT, expectedState Expect(status.ExecTime).To(BeTemporally("~", time.Now(), 200*time.Millisecond)) } -func assertReportMetric(expectedMetric, actualMetric []*types.PUReportedMetric) { +func assertReportMetric(expectedMetric, actualMetric []*reportingTypes.PUReportedMetric) { sort.Slice(expectedMetric, func(i, j int) bool { return expectedMetric[i].ConnectionDetails.SourceID < expectedMetric[j].ConnectionDetails.SourceID }) @@ -6308,14 +6309,14 @@ func assertDestinationTransform( expectations transformExpectation, ) func( ctx context.Context, - clientEvents []transformer.TransformerEvent, + clientEvents []types.TransformerEvent, batchSize int, -) transformer.Response { +) types.Response { return func( ctx context.Context, - clientEvents []transformer.TransformerEvent, + clientEvents []types.TransformerEvent, batchSize int, - ) transformer.Response { + ) types.Response { defer GinkgoRecover() Expect(clientEvents).To(HaveLen(expectations.events)) @@ -6395,14 +6396,14 @@ func assertDestinationTransform( Expect(strings.Join(messageIDs, ",")).To(Equal(expectations.messageIds)) - return transformer.Response{ - Events: []transformer.TransformerResponse{ + return types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "int-value": 0, "string-value": fmt.Sprintf("value-%s", destinationID), }, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ SourceID: "source-from-transformer", // transformer should replay source id SourceName: "source-from-transformer-name", DestinationID: "destination-from-transformer", // transformer should replay destination id @@ -6413,7 +6414,7 @@ func assertDestinationTransform( "int-value": 1, "string-value": fmt.Sprintf("value-%s", destinationID), }, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ SourceID: "source-from-transformer", // transformer should replay source id SourceName: "source-from-transformer-name", DestinationID: "destination-from-transformer", // transformer should replay destination id @@ -6617,7 +6618,7 @@ var _ = Describe("TestConfigFilter", func() { Enabled: true, IsProcessorEnabled: true, } - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6636,7 +6637,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6668,7 +6669,7 @@ var _ = Describe("TestConfigFilter", func() { Expect(json.Unmarshal([]byte(intgConfigStr), &intgConfig)).To(BeNil()) Expect(json.Unmarshal([]byte(destDefStr), &destDef)).To(BeNil()) intgConfig.DestinationDefinition = destDef - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6687,7 +6688,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6714,7 +6715,7 @@ var _ = Describe("TestConfigFilter", func() { Enabled: true, IsProcessorEnabled: true, } - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6735,7 +6736,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6749,12 +6750,12 @@ var _ = Describe("TestConfigFilter", func() { func Test_GetTimestampFromEvent(t *testing.T) { input := []struct { - event transformer.TransformerEvent + event types.TransformerEvent timestamp time.Time expectedTimeStamp time.Time }{ { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6763,7 +6764,7 @@ func Test_GetTimestampFromEvent(t *testing.T) { expectedTimeStamp: time.Date(2021, 6, 9, 9, 0, 0, 0, time.UTC), }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, timestamp: time.Now(), @@ -6781,7 +6782,7 @@ func Test_GetTimestampFromEvent(t *testing.T) { func Test_EnhanceWithTimeFields(t *testing.T) { input := []struct { - event transformer.TransformerEvent + event types.TransformerEvent singularEvent types.SingularEventT recievedAt time.Time expectedSentAt string @@ -6789,7 +6790,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp string }{ { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6804,7 +6805,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6816,7 +6817,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, singularEvent: types.SingularEventT{}, @@ -6826,7 +6827,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, singularEvent: types.SingularEventT{ @@ -6860,7 +6861,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 1}}, routerDestIDs: []string{"1"}, - reportMetrics: []*types.PUReportedMetric{{}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 1}, dedupKeys: map[string]struct{}{"1": {}}, totalEvents: 1, @@ -6876,7 +6877,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 2}}, routerDestIDs: []string{"2"}, - reportMetrics: []*types.PUReportedMetric{{}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 2}, dedupKeys: map[string]struct{}{"2": {}}, totalEvents: 1, @@ -6894,7 +6895,7 @@ func TestStoreMessageMerge(t *testing.T) { }, []*jobsdb.JobT{{JobID: 3}}, []string{"3"}, - []*types.PUReportedMetric{{}}, + []*reportingTypes.PUReportedMetric{{}}, map[dupStatKey]int{{sourceID: "1"}: 3}, map[string]struct{}{"3": {}}, 1, @@ -6958,7 +6959,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 1}, {JobID: 2}, {JobID: 3}}, routerDestIDs: []string{"1", "2", "3"}, - reportMetrics: []*types.PUReportedMetric{{}, {}, {}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}, {}, {}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 6}, dedupKeys: map[string]struct{}{"1": {}, "2": {}, "3": {}}, totalEvents: 3, diff --git a/processor/trackingplan.go b/processor/trackingplan.go index a085517b6f..0322277e6a 100644 --- a/processor/trackingplan.go +++ b/processor/trackingplan.go @@ -7,11 +7,12 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/stats" + backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" ) type TrackingPlanStatT struct { @@ -24,7 +25,7 @@ type TrackingPlanStatT struct { // reportViolations It is going add violationErrors in context depending upon certain criteria: // 1. sourceSchemaConfig in Metadata.MergedTpConfig should be true -func reportViolations(validateEvent *transformer.TransformerResponse, trackingPlanId string, trackingPlanVersion int) { +func reportViolations(validateEvent *types.TransformerResponse, trackingPlanId string, trackingPlanVersion int) { if validateEvent.Metadata.MergedTpConfig["propagateValidationErrors"] == "false" { return } @@ -52,7 +53,7 @@ func reportViolations(validateEvent *transformer.TransformerResponse, trackingPl // enhanceWithViolation It enhances extra information of ValidationErrors in context for: // 1. response.Events // 1. response.FailedEvents -func enhanceWithViolation(response transformer.Response, trackingPlanId string, trackingPlanVersion int) { +func enhanceWithViolation(response types.Response, trackingPlanId string, trackingPlanVersion int) { for i := range response.Events { validatedEvent := &response.Events[i] reportViolations(validatedEvent, trackingPlanId, trackingPlanVersion) @@ -68,9 +69,9 @@ func enhanceWithViolation(response transformer.Response, trackingPlanId string, // The Response will contain both the Events and FailedEvents // 1. eventsToTransform gets added to validatedEventsBySourceId // 2. failedJobs gets added to validatedErrorJobs -func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]transformer.TransformerEvent, eventsByMessageID map[string]types.SingularEventWithReceivedAt) (map[SourceIDT][]transformer.TransformerEvent, []*types.PUReportedMetric, []*jobsdb.JobT, map[SourceIDT]bool) { - validatedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) - validatedReportMetrics := make([]*types.PUReportedMetric, 0) +func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]types.TransformerEvent, eventsByMessageID map[string]types.SingularEventWithReceivedAt) (map[SourceIDT][]types.TransformerEvent, []*reportingTypes.PUReportedMetric, []*jobsdb.JobT, map[SourceIDT]bool) { + validatedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) + validatedReportMetrics := make([]*reportingTypes.PUReportedMetric, 0) validatedErrorJobs := make([]*jobsdb.JobT, 0) trackingPlanEnabledMap := make(map[SourceIDT]bool) @@ -84,7 +85,7 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans isTpExists := eventList[0].Metadata.TrackingPlanID != "" if !isTpExists { // pass on the jobs for transformation(User, Dest) - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventList...) continue } @@ -101,7 +102,7 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans // This is a safety check we are adding so that if something unexpected comes from transformer // We are ignoring it. if (len(response.Events) + len(response.FailedEvents)) != len(eventList) { - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventList...) continue } @@ -111,9 +112,9 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans // This is being used to distinguish the flows in reporting service trackingPlanEnabledMap[sourceId] = true - var successMetrics []*types.PUReportedMetric - eventsToTransform, successMetrics, _, _ := proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, backendconfig.Connection{}, types.DESTINATION_FILTER, types.TRACKINGPLAN_VALIDATOR) // Note: Sending false for usertransformation enabled is safe because this stage is before user transformation. - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, types.DESTINATION_FILTER, types.TRACKINGPLAN_VALIDATOR) + var successMetrics []*reportingTypes.PUReportedMetric + eventsToTransform, successMetrics, _, _ := proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, backendconfig.Connection{}, reportingTypes.DESTINATION_FILTER, reportingTypes.TRACKINGPLAN_VALIDATOR) // Note: Sending false for usertransformation enabled is safe because this stage is before user transformation. + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, reportingTypes.DESTINATION_FILTER, reportingTypes.TRACKINGPLAN_VALIDATOR) validationStat.numValidationSuccessEvents.Count(len(eventsToTransform)) validationStat.numValidationFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) @@ -134,16 +135,16 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans if len(eventsToTransform) == 0 { continue } - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventsToTransform...) } return validatedEventsBySourceId, validatedReportMetrics, validatedErrorJobs, trackingPlanEnabledMap } // makeCommonMetadataFromTransformerEvent Creates a new Metadata instance -func makeCommonMetadataFromTransformerEvent(transformerEvent *transformer.TransformerEvent) *transformer.Metadata { +func makeCommonMetadataFromTransformerEvent(transformerEvent *types.TransformerEvent) *types.Metadata { metadata := transformerEvent.Metadata - commonMetaData := transformer.Metadata{ + commonMetaData := types.Metadata{ SourceID: metadata.SourceID, SourceName: metadata.SourceName, SourceType: metadata.SourceType, @@ -159,7 +160,7 @@ func makeCommonMetadataFromTransformerEvent(transformerEvent *transformer.Transf } // newValidationStat Creates a new TrackingPlanStatT instance -func (proc *Handle) newValidationStat(metadata *transformer.Metadata) *TrackingPlanStatT { +func (proc *Handle) newValidationStat(metadata *types.Metadata) *TrackingPlanStatT { tags := map[string]string{ "destination": metadata.DestinationID, "destType": metadata.DestinationType, diff --git a/processor/trackingplan_test.go b/processor/trackingplan_test.go index 88366df505..6ced145fac 100644 --- a/processor/trackingplan_test.go +++ b/processor/trackingplan_test.go @@ -3,13 +3,13 @@ package processor import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/rudderlabs/rudder-server/processor/types" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/stretchr/testify/assert" ) func TestReportViolations(t *testing.T) { - eventsFromTransformerResponse := func(response *transformer.Response) (events []transformer.TransformerResponse) { + eventsFromTransformerResponse := func(response *types.Response) (events []types.TransformerResponse) { events = append(events, response.Events...) events = append(events, response.FailedEvents...) return @@ -21,10 +21,10 @@ func TestReportViolations(t *testing.T) { trackingPlanVersion int ) - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -34,9 +34,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -64,10 +64,10 @@ func TestReportViolations(t *testing.T) { trackingPlanVersion int ) - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -77,9 +77,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -87,7 +87,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": 1234, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -109,10 +109,10 @@ func TestReportViolations(t *testing.T) { }) t.Run("Propagate validation errors", func(t *testing.T) { - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -120,7 +120,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": map[string]interface{}{}, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -132,7 +132,7 @@ func TestReportViolations(t *testing.T) { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -140,7 +140,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": nil, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -152,9 +152,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -162,7 +162,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": map[string]interface{}{}, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -174,13 +174,13 @@ func TestReportViolations(t *testing.T) { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, }, Output: map[string]interface{}{}, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ diff --git a/processor/transformer/transformer.go b/processor/transformer/transformer.go index 57d51d977f..5208f645e7 100644 --- a/processor/transformer/transformer.go +++ b/processor/transformer/transformer.go @@ -16,6 +16,8 @@ import ( "sync" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/bufbuild/httplb" "github.com/bufbuild/httplb/resolver" "github.com/cenkalti/backoff" @@ -26,11 +28,10 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" - backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/processor/integrations" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/httputil" "github.com/rudderlabs/rudder-server/utils/sysUtils" - "github.com/rudderlabs/rudder-server/utils/types" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -48,66 +49,6 @@ const ( var json = jsoniter.ConfigCompatibleWithStandardLibrary -type Metadata struct { - SourceID string `json:"sourceId"` - SourceName string `json:"sourceName"` - OriginalSourceID string `json:"originalSourceId"` - WorkspaceID string `json:"workspaceId"` - Namespace string `json:"namespace"` - InstanceID string `json:"instanceId"` - SourceType string `json:"sourceType"` - SourceCategory string `json:"sourceCategory"` - TrackingPlanID string `json:"trackingPlanId"` - TrackingPlanVersion int `json:"trackingPlanVersion"` - SourceTpConfig map[string]map[string]interface{} `json:"sourceTpConfig"` - MergedTpConfig map[string]interface{} `json:"mergedTpConfig"` - DestinationID string `json:"destinationId"` - JobID int64 `json:"jobId"` - SourceJobID string `json:"sourceJobId"` - SourceJobRunID string `json:"sourceJobRunId"` - SourceTaskRunID string `json:"sourceTaskRunId"` - RecordID interface{} `json:"recordId"` - DestinationType string `json:"destinationType"` - DestinationName string `json:"destinationName"` - MessageID string `json:"messageId"` - OAuthAccessToken string `json:"oauthAccessToken"` - TraceParent string `json:"traceparent"` - // set by user_transformer to indicate transformed event is part of group indicated by messageIDs - MessageIDs []string `json:"messageIds"` - RudderID string `json:"rudderId"` - ReceivedAt string `json:"receivedAt"` - EventName string `json:"eventName"` - EventType string `json:"eventType"` - SourceDefinitionID string `json:"sourceDefinitionId"` - DestinationDefinitionID string `json:"destinationDefinitionId"` - TransformationID string `json:"transformationId"` - TransformationVersionID string `json:"transformationVersionId"` - SourceDefinitionType string `json:"-"` -} - -func (m Metadata) GetMessagesIDs() []string { - if len(m.MessageIDs) > 0 { - return m.MessageIDs - } - return []string{m.MessageID} -} - -type TransformerEvent struct { - Message types.SingularEventT `json:"message"` - Metadata Metadata `json:"metadata"` - Destination backendconfig.DestinationT `json:"destination"` - Connection backendconfig.Connection `json:"connection"` - Libraries []backendconfig.LibraryT `json:"libraries"` - Credentials []Credential `json:"credentials"` -} - -type Credential struct { - ID string `json:"id"` - Key string `json:"key"` - Value string `json:"value"` - IsSecret bool `json:"isSecret"` -} - func isJobTerminated(status int) bool { if status == http.StatusTooManyRequests || status == http.StatusRequestTimeout { return false @@ -115,29 +56,6 @@ func isJobTerminated(status int) bool { return status >= http.StatusOK && status < http.StatusInternalServerError } -type TransformerResponse struct { - // Not marking this Singular Event, since this not a RudderEvent - Output map[string]interface{} `json:"output"` - Metadata Metadata `json:"metadata"` - StatusCode int `json:"statusCode"` - Error string `json:"error"` - ValidationErrors []ValidationError `json:"validationErrors"` - StatTags map[string]string `json:"statTags"` -} - -type ValidationError struct { - Type string `json:"type"` - Message string `json:"message"` - Meta map[string]string `json:"meta"` - Property string `json:"property"` -} - -// Response represents a Transformer response -type Response struct { - Events []TransformerResponse - FailedEvents []TransformerResponse -} - type Opt func(*handle) func WithClient(client HTTPDoer) Opt { @@ -148,9 +66,9 @@ func WithClient(client HTTPDoer) Opt { // Transformer provides methods to transform events type Transformer interface { - Transform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response - UserTransform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response - Validate(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response + Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response + UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response + Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response } type HTTPDoer interface { @@ -264,17 +182,17 @@ func NewTransformer(conf *config.Config, log logger.Logger, stat stats.Stats, op } // Transform function is used to invoke destination transformer API -func (trans *handle) Transform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.destTransformURL(clientEvents[0].Destination.DestinationDefinition.Name), batchSize, destTransformerStage) } // UserTransform function is used to invoke user transformer API -func (trans *handle) UserTransform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.userTransformURL(), batchSize, userTransformerStage) } // Validate function is used to invoke tracking plan validation API -func (trans *handle) Validate(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.trackingPlanValidationURL(), batchSize, trackingPlanValidationStage) } @@ -288,13 +206,13 @@ func (t *HTTPLBTransport) NewRoundTripper(scheme, target string, config httplb.T func (trans *handle) transform( ctx context.Context, - clientEvents []TransformerEvent, + clientEvents []types.TransformerEvent, url string, batchSize int, stage string, -) Response { +) types.Response { if len(clientEvents) == 0 { - return Response{} + return types.Response{} } // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation // flip back afterwards @@ -334,14 +252,14 @@ func (trans *handle) transform( ).Observe(float64(len(batches))) trace.Logf(ctx, "request", "batch_count: %d", len(batches)) - transformResponse := make([][]TransformerResponse, len(batches)) + transformResponse := make([][]types.TransformerResponse, len(batches)) var wg sync.WaitGroup wg.Add(len(batches)) lo.ForEach( batches, - func(batch []TransformerEvent, i int) { + func(batch []types.TransformerEvent, i int) { trans.guardConcurrency <- struct{}{} go func() { trace.WithRegion(ctx, "request", func() { @@ -354,8 +272,8 @@ func (trans *handle) transform( ) wg.Wait() - var outClientEvents []TransformerResponse - var failedEvents []TransformerResponse + var outClientEvents []types.TransformerResponse + var failedEvents []types.TransformerResponse for _, batch := range transformResponse { // Transform is one to many mapping so returned @@ -376,13 +294,13 @@ func (trans *handle) transform( trans.sentStat.Count(len(clientEvents)) trans.receivedStat.Count(len(outClientEvents)) - return Response{ + return types.Response{ Events: outClientEvents, FailedEvents: failedEvents, } } -func (trans *handle) request(ctx context.Context, url, stage string, data []TransformerEvent) []TransformerResponse { +func (trans *handle) request(ctx context.Context, url, stage string, data []types.TransformerEvent) []types.TransformerResponse { // Call remote transformation var ( rawJSON []byte @@ -462,7 +380,7 @@ func (trans *handle) request(ctx context.Context, url, stage string, data []Tran trans.logger.Errorf("Transformer returned status code: %v", statusCode) } - var transformerResponses []TransformerResponse + var transformerResponses []types.TransformerResponse switch statusCode { case http.StatusOK: integrations.CollectIntgTransformErrorStats(respData) @@ -481,7 +399,7 @@ func (trans *handle) request(ctx context.Context, url, stage string, data []Tran default: for i := range data { transformEvent := &data[i] - resp := TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} + resp := types.TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} transformerResponses = append(transformerResponses, resp) } } @@ -554,8 +472,8 @@ func (trans *handle) doPost(ctx context.Context, rawJSON []byte, url, stage stri // perform version compatibility check only on success if resp.StatusCode == http.StatusOK { transformerAPIVersion, _ := strconv.Atoi(resp.Header.Get("apiVersion")) - if types.SupportedTransformerApiVersion != transformerAPIVersion { - unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", types.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) + if reportingTypes.SupportedTransformerApiVersion != transformerAPIVersion { + unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", reportingTypes.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) trans.logger.Error(unexpectedVersionError) panic(unexpectedVersionError) } diff --git a/processor/transformer/transformer_test.go b/processor/transformer/transformer_test.go index a920831a99..47007fefc2 100644 --- a/processor/transformer/transformer_test.go +++ b/processor/transformer/transformer_test.go @@ -13,46 +13,45 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" - "github.com/rudderlabs/rudder-go-kit/testhelper/rand" - "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" - warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" - - "github.com/rudderlabs/rudder-server/utils/types" + "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" + "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/logger/mock_logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-go-kit/stats/memstats" + "github.com/rudderlabs/rudder-go-kit/testhelper/rand" backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/gateway/response" - - "github.com/rudderlabs/rudder-go-kit/config" - "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" + warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) type fakeTransformer struct { - requests [][]TransformerEvent + requests [][]types.TransformerEvent t testing.TB } func (t *fakeTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(t.t, json.NewDecoder(r.Body).Decode(&reqBody)) t.requests = append(t.requests, reqBody) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) for i := range reqBody { statusCode := int(reqBody[i].Message["forceStatusCode"].(float64)) delete(reqBody[i].Message, "forceStatusCode") reqBody[i].Message["echo-key-1"] = reqBody[i].Message["src-key-1"] - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: statusCode, @@ -62,7 +61,7 @@ func (t *fakeTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - w.Header().Set("apiVersion", strconv.Itoa(types.SupportedTransformerApiVersion)) + w.Header().Set("apiVersion", strconv.Itoa(reportingTypes.SupportedTransformerApiVersion)) require.NoError(t.t, json.NewEncoder(w).Encode(responses)) } @@ -83,10 +82,10 @@ type endlessLoopTransformer struct { func (elt *endlessLoopTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { elt.retryCount++ - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(elt.t, json.NewDecoder(r.Body).Decode(&reqBody)) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) if elt.retryCount < elt.maxRetryCount { http.Error(w, response.MakeResponse(elt.statusError), elt.statusCode) @@ -94,7 +93,7 @@ func (elt *endlessLoopTransformer) ServeHTTP(w http.ResponseWriter, r *http.Requ } for i := range reqBody { - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: http.StatusOK, @@ -122,13 +121,13 @@ func (et *endpointTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) return } - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(et.t, json.NewDecoder(r.Body).Decode(&reqBody)) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) for i := range reqBody { - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: http.StatusOK, @@ -136,7 +135,7 @@ func (et *endpointTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) } } - w.Header().Set("apiVersion", strconv.Itoa(types.SupportedTransformerApiVersion)) + w.Header().Set("apiVersion", strconv.Itoa(reportingTypes.SupportedTransformerApiVersion)) require.NoError(et.t, json.NewEncoder(w).Encode(responses)) } @@ -194,15 +193,15 @@ func TestTransformer(t *testing.T) { eventsCount := tt.eventsCount failEvery := tt.failEvery - events := make([]TransformerEvent, eventsCount) - expectedResponse := Response{} + events := make([]types.TransformerEvent, eventsCount) + expectedResponse := types.Response{} transformationID := rand.String(10) destinationConfig := backendconfigtest.NewDestinationBuilder("WEBHOOK"). WithUserTransformation(transformationID, rand.String(10)).Build() - metadata := Metadata{ + Metadata := types.Metadata{ DestinationType: destinationConfig.DestinationDefinition.Name, SourceID: rand.String(10), DestinationID: destinationConfig.ID, @@ -217,17 +216,17 @@ func TestTransformer(t *testing.T) { statusCode = http.StatusBadRequest } - metadata := metadata - metadata.MessageID = msgID + Metadata := Metadata + Metadata.MessageID = msgID - events[i] = TransformerEvent{ - Metadata: metadata, + events[i] = types.TransformerEvent{ + Metadata: Metadata, Message: map[string]interface{}{ "src-key-1": msgID, "forceStatusCode": statusCode, }, Destination: destinationConfig, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -237,8 +236,8 @@ func TestTransformer(t *testing.T) { }, } - tResp := TransformerResponse{ - Metadata: metadata, + tResp := types.TransformerResponse{ + Metadata: Metadata, StatusCode: statusCode, Output: map[string]interface{}{ "src-key-1": msgID, @@ -263,7 +262,7 @@ func TestTransformer(t *testing.T) { for _, m := range metrics { require.Equal(t, stats.Tags{ "stage": "test-stage", - "sourceId": metadata.SourceID, + "sourceId": Metadata.SourceID, "destinationType": destinationConfig.DestinationDefinition.Name, "destinationId": destinationConfig.ID, "transformationId": destinationConfig.Transformations[0].ID, @@ -271,7 +270,7 @@ func TestTransformer(t *testing.T) { // Legacy tags: to be removed "dest_type": destinationConfig.DestinationDefinition.Name, "dest_id": destinationConfig.ID, - "src_id": metadata.SourceID, + "src_id": Metadata.SourceID, }, m.Tags) } } @@ -280,14 +279,14 @@ func TestTransformer(t *testing.T) { t.Run("timeout", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ "src-key-1": msgID, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -303,7 +302,7 @@ func TestTransformer(t *testing.T) { expectedRetries int expectPanic bool stage string - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse failOnUserTransformTimeout bool }{ { @@ -318,9 +317,9 @@ func TestTransformer(t *testing.T) { retries: 3, stage: userTransformerStage, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: TransformerRequestTimeout, @@ -385,7 +384,7 @@ func TestTransformer(t *testing.T) { rsp := tr.request(context.TODO(), srv.URL, tc.stage, events) require.Len(t, rsp, 1) require.Equal(t, rsp[0].StatusCode, TransformerRequestTimeout) - require.Equal(t, rsp[0].Metadata, Metadata{ + require.Equal(t, rsp[0].Metadata, types.Metadata{ MessageID: msgID, }) require.Contains(t, rsp[0].Error, "transformer request timed out:") @@ -396,14 +395,14 @@ func TestTransformer(t *testing.T) { t.Run("endless retries in case of control plane down", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ "src-key-1": msgID, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -417,7 +416,7 @@ func TestTransformer(t *testing.T) { maxRetryCount: 3, statusCode: StatusCPDown, statusError: "control plane not reachable", - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, t: t, } @@ -436,9 +435,9 @@ func TestTransformer(t *testing.T) { tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) - require.Equal(t, rsp, []TransformerResponse{ + require.Equal(t, rsp, []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -452,8 +451,8 @@ func TestTransformer(t *testing.T) { t.Run("retries", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -467,7 +466,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -485,7 +484,7 @@ func TestTransformer(t *testing.T) { statusError string expectedRetries int expectPanic bool - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse failOnError bool }{ { @@ -506,9 +505,9 @@ func TestTransformer(t *testing.T) { statusError: "too many requests", expectedRetries: 4, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: TransformerRequestFailure, @@ -525,9 +524,9 @@ func TestTransformer(t *testing.T) { statusError: "control plane not reachable", expectedRetries: 3, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -548,7 +547,7 @@ func TestTransformer(t *testing.T) { maxRetryCount: tc.maxRetryCount, statusCode: tc.statusCode, statusError: tc.statusError, - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, t: t, } @@ -584,8 +583,8 @@ func TestTransformer(t *testing.T) { t.Run("version compatibility", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -599,7 +598,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -614,15 +613,15 @@ func TestTransformer(t *testing.T) { apiVersion int skipApiVersion bool expectPanic bool - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse }{ { name: "compatible api version", - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -683,21 +682,21 @@ func TestTransformer(t *testing.T) { t.Run("endpoints", func(t *testing.T) { msgID := "messageID-0" - expectedResponse := Response{ - Events: []TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "src-key-1": msgID, }, - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, }, }, } - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -714,7 +713,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -779,8 +778,8 @@ func TestTransformer(t *testing.T) { tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -797,7 +796,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", diff --git a/processor/types/types.go b/processor/types/types.go new file mode 100644 index 0000000000..e73d95aace --- /dev/null +++ b/processor/types/types.go @@ -0,0 +1,122 @@ +package types + +import ( + "time" + + backendconfig "github.com/rudderlabs/rudder-server/backend-config" +) + +// SingularEventT single event structure +type SingularEventT map[string]interface{} + +// GetRudderEventVal returns the value corresponding to the key in the message structure +func GetRudderEventVal(key string, rudderEvent SingularEventT) (interface{}, bool) { + rudderVal, ok := rudderEvent[key] + if !ok { + return nil, false + } + return rudderVal, true +} + +type SingularEventWithReceivedAt struct { + SingularEvent SingularEventT + ReceivedAt time.Time +} + +// GatewayBatchRequest batch request structure +type GatewayBatchRequest struct { + Batch []SingularEventT `json:"batch"` + RequestIP string `json:"requestIP"` + ReceivedAt time.Time `json:"receivedAt"` +} + +type TransformerEvent struct { + Message SingularEventT `json:"message"` + Metadata Metadata `json:"metadata"` + Destination backendconfig.DestinationT `json:"destination"` + Connection backendconfig.Connection `json:"connection"` + Libraries []backendconfig.LibraryT `json:"libraries"` + Credentials []Credential `json:"credentials"` +} + +type Credential struct { + ID string `json:"id"` + Key string `json:"key"` + Value string `json:"value"` + IsSecret bool `json:"isSecret"` +} + +type Metadata struct { + SourceID string `json:"sourceId"` + SourceName string `json:"sourceName"` + OriginalSourceID string `json:"originalSourceId"` + WorkspaceID string `json:"workspaceId"` + Namespace string `json:"namespace"` + InstanceID string `json:"instanceId"` + SourceType string `json:"sourceType"` + SourceCategory string `json:"sourceCategory"` + TrackingPlanID string `json:"trackingPlanId"` + TrackingPlanVersion int `json:"trackingPlanVersion"` + SourceTpConfig map[string]map[string]interface{} `json:"sourceTpConfig"` + MergedTpConfig map[string]interface{} `json:"mergedTpConfig"` + DestinationID string `json:"destinationId"` + JobID int64 `json:"jobId"` + SourceJobID string `json:"sourceJobId"` + SourceJobRunID string `json:"sourceJobRunId"` + SourceTaskRunID string `json:"sourceTaskRunId"` + RecordID interface{} `json:"recordId"` + DestinationType string `json:"destinationType"` + DestinationName string `json:"destinationName"` + MessageID string `json:"messageId"` + OAuthAccessToken string `json:"oauthAccessToken"` + TraceParent string `json:"traceparent"` + // set by user_transformer to indicate transformed event is part of group indicated by messageIDs + MessageIDs []string `json:"messageIds"` + RudderID string `json:"rudderId"` + ReceivedAt string `json:"receivedAt"` + EventName string `json:"eventName"` + EventType string `json:"eventType"` + SourceDefinitionID string `json:"sourceDefinitionId"` + DestinationDefinitionID string `json:"destinationDefinitionId"` + TransformationID string `json:"transformationId"` + TransformationVersionID string `json:"transformationVersionId"` + SourceDefinitionType string `json:"-"` +} + +func (m Metadata) GetMessagesIDs() []string { + if len(m.MessageIDs) > 0 { + return m.MessageIDs + } + return []string{m.MessageID} +} + +type TransformerResponse struct { + // Not marking this Singular Event, since this not a RudderEvent + Output map[string]interface{} `json:"output"` + Metadata Metadata `json:"metadata"` + StatusCode int `json:"statusCode"` + Error string `json:"error"` + ValidationErrors []ValidationError `json:"validationErrors"` + StatTags map[string]string `json:"statTags"` +} + +type ValidationError struct { + Type string `json:"type"` + Message string `json:"message"` + Meta map[string]string `json:"meta"` + Property string `json:"property"` +} + +// Response represents a Transformer response +type Response struct { + Events []TransformerResponse + FailedEvents []TransformerResponse +} + +type EventParams struct { + SourceJobRunId string `json:"source_job_run_id"` + SourceId string `json:"source_id"` + SourceTaskRunId string `json:"source_task_run_id"` + TraceParent string `json:"traceparent"` + DestinationID string `json:"destination_id"` +} diff --git a/services/debugger/transformation/transformationStatusUploader.go b/services/debugger/transformation/transformationStatusUploader.go index ad79da6426..6cb63312c3 100644 --- a/services/debugger/transformation/transformationStatusUploader.go +++ b/services/debugger/transformation/transformationStatusUploader.go @@ -6,6 +6,8 @@ import ( "sync" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + jsoniter "github.com/json-iterator/go" "github.com/samber/lo" @@ -13,21 +15,20 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/rruntime" "github.com/rudderlabs/rudder-server/services/debugger" "github.com/rudderlabs/rudder-server/services/debugger/cache" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) type TransformationStatusT struct { SourceID string DestID string Destination *backendconfig.DestinationT - UserTransformedEvents []transformer.TransformerEvent + UserTransformedEvents []types.TransformerEvent EventsByMessageID map[string]types.SingularEventWithReceivedAt - FailedEvents []transformer.TransformerResponse + FailedEvents []types.TransformerResponse UniqueMessageIds map[string]struct{} } @@ -233,13 +234,13 @@ func (ts *TransformationStatusT) Limit( append( lo.Map( ts.UserTransformedEvents, - func(event transformer.TransformerEvent, _ int) string { + func(event types.TransformerEvent, _ int) string { return event.Metadata.MessageID }, ), lo.Map( ts.FailedEvents, - func(event transformer.TransformerResponse, _ int) string { + func(event types.TransformerResponse, _ int) string { return event.Metadata.MessageID }, )..., @@ -383,7 +384,7 @@ func (h *Handle) processRecordTransformationStatus(tStatus *TransformationStatus } var isError bool switch failedEvent.StatusCode { - case types.FilterEventCode: + case reportingTypes.FilterEventCode: eventAfter.IsDropped = true isError = false default: diff --git a/services/debugger/transformation/transformationStatusUploader_test.go b/services/debugger/transformation/transformationStatusUploader_test.go index 9767e8e916..4c96bbbd8d 100644 --- a/services/debugger/transformation/transformationStatusUploader_test.go +++ b/services/debugger/transformation/transformationStatusUploader_test.go @@ -18,10 +18,9 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" mocksBackendConfig "github.com/rudderlabs/rudder-server/mocks/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/pubsub" testutils "github.com/rudderlabs/rudder-server/utils/tests" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( @@ -324,13 +323,13 @@ var _ = Describe("eventDeliveryStatusUploader", func() { func TestLimit(t *testing.T) { var ( singularEvent1 = types.SingularEventT{"payload": "event-1"} - metadata1 = transformer.Metadata{MessageID: "message-id-1"} + metadata1 = types.Metadata{MessageID: "message-id-1"} singularEvent2 = types.SingularEventT{"payload": "event-2"} - metadata2 = transformer.Metadata{MessageID: "message-id-2"} + metadata2 = types.Metadata{MessageID: "message-id-2"} singularEvent3 = types.SingularEventT{"payload": "event-3"} - metadata3 = transformer.Metadata{MessageID: "message-id-3"} + metadata3 = types.Metadata{MessageID: "message-id-3"} singularEvent4 = types.SingularEventT{"payload": "event-4"} - metadata4 = transformer.Metadata{MessageID: "message-id-4"} + metadata4 = types.Metadata{MessageID: "message-id-4"} now = time.Now() limit = 1 ) @@ -339,7 +338,7 @@ func TestLimit(t *testing.T) { Destination: &sampleBackendConfig.Sources[1].Destinations[1], DestID: sampleBackendConfig.Sources[1].Destinations[1].ID, SourceID: sampleBackendConfig.Sources[1].ID, - UserTransformedEvents: []transformer.TransformerEvent{ + UserTransformedEvents: []types.TransformerEvent{ { Message: singularEvent1, Metadata: metadata1, @@ -367,7 +366,7 @@ func TestLimit(t *testing.T) { ReceivedAt: now, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: singularEvent1, Metadata: metadata3, @@ -401,7 +400,7 @@ func TestLimit(t *testing.T) { ) require.Equal( t, - []transformer.TransformerEvent{ + []types.TransformerEvent{ { Message: singularEvent1, Metadata: metadata1, @@ -411,7 +410,7 @@ func TestLimit(t *testing.T) { ) require.Equal( t, - []transformer.TransformerResponse{ + []types.TransformerResponse{ { Output: singularEvent1, Metadata: metadata3, diff --git a/testhelper/transformertest/builder.go b/testhelper/transformertest/builder.go index cdad3bb214..acd59c638c 100644 --- a/testhelper/transformertest/builder.go +++ b/testhelper/transformertest/builder.go @@ -7,11 +7,10 @@ import ( "net/http/httptest" "strings" - "github.com/rudderlabs/rudder-server/router/types" - "github.com/tidwall/sjson" - "github.com/rudderlabs/rudder-server/processor/transformer" + processorTypes "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/router/types" ) // NewBuilder returns a new test transformer Builder @@ -151,7 +150,7 @@ func transformerFunc(h TransformerHandler) http.HandlerFunc { w.WriteHeader(http.StatusInternalServerError) return } - var request []transformer.TransformerEvent + var request []processorTypes.TransformerEvent if err := json.Unmarshal(data, &request); err != nil { w.WriteHeader(http.StatusBadRequest) return diff --git a/testhelper/transformertest/handler_funcs.go b/testhelper/transformertest/handler_funcs.go index 563c46fdd1..efa0804e8c 100644 --- a/testhelper/transformertest/handler_funcs.go +++ b/testhelper/transformertest/handler_funcs.go @@ -7,20 +7,20 @@ import ( "github.com/rudderlabs/rudder-server/router/types" "github.com/rudderlabs/rudder-server/processor/integrations" - "github.com/rudderlabs/rudder-server/processor/transformer" + processorTypes "github.com/rudderlabs/rudder-server/processor/types" ) // TransformerHandler is a function that takes a transformer request and returns a response -type TransformerHandler func(request []transformer.TransformerEvent) []transformer.TransformerResponse +type TransformerHandler func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse // RouterTransformerHandler is a function that takes a router transformer request and returns a response type RouterTransformerHandler func(request types.TransformMessageT) types.DestinationJobs // MirroringTransformerHandler mirrors the request payload in the response -var MirroringTransformerHandler TransformerHandler = func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { +var MirroringTransformerHandler TransformerHandler = func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, @@ -46,10 +46,10 @@ var MirroringRouterTransformerHandler RouterTransformerHandler = func(request ty // ErrorTransformerHandler mirrors the request payload in the response but uses an error status code func ErrorTransformerHandler(code int, err string) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: code, @@ -61,11 +61,11 @@ func ErrorTransformerHandler(code int, err string) TransformerHandler { } // ViolationErrorTransformerHandler mirrors the request payload in the response but uses an error status code along with the provided validation errors -func ViolationErrorTransformerHandler(code int, err string, validationErrors []transformer.ValidationError) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { +func ViolationErrorTransformerHandler(code int, err string, validationErrors []processorTypes.ValidationError) TransformerHandler { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: code, @@ -78,19 +78,19 @@ func ViolationErrorTransformerHandler(code int, err string, validationErrors []t } // EmptyTransformerHandler returns an empty response -var EmptyTransformerHandler TransformerHandler = func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return []transformer.TransformerResponse{} +var EmptyTransformerHandler TransformerHandler = func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return []processorTypes.TransformerResponse{} } // DestTransformerHandler returns an empty response -func DestTransformerHandler(f func(event transformer.TransformerEvent) integrations.PostParametersT) func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return func(request []transformer.TransformerEvent) (res []transformer.TransformerResponse) { +func DestTransformerHandler(f func(event processorTypes.TransformerEvent) integrations.PostParametersT) func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return func(request []processorTypes.TransformerEvent) (res []processorTypes.TransformerResponse) { for _, req := range request { postParameters := f(req) jsonString, _ := json.Marshal(postParameters) var output map[string]interface{} _ = json.Unmarshal(jsonString, &output) - res = append(res, transformer.TransformerResponse{ + res = append(res, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: output, StatusCode: http.StatusOK, @@ -101,8 +101,8 @@ func DestTransformerHandler(f func(event transformer.TransformerEvent) integrati } // RESTJSONDestTransformerHandler transforms the request payload into a REST JSON destination request using the original message as the payload -func RESTJSONDestTransformerHandler(method, url string) func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return DestTransformerHandler(func(event transformer.TransformerEvent) integrations.PostParametersT { +func RESTJSONDestTransformerHandler(method, url string) func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return DestTransformerHandler(func(event processorTypes.TransformerEvent) integrations.PostParametersT { return integrations.PostParametersT{ Type: "REST", URL: url, @@ -116,10 +116,10 @@ func RESTJSONDestTransformerHandler(method, url string) func(request []transform // WarehouseTransformerHandler mirrors the request payload in the response but uses an error, status code along with warehouse compatible output func WarehouseTransformerHandler(tableName string, code int, err string) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: map[string]interface{}{ "table": tableName, diff --git a/utils/misc/misc.go b/utils/misc/misc.go index 0c0677e114..6f69496816 100644 --- a/utils/misc/misc.go +++ b/utils/misc/misc.go @@ -34,7 +34,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-server/utils/httputil" - "github.com/rudderlabs/rudder-server/utils/types" ) var ( @@ -112,15 +111,6 @@ func GetMD5Hash(input string) string { return hex.EncodeToString(hash[:]) } -// GetRudderEventVal returns the value corresponding to the key in the message structure -func GetRudderEventVal(key string, rudderEvent types.SingularEventT) (interface{}, bool) { - rudderVal, ok := rudderEvent[key] - if !ok { - return nil, false - } - return rudderVal, true -} - // RemoveFilePaths removes filePaths as well as cleans up the empty folder structure. func RemoveFilePaths(filePaths ...string) { for _, fp := range filePaths { diff --git a/utils/types/types.go b/utils/types/types.go index 6e2b0ce3cc..b82a54dd90 100644 --- a/utils/types/types.go +++ b/utils/types/types.go @@ -4,7 +4,6 @@ package types import ( "context" - "time" "github.com/rudderlabs/rudder-server/enterprise/suppress-user/model" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck @@ -16,29 +15,6 @@ const ( DrainEventCode = 410 ) -// SingularEventT single event structure -type SingularEventT map[string]interface{} - -type SingularEventWithReceivedAt struct { - SingularEvent SingularEventT - ReceivedAt time.Time -} - -// GatewayBatchRequest batch request structure -type GatewayBatchRequest struct { - Batch []SingularEventT `json:"batch"` - RequestIP string `json:"requestIP"` - ReceivedAt time.Time `json:"receivedAt"` -} - -type EventParams struct { - SourceJobRunId string `json:"source_job_run_id"` - SourceId string `json:"source_id"` - SourceTaskRunId string `json:"source_task_run_id"` - TraceParent string `json:"traceparent"` - DestinationID string `json:"destination_id"` -} - // UserSuppression is interface to access Suppress user feature type UserSuppression interface { GetSuppressedUser(workspaceID, userID, sourceID string) *model.Metadata diff --git a/warehouse/integrations/testhelper/staging.go b/warehouse/integrations/testhelper/staging.go index b9daf20614..84e07b08c3 100644 --- a/warehouse/integrations/testhelper/staging.go +++ b/warehouse/integrations/testhelper/staging.go @@ -13,6 +13,8 @@ import ( "text/template" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/google/uuid" "github.com/samber/lo" "github.com/stretchr/testify/require" @@ -128,14 +130,14 @@ func prepareStagingFilePathUsingEventsFile(t testing.TB, testConfig *TestConfig) }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) tr := transformer.NewTransformer(c, logger.NOP, stats.Default) response := tr.Transform(context.Background(), transformerEvents, 100) require.Zero(t, len(response.FailedEvents)) - responseOutputs := lo.Map(response.Events, func(r transformer.TransformerResponse, index int) map[string]interface{} { + responseOutputs := lo.Map(response.Events, func(r types.TransformerResponse, index int) map[string]interface{} { return r.Output })