Skip to content

Commit

Permalink
feat: remove unnecessary fields from UT payload (#5422)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cisse21
cisse21 authored Jan 15, 2025
1 parent ef1e202 commit 441f765
Show file tree
Hide file tree
Showing 2 changed files with 159 additions and 12 deletions.
26 changes: 25 additions & 1 deletion processor/transformer/transformer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,24 @@ type TransformerEvent struct {
Credentials []Credential `json:"credentials"`
}

// GetVersionsOnly removes the connection and credentials from the event
// along with pruning the destination to only include the transformation versionID
// before sending it to the transformer thereby reducing the payload size
func (e *TransformerEvent) GetVersionsOnly() *TransformerEvent {
tmCopy := *e
transformations := make([]backendconfig.TransformationT, 0, len(e.Destination.Transformations))
for _, t := range e.Destination.Transformations {
transformations = append(transformations, backendconfig.TransformationT{
VersionID: t.VersionID,
})
}
tmCopy.Destination = backendconfig.DestinationT{
Transformations: transformations,
}
tmCopy.Connection = backendconfig.Connection{}
return &tmCopy
}

type Credential struct {
ID string `json:"id"`
Key string `json:"key"`
Expand Down Expand Up @@ -270,7 +288,13 @@ func (trans *handle) Transform(ctx context.Context, clientEvents []TransformerEv

// UserTransform function is used to invoke user transformer API
func (trans *handle) UserTransform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response {
return trans.transform(ctx, clientEvents, trans.userTransformURL(), batchSize, userTransformerStage)
var dehydratedClientEvents []TransformerEvent
for _, clientEvent := range clientEvents {
dehydratedClientEvent := clientEvent.GetVersionsOnly()
dehydratedClientEvents = append(dehydratedClientEvents, *dehydratedClientEvent)
}

return trans.transform(ctx, dehydratedClientEvents, trans.userTransformURL(), batchSize, userTransformerStage)
}

// Validate function is used to invoke tracking plan validation API
Expand Down
145 changes: 134 additions & 11 deletions processor/transformer/transformer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,22 @@ import (
"testing"
"time"

"go.uber.org/mock/gomock"

"github.com/rudderlabs/rudder-go-kit/testhelper/rand"
"github.com/rudderlabs/rudder-server/testhelper/backendconfigtest"
warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils"

"github.com/rudderlabs/rudder-server/utils/types"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"

"github.com/rudderlabs/rudder-go-kit/config"
"github.com/rudderlabs/rudder-go-kit/logger"
"github.com/rudderlabs/rudder-go-kit/logger/mock_logger"
"github.com/rudderlabs/rudder-go-kit/stats"
"github.com/rudderlabs/rudder-go-kit/stats/memstats"
"github.com/rudderlabs/rudder-go-kit/testhelper/rand"

backendconfig "github.com/rudderlabs/rudder-server/backend-config"
"github.com/rudderlabs/rudder-server/gateway/response"

"github.com/rudderlabs/rudder-go-kit/config"
"github.com/rudderlabs/rudder-go-kit/logger"
"github.com/rudderlabs/rudder-server/testhelper/backendconfigtest"
"github.com/rudderlabs/rudder-server/utils/types"
warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils"
)

type fakeTransformer struct {
Expand Down Expand Up @@ -889,3 +886,129 @@ func TestLongRunningTransformation(t *testing.T) {
cancel()
})
}

func TestTransformerEvent_Dehydrate(t *testing.T) {
testCases := []struct {
name string
event *TransformerEvent
expected *TransformerEvent
}{
{
name: "remove connections",
event: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{
SourceID: "source-id",
DestinationID: "destination-id",
},
Destination: backendconfig.DestinationT{
Transformations: make([]backendconfig.TransformationT, 0),
},
},
expected: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{},
Destination: backendconfig.DestinationT{
Transformations: make([]backendconfig.TransformationT, 0),
},
},
},
{
name: "remove everything except transformations in destination",
event: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{
SourceID: "source-id",
DestinationID: "destination-id",
},
Destination: backendconfig.DestinationT{
ID: "destination-id",
Transformations: make([]backendconfig.TransformationT, 0),
Name: "destination-name",
Config: map[string]interface{}{
"key": "value",
"key2": map[string]interface{}{
"key": "value",
},
"key3": []interface{}{"value"},
},
DestinationDefinition: backendconfig.DestinationDefinitionT{
Name: "destination-definition-name",
Config: map[string]interface{}{},
ResponseRules: map[string]interface{}{},
},
},
},
expected: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{},
Destination: backendconfig.DestinationT{
Transformations: make([]backendconfig.TransformationT, 0),
},
},
},
{
name: "remove everything except transformations in destination with multiple transformer versions",
event: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{
SourceID: "source-id",
DestinationID: "destination-id",
},
Destination: backendconfig.DestinationT{
ID: "destination-id",
Transformations: []backendconfig.TransformationT{
{
ID: "transformation-id",
VersionID: "version-id",
Config: map[string]interface{}{},
},
{
ID: "transformation-id-2",
VersionID: "version-id-2",
Config: map[string]interface{}{},
},
},
Name: "destination-name",
Config: map[string]interface{}{
"key": "value",
"key2": map[string]interface{}{
"key": "value",
},
"key3": []interface{}{"value"},
},
DestinationDefinition: backendconfig.DestinationDefinitionT{
Name: "destination-definition-name",
Config: map[string]interface{}{},
ResponseRules: map[string]interface{}{},
},
},
},
expected: &TransformerEvent{
Metadata: Metadata{},
Message: map[string]interface{}{},
Connection: backendconfig.Connection{},
Destination: backendconfig.DestinationT{
Transformations: []backendconfig.TransformationT{
{
VersionID: "version-id",
},
{
VersionID: "version-id-2",
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, tc.event.GetVersionsOnly())
})
}
}

0 comments on commit 441f765

Please sign in to comment.