Split serverless into independent directory and github-actions runner (…

…#1266)

Signed-off-by: Annanay <[email protected]>

.github/workflows/ci.yml

@@ -42,6 +42,21 @@ jobs:
   integration-tests:
     name: Test integration e2e suite
     runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-e2e
+
+  integration-tests-serverless:
+    name: Test serverless integration e2e suite
+    runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.17
         uses: actions/setup-go@v2
@@ -55,7 +70,7 @@ jobs:
         uses: buildpacks/github-actions/[email protected]
 
       - name: Test
-        run: make test-e2e
+        run: make test-e2e-serverless
 
   build:
     name: Build
@@ -130,7 +145,7 @@ jobs:
           curl -fSL -o jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz
           tar -xvf jsonnet.tar.gz -C /usr/local/bin/
           chmod a+x /usr/local/bin/jsonnet
-          
+
           go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
 
           curl -fSL -o /usr/local/bin/tk https://github.com/grafana/tanka/releases/download/v0.19.0/tk-linux-amd64

Makefile

@@ -73,13 +73,19 @@ benchmark:
 test-with-cover: test-serverless
 	$(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) $(ALL_PKGS)
 
+# runs e2e tests in the top level integration/e2e directory
 .PHONY: test-e2e
-test-e2e: docker-tempo docker-serverless
+test-e2e: docker-tempo
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e
 
+# runs only serverless e2e tests
+.PHONY: test-e2e-serverless
+test-e2e-serverless: docker-tempo docker-serverless
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/serverless
+
 # test-all/bench use a docker image so build it first to make sure we're up to date
 .PHONY: test-all
-test-all: test-with-cover test-e2e
+test-all: test-with-cover test-e2e test-e2e-serverless
 
 .PHONY: test-bench
 test-bench: docker-tempo
@@ -130,7 +136,7 @@ PROTO_INCLUDES = -I$(PROTO_INTERMEDIATE_DIR)
 PROTO_GEN = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
 
 .PHONY: gen-proto
-gen-proto: 
+gen-proto:
 	@echo --
 	@echo -- Deleting existing
 	@echo --
@@ -162,7 +168,7 @@ gen-proto:
 	find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i $(SED_OPTS) 's+import "opentelemetry/proto/+import "+g'
 
 	@echo --
-	@echo -- Gen proto -- 
+	@echo -- Gen proto --
 	@echo --
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/common/v1/common.proto,./pkg/tempopb/)
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/resource/v1/resource.proto,./pkg/tempopb/)

integration/e2e/compression_test.go

@@ -7,11 +7,12 @@ import (
 	"time"
 
 	"github.com/gogo/protobuf/jsonpb"
+	"github.com/stretchr/testify/require"
+
 	"github.com/grafana/e2e"
 	util "github.com/grafana/tempo/integration"
 	"github.com/grafana/tempo/pkg/tempopb"
 	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -28,7 +29,7 @@ func TestCompression(t *testing.T) {
 	require.NoError(t, s.StartAndWaitReady(tempo))
 
 	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := newJaegerGRPCClient(tempo.Endpoint(14250))
+	c, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c)
 

integration/e2e/e2e_test.go

@@ -7,22 +7,18 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"sync"
 	"testing"
 	"text/template"
 	"time"
 
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	jaeger_grpc "github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc"
 	thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/zap"
-	"google.golang.org/grpc"
 	"gopkg.in/yaml.v2"
 
+	"github.com/grafana/e2e"
+	e2edb "github.com/grafana/e2e/db"
 	"github.com/grafana/tempo/cmd/tempo/app"
 	util "github.com/grafana/tempo/integration"
 	"github.com/grafana/tempo/integration/e2e/backend"
@@ -33,7 +29,6 @@ import (
 
 const (
 	configMicroservices = "config-microservices.tmpl.yaml"
-	configServerless    = "config-serverless.yaml"
 	configHA            = "config-scalable-single-binary.yaml"
 
 	configAllInOneS3      = "config-all-in-one-s3.yaml"
@@ -80,7 +75,7 @@ func TestAllInOne(t *testing.T) {
 			require.NoError(t, s.StartAndWaitReady(tempo))
 
 			// Get port for the Jaeger gRPC receiver endpoint
-			c, err := newJaegerGRPCClient(tempo.Endpoint(14250))
+			c, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))
 			require.NoError(t, err)
 			require.NotNil(t, c)
 
@@ -105,7 +100,7 @@ func TestAllInOne(t *testing.T) {
 			queryAndAssertTrace(t, apiClient, info)
 
 			// search an in-memory trace
-			searchAndAssertTrace(t, apiClient, info)
+			util.SearchAndAssertTrace(t, apiClient, info)
 
 			// flush trace to backend
 			res, err := e2e.DoGet("http://" + tempo.Endpoint(3200) + "/flush")
@@ -130,7 +125,7 @@ func TestAllInOne(t *testing.T) {
 
 			// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
 			now := time.Now()
-			searchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
+			util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
 		})
 	}
 }
@@ -234,7 +229,7 @@ func TestMicroservicesWithKVStores(t *testing.T) {
 			require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
 
 			// Get port for the Jaeger gRPC receiver endpoint
-			c, err := newJaegerGRPCClient(tempoDistributor.Endpoint(14250))
+			c, err := util.NewJaegerGRPCClient(tempoDistributor.Endpoint(14250))
 			require.NoError(t, err)
 			require.NotNil(t, c)
 
@@ -261,7 +256,7 @@ func TestMicroservicesWithKVStores(t *testing.T) {
 			queryAndAssertTrace(t, apiClient, info)
 
 			// search an in-memory trace
-			searchAndAssertTrace(t, apiClient, info)
+			util.SearchAndAssertTrace(t, apiClient, info)
 
 			// flush trace to backend
 			res, err := e2e.DoGet("http://" + tempoIngester1.Endpoint(3200) + "/flush")
@@ -303,11 +298,11 @@ func TestMicroservicesWithKVStores(t *testing.T) {
 			queryAndAssertTrace(t, apiClient, info)
 
 			// search an in-memory trace
-			searchAndAssertTrace(t, apiClient, info)
+			util.SearchAndAssertTrace(t, apiClient, info)
 
 			// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
 			now := time.Now()
-			searchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
+			util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
 
 			// stop another ingester and confirm things fail
 			err = tempoIngester1.Kill()
@@ -370,15 +365,15 @@ func TestScalableSingleBinary(t *testing.T) {
 	require.NoError(t, tempo2.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
 	require.NoError(t, tempo3.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
 
-	c1, err := newJaegerGRPCClient(tempo1.Endpoint(14250))
+	c1, err := util.NewJaegerGRPCClient(tempo1.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c1)
 
-	c2, err := newJaegerGRPCClient(tempo2.Endpoint(14250))
+	c2, err := util.NewJaegerGRPCClient(tempo2.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c2)
 
-	c3, err := newJaegerGRPCClient(tempo3.Endpoint(14250))
+	c3, err := util.NewJaegerGRPCClient(tempo3.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c3)
 
@@ -468,81 +463,6 @@ func queryAndAssertTrace(t *testing.T, client *tempoUtil.Client, info *tempoUtil
 	require.True(t, equalTraces(resp, expected))
 }
 
-func searchAndAssertTrace(t *testing.T, client *tempoUtil.Client, info *tempoUtil.TraceInfo) {
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	attr := tempoUtil.RandomAttrFromTrace(expected)
-
-	// verify attribute is present in tags
-	tagsResp, err := client.SearchTags()
-	require.NoError(t, err)
-	require.Contains(t, tagsResp.TagNames, attr.Key)
-
-	// verify attribute value is present in tag values
-	tagValuesResp, err := client.SearchTagValues(attr.Key)
-	require.NoError(t, err)
-	require.Contains(t, tagValuesResp.TagValues, strings.ToLower(attr.GetValue().GetStringValue()))
-
-	// verify trace can be found using attribute
-	resp, err := client.Search(attr.GetKey() + "=" + attr.GetValue().GetStringValue())
-	require.NoError(t, err)
-
-	hasHex := func(hexId string, resp *tempopb.SearchResponse) bool {
-		for _, s := range resp.Traces {
-			equal, err := tempoUtil.EqualHexStringTraceIDs(s.TraceID, hexId)
-			require.NoError(t, err)
-			if equal {
-				return true
-			}
-		}
-
-		return false
-	}
-
-	require.True(t, hasHex(info.HexID(), resp))
-}
-
-// by passing a time range and using a query_ingesters_until/backend_after of 0 we can force the queriers
-// to look in the backend blocks
-func searchAndAssertTraceBackend(t *testing.T, client *tempoUtil.Client, info *tempoUtil.TraceInfo, start int64, end int64) {
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	attr := tempoUtil.RandomAttrFromTrace(expected)
-
-	// verify trace can be found using attribute and time range
-	resp, err := client.SearchWithRange(attr.GetKey()+"="+attr.GetValue().GetStringValue(), start, end)
-	require.NoError(t, err)
-
-	hasHex := func(hexId string, resp *tempopb.SearchResponse) bool {
-		for _, s := range resp.Traces {
-			equal, err := tempoUtil.EqualHexStringTraceIDs(s.TraceID, hexId)
-			require.NoError(t, err)
-			if equal {
-				return true
-			}
-		}
-
-		return false
-	}
-
-	require.True(t, hasHex(info.HexID(), resp))
-}
-
-func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) {
-	// new jaeger grpc exporter
-	conn, err := grpc.Dial(endpoint, grpc.WithInsecure())
-	if err != nil {
-		return nil, err
-	}
-	logger, err := zap.NewDevelopment()
-	if err != nil {
-		return nil, err
-	}
-	return jaeger_grpc.NewReporter(conn, nil, logger), err
-}
-
 func equalTraces(a, b *tempopb.Trace) bool {
 	trace.SortTrace(a)
 	trace.SortTrace(b)

integration/e2e/limits_test.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"testing"
 
-	util "github.com/grafana/tempo/integration"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/e2e"
-	"github.com/stretchr/testify/require"
+	util "github.com/grafana/tempo/integration"
 )
 
 const (
@@ -25,7 +25,7 @@ func TestLimits(t *testing.T) {
 	require.NoError(t, s.StartAndWaitReady(tempo))
 
 	// Get port for the otlp receiver endpoint
-	c, err := newJaegerGRPCClient(tempo.Endpoint(14250))
+	c, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c)
 

integration/e2e/receivers_test.go

@@ -5,7 +5,6 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/grafana/e2e"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/jaegerexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter"
 	"github.com/stretchr/testify/assert"
@@ -23,6 +22,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 
+	"github.com/grafana/e2e"
 	util "github.com/grafana/tempo/integration"
 	tempoUtil "github.com/grafana/tempo/pkg/util"
 	"github.com/grafana/tempo/pkg/util/test"

integration/e2e/serverless_test.go → integration/e2e/serverless/serverless_test.go

@@ -1,18 +1,22 @@
-package e2e
+package serverless
 
 import (
 	"testing"
 	"time"
 
-	"github.com/grafana/e2e"
-	e2e_db "github.com/grafana/e2e/db"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/e2e"
+	e2e_db "github.com/grafana/e2e/db"
 	util "github.com/grafana/tempo/integration"
 	tempoUtil "github.com/grafana/tempo/pkg/util"
 )
 
+const (
+	configServerless = "config-serverless.yaml"
+)
+
 func TestServerless(t *testing.T) {
 	s, err := e2e.NewScenario("tempo_e2e")
 	require.NoError(t, err)
@@ -49,7 +53,7 @@ func TestServerless(t *testing.T) {
 	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`cortex_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
 
 	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := newJaegerGRPCClient(tempoDistributor.Endpoint(14250))
+	c, err := util.NewJaegerGRPCClient(tempoDistributor.Endpoint(14250))
 	require.NoError(t, err)
 	require.NotNil(t, c)
 
@@ -81,7 +85,7 @@ func TestServerless(t *testing.T) {
 
 	// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
 	now := time.Now()
-	searchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
+	util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
 }
 
 func newTempoServerless() *e2e.HTTPService {