Integration tests proposal (#5475)

* add integration tests

* add readme

* use otlp mimir endpoint

* remove docker cleanup to test cache

* cleanup docker images when using docker compose down

* try running integration tests only on PRs

.github/workflows/integration-tests.yml

@@ -0,0 +1,17 @@
+name: Integration Tests
+on:
+  pull_request:
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.0"
+    - name: Set OTEL Exporter Endpoint
+      run: echo "OTEL_EXPORTER_ENDPOINT=http://172.17.0.1:8080" >> $GITHUB_ENV
+    - name: Run tests
+      run: make integration-test

Makefile

@@ -15,8 +15,9 @@
 ##
 ## Targets for running tests:
 ##
-##   test  Run tests
-##   lint  Lint code
+##   test              Run tests
+##   lint              Lint code
+##   integration-tests Run integration tests
 ##
 ## Targets for building binaries:
 ##
@@ -172,13 +173,17 @@ lint: agentlint
 # We have to run test twice: once for all packages with -race and then once
 # more without -race for packages that have known race detection issues.
 test:
-	$(GO_ENV) go test $(GO_FLAGS) -race ./...
+	$(GO_ENV) go test $(GO_FLAGS) -race $(shell go list ./... | grep -v /integration-tests/)
 	$(GO_ENV) go test $(GO_FLAGS) ./pkg/integrations/node_exporter ./pkg/logs ./pkg/operator ./pkg/util/k8s ./component/otelcol/processor/tail_sampling ./component/loki/source/file
 
 test-packages:
 	docker pull $(BUILD_IMAGE)
 	go test -tags=packaging  ./packaging
 
+.PHONY: integration-tests
+integration-test:
+	cd integration-tests && $(GO_ENV) go run .
+
 #
 # Targets for building binaries
 #

integration-tests/README.md

@@ -0,0 +1,29 @@
+# Integration tests
+
+This document provides an outline of how to run and add new integration tests to the project.
+
+The purpose of these tests is to verify simple, happy-path pipelines to catch issues between the agent and external dependencies.
+
+The external dependencies are launched as Docker containers.
+
+## Running tests
+
+Execute the integration tests using the following command:
+
+`go run .`
+
+### Flags
+
+* `--skip-build`: Run the integration tests without building the agent (default: `false`)
+* `--test`: Specifies a particular directory within the tests directory to run (default: runs all tests)
+
+## Adding new tests
+
+Follow these steps to add a new integration test to the project:
+
+1. If the test requires external resources, define them as Docker images within the `docker-compose.yaml` file.
+2. Create a new directory under the tests directory to house the files for the new test.
+3. Within the new test directory, create a file named `config.river` to hold the pipeline configuration you want to test.
+4. Create a `_test.go` file within the new test directory. This file should contain the Go code necessary to run the test and verify the data processing through the pipeline.
+
+ _NOTE_: The tests run concurrently. Each agent must tag its data with a label that corresponds to its specific configuration. This ensures the correct data verification during the Go testing process.

integration-tests/common/common.go

@@ -0,0 +1,34 @@
+package common
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"time"
+)
+
+type Unmarshaler interface {
+	Unmarshal([]byte) error
+}
+
+const DefaultRetryInterval = 100 * time.Millisecond
+const DefaultTimeout = time.Minute
+
+func FetchDataFromURL(url string, target Unmarshaler) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.New("Non-OK HTTP status: " + resp.Status)
+	}
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return target.Unmarshal(bodyBytes)
+}

integration-tests/common/log.go

@@ -0,0 +1,20 @@
+package common
+
+import "encoding/json"
+
+type LogResponse struct {
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string    `json:"resultType"`
+		Result     []LogData `json:"result"`
+	} `json:"data"`
+}
+
+type LogData struct {
+	Stream map[string]string `json:"stream"`
+	Values [][2]string       `json:"values"`
+}
+
+func (m *LogResponse) Unmarshal(data []byte) error {
+	return json.Unmarshal(data, m)
+}

integration-tests/common/metric.go

@@ -0,0 +1,49 @@
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type MetricResponse struct {
+	Status string     `json:"status"`
+	Data   MetricData `json:"data"`
+}
+
+type MetricData struct {
+	ResultType string         `json:"resultType"`
+	Result     []MetricResult `json:"result"`
+}
+
+// TODO: check for the type
+type MetricResult struct {
+	Metric Metric `json:"metric"`
+	Value  Value  `json:"value"`
+}
+
+type Value struct {
+	Timestamp int64
+	Value     string
+}
+
+func (v *Value) UnmarshalJSON(b []byte) error {
+	var arr []interface{}
+	if err := json.Unmarshal(b, &arr); err != nil {
+		return err
+	}
+	if len(arr) != 2 {
+		return fmt.Errorf("expected 2 values, got %d", len(arr))
+	}
+	v.Timestamp, _ = arr[0].(int64)
+	v.Value, _ = arr[1].(string)
+	return nil
+}
+
+type Metric struct {
+	TestName string `json:"test_name"`
+	Name     string `json:"__name__"`
+}
+
+func (m *MetricResponse) Unmarshal(data []byte) error {
+	return json.Unmarshal(data, m)
+}

integration-tests/configs/mimir/mimir.yaml

@@ -0,0 +1,63 @@
+# Do not use this configuration in production.
+# It is for demonstration purposes only.
+multitenancy_enabled: false
+
+activity_tracker: {}
+
+alertmanager: {}
+
+alertmanager_storage:
+  backend: local
+
+server:
+  http_listen_port: 9009
+
+  # Configure the server to allow messages up to 100MB.
+  grpc_server_max_recv_msg_size: 104857600
+  grpc_server_max_send_msg_size: 104857600
+  grpc_server_max_concurrent_streams: 1000
+
+distributor:
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    grpc_compression: gzip
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+
+ingester:
+  ring:
+    final_sleep: 0s
+    kvstore:
+      store: inmemory
+    min_ready_duration: 0s
+    num_tokens: 512
+    replication_factor: 1
+
+blocks_storage:
+  backend: filesystem
+  bucket_store:
+    sync_dir: /tmp/mimir/tsdb-sync
+  filesystem:
+    dir: /tmp/mimir/blocks
+  tsdb:
+    dir: /tmp/mimir/tsdb
+
+compactor:
+  sharding_ring:
+    kvstore:
+      store: inmemory
+
+ruler:
+  enable_api: true
+
+ruler_storage:
+  backend: filesystem
+  local:
+    directory: /tmp/mimir/rules
+
+limits:
+  ingestion_burst_size: 500000
+  ingestion_rate: 250000

integration-tests/configs/otel-collector-contrib/otel-collector-contrib.yaml

@@ -0,0 +1,27 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+exporters:
+  logging:
+
+  otlphttp:
+    endpoint: ${OTEL_EXPORTER_ENDPOINT}
+
+
+connectors:
+  spanmetrics:
+    namespace: span.metrics
+    exemplars:
+      enabled: true
+    metrics_flush_interval: 1s
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [spanmetrics]
+    metrics:
+      receivers: [spanmetrics]
+      exporters: [otlphttp]

integration-tests/configs/otel-gen-client/Dockerfile

@@ -0,0 +1,11 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+FROM golang:1.21 as build
+WORKDIR /app/
+COPY go.mod go.sum ./
+RUN go mod download
+COPY ./integration-tests/configs/otel-gen-client/ ./
+RUN CGO_ENABLED=0 go build -o main main.go
+FROM alpine:3.18
+COPY --from=build /app/main /app/main
+CMD ["/app/main"]