From 58fc208c1e1305c7a9c8ff0d0d3c9d6c9ec2ed97 Mon Sep 17 00:00:00 2001 From: Laurent Dechoux <34473609+iotafr@users.noreply.github.com> Date: Tue, 29 Dec 2020 09:02:17 +0100 Subject: [PATCH] Migrate to go1.15 and confluent-kafka-go v1.5.2 (#10) * update confluent-kafka-go dependency to v1.5.2 * fix build * fix tests * add logs * Fix worflow * Fix integration test (#9) * use environnement variable to define boostrap server --- .github/workflows/go.yml | 6 +++--- Makefile | 8 ++++---- docker-compose.yml | 2 ++ examples/custom_collector/main.go | 2 +- examples/custom_collector/projector.go | 2 +- examples/custom_collector/prometheus.go | 2 +- examples/custom_projector/main.go | 2 +- examples/custom_projector/redis_projector.go | 2 +- examples/custom_transformer/header_transformer.go | 2 +- examples/custom_transformer/main.go | 2 +- examples/go.mod | 4 ++-- examples/go.sum | 6 ++---- go.mod | 4 ++-- go.sum | 4 ++-- internal/instrument/noop.go | 2 +- internal/test/collector.go | 2 +- internal/test/collector_test.go | 2 +- internal/test/helper.go | 4 ++-- internal/test/projector.go | 2 +- internal/test/transformer.go | 2 +- internal/test/transformer_test.go | 2 +- internal/transformer/kafka/consumer.go | 2 +- internal/transformer/kafka/producer.go | 2 +- internal/transformer/projector.go | 2 +- internal/transformer/transformer.go | 2 +- internal/transformer/workers.go | 2 +- pkg/instrument/collector.go | 2 +- pkg/transformer/kafka/kafka.go | 2 +- pkg/transformer/passthrough.go | 2 +- pkg/transformer/projector.go | 2 +- pkg/transformer/transformer.go | 2 +- scripts/check_govet.sh | 2 +- scripts/run_tests.sh | 4 ++-- 33 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2196773..39e367b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,7 @@ on: pull_request: branches: - master - - developr + - develop jobs: @@ -17,10 +17,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.14 + - name: Set up Go 1.15 uses: actions/setup-go@v1 with: - go-version: 1.14 + go-version: 1.15 id: go - name: Check out code into the Go module directory diff --git a/Makefile b/Makefile index 24f8c46..7d4fa70 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -GOLANG_VERSION?=1.14 +GOLANG_VERSION?=1.15 .PHONY: dev.up dev.up: @@ -10,8 +10,8 @@ dev.down: .PHONY: build build: - go build -v ./... - cd examples && go build -v ./... + go build -tags musl -v ./... + cd examples && go build -v -tags musl ./... .PHONY: verify verify: build @@ -21,7 +21,7 @@ verify: build .PHONY: tests tests: verify - go test -count=1 -v -tags integration ./... + go test -count=1 -v -tags integration -tags musl ./... .PHONY: tests.docker tests.docker: diff --git a/docker-compose.yml b/docker-compose.yml index 1abd2cd..eb50a02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: context: . dockerfile: Dockerfile.test command: ["bash", "./scripts/run_tests.sh"] + environment: + - KAFKA_BOOTSTRAP_SERVER=kafka:29092 depends_on: - kafka diff --git a/examples/custom_collector/main.go b/examples/custom_collector/main.go index 9e94bc2..d29ab2d 100644 --- a/examples/custom_collector/main.go +++ b/examples/custom_collector/main.go @@ -7,8 +7,8 @@ import ( "os/signal" "syscall" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer/kafka" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" "github.com/prometheus/client_golang/prometheus/promhttp" ) diff --git a/examples/custom_collector/projector.go b/examples/custom_collector/projector.go index c0d7a9b..a5f4c5d 100644 --- a/examples/custom_collector/projector.go +++ b/examples/custom_collector/projector.go @@ -3,7 +3,7 @@ package main import ( "sync/atomic" - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + kafka "github.com/confluentinc/confluent-kafka-go/kafka" ) type customProjector struct { diff --git a/examples/custom_collector/prometheus.go b/examples/custom_collector/prometheus.go index 7424dc5..cd3207c 100644 --- a/examples/custom_collector/prometheus.go +++ b/examples/custom_collector/prometheus.go @@ -4,9 +4,9 @@ import ( "log" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/prometheus/client_golang/prometheus" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) type promCollector struct { diff --git a/examples/custom_projector/main.go b/examples/custom_projector/main.go index 3ce4652..04fb98c 100644 --- a/examples/custom_projector/main.go +++ b/examples/custom_projector/main.go @@ -6,8 +6,8 @@ import ( "os/signal" "syscall" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer/kafka" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) func main() { diff --git a/examples/custom_projector/redis_projector.go b/examples/custom_projector/redis_projector.go index 52020d0..87e3b4f 100644 --- a/examples/custom_projector/redis_projector.go +++ b/examples/custom_projector/redis_projector.go @@ -5,8 +5,8 @@ import ( "strconv" "time" + kafka "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/go-redis/redis/v7" - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // RedisProjector will project message to redis by implementing transformer.Projector interface diff --git a/examples/custom_transformer/header_transformer.go b/examples/custom_transformer/header_transformer.go index 2d52f81..77405aa 100644 --- a/examples/custom_transformer/header_transformer.go +++ b/examples/custom_transformer/header_transformer.go @@ -3,8 +3,8 @@ package main import ( "time" + kafka "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/logger" - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) type headerTransformer struct { diff --git a/examples/custom_transformer/main.go b/examples/custom_transformer/main.go index 1ceff11..dc675b3 100644 --- a/examples/custom_transformer/main.go +++ b/examples/custom_transformer/main.go @@ -6,9 +6,9 @@ import ( "os/signal" "syscall" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer/kafka" "github.com/sirupsen/logrus" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) func main() { diff --git a/examples/go.mod b/examples/go.mod index c46bd68..55c34eb 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,13 +1,13 @@ module github.com/etf1/kafka-transformer/examples -go 1.14 +go 1.15 require ( + github.com/confluentinc/confluent-kafka-go v1.5.2 github.com/etf1/kafka-transformer v0.0.0-20200327090708-353621d904e9 github.com/go-redis/redis/v7 v7.2.0 github.com/prometheus/client_golang v1.5.1 github.com/sirupsen/logrus v1.4.2 - gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 ) replace github.com/etf1/kafka-transformer => ../ diff --git a/examples/go.sum b/examples/go.sum index 0b19125..e95d651 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -8,6 +8,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E= +github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -111,10 +113,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.0 h1:70Hht0HKadDe6GpSgstEtYrDMtHo3ZqK+3KeHepusaw= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/go.mod b/go.mod index e3a7442..460313b 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/etf1/kafka-transformer -go 1.14 +go 1.15 -require gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 +require github.com/confluentinc/confluent-kafka-go v1.5.2 diff --git a/go.sum b/go.sum index 21841b7..9a68628 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,2 @@ -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= +github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E= +github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= diff --git a/internal/instrument/noop.go b/internal/instrument/noop.go index 08f203d..03c9bd3 100644 --- a/internal/instrument/noop.go +++ b/internal/instrument/noop.go @@ -4,8 +4,8 @@ import ( "log" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/instrument" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) type noop struct { diff --git a/internal/test/collector.go b/internal/test/collector.go index 6b687b6..8dcd6fa 100644 --- a/internal/test/collector.go +++ b/internal/test/collector.go @@ -5,8 +5,8 @@ import ( "testing" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/instrument" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // SliceCollector is a slice implementation of the Collector interface diff --git a/internal/test/collector_test.go b/internal/test/collector_test.go index bf38c4a..03ad4cd 100644 --- a/internal/test/collector_test.go +++ b/internal/test/collector_test.go @@ -5,8 +5,8 @@ package test import ( "testing" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer/kafka" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Default case with a simple collector diff --git a/internal/test/helper.go b/internal/test/helper.go index 520cbea..95f4363 100644 --- a/internal/test/helper.go +++ b/internal/test/helper.go @@ -11,7 +11,7 @@ import ( "testing" "time" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" ) type dummyTransformer struct{} @@ -72,7 +72,7 @@ func messages(topic string, count int) []*confluent.Message { messages := make([]*confluent.Message, 0) for i := 1; i <= count; i++ { - messages = append(messages, message(topic, "message"+string(i))) + messages = append(messages, message(topic, fmt.Sprintf("message%v", i))) } return messages diff --git a/internal/test/projector.go b/internal/test/projector.go index 5618797..9837a3a 100644 --- a/internal/test/projector.go +++ b/internal/test/projector.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + "github.com/confluentinc/confluent-kafka-go/kafka" ) // SliceProjector will simply write the message in a slice diff --git a/internal/test/transformer.go b/internal/test/transformer.go index a3cf84b..4ebd3aa 100644 --- a/internal/test/transformer.go +++ b/internal/test/transformer.go @@ -1,8 +1,8 @@ package test import ( + "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer" - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) type unstableTransformer struct { diff --git a/internal/test/transformer_test.go b/internal/test/transformer_test.go index 1639ad7..43c8f0e 100644 --- a/internal/test/transformer_test.go +++ b/internal/test/transformer_test.go @@ -5,8 +5,8 @@ package test import ( "testing" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/transformer/kafka" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Default case with a simple transformer diff --git a/internal/transformer/kafka/consumer.go b/internal/transformer/kafka/consumer.go index 752d149..e35c776 100644 --- a/internal/transformer/kafka/consumer.go +++ b/internal/transformer/kafka/consumer.go @@ -5,10 +5,10 @@ import ( "sync" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" _instrument "github.com/etf1/kafka-transformer/internal/instrument" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Consumer represents the kafka consumer which will consume messages from a topic diff --git a/internal/transformer/kafka/producer.go b/internal/transformer/kafka/producer.go index 94b72c3..52d3458 100644 --- a/internal/transformer/kafka/producer.go +++ b/internal/transformer/kafka/producer.go @@ -4,10 +4,10 @@ import ( "log" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" _instrument "github.com/etf1/kafka-transformer/internal/instrument" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Producer represents the kafka producer which will produce diff --git a/internal/transformer/projector.go b/internal/transformer/projector.go index 640fbb8..b4858a6 100644 --- a/internal/transformer/projector.go +++ b/internal/transformer/projector.go @@ -6,11 +6,11 @@ import ( "sync" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" _instrument "github.com/etf1/kafka-transformer/internal/instrument" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" pkg "github.com/etf1/kafka-transformer/pkg/transformer" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Projector will project kafka message to kafka topic or somewhere else diff --git a/internal/transformer/transformer.go b/internal/transformer/transformer.go index 786d3e5..d411e50 100644 --- a/internal/transformer/transformer.go +++ b/internal/transformer/transformer.go @@ -6,10 +6,10 @@ import ( "sync" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" "github.com/etf1/kafka-transformer/pkg/transformer" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Transformer represents the transformer which will perform the custom transformation diff --git a/internal/transformer/workers.go b/internal/transformer/workers.go index 63178b6..a5c96a9 100644 --- a/internal/transformer/workers.go +++ b/internal/transformer/workers.go @@ -6,11 +6,11 @@ import ( "sync" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" _instrument "github.com/etf1/kafka-transformer/internal/instrument" "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" pkg "github.com/etf1/kafka-transformer/pkg/transformer" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Workers is a pool of goroutines used for parallel transformation diff --git a/pkg/instrument/collector.go b/pkg/instrument/collector.go index 9377628..a57918f 100644 --- a/pkg/instrument/collector.go +++ b/pkg/instrument/collector.go @@ -3,7 +3,7 @@ package instrument import ( "time" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" ) // Collector allows to specify a collector for all the main actions of the kafka transformer. diff --git a/pkg/transformer/kafka/kafka.go b/pkg/transformer/kafka/kafka.go index 9815a65..78624e4 100644 --- a/pkg/transformer/kafka/kafka.go +++ b/pkg/transformer/kafka/kafka.go @@ -6,6 +6,7 @@ import ( "sync" "time" + confluent "github.com/confluentinc/confluent-kafka-go/kafka" _instrument "github.com/etf1/kafka-transformer/internal/instrument" _logger "github.com/etf1/kafka-transformer/internal/logger" internal "github.com/etf1/kafka-transformer/internal/transformer" @@ -13,7 +14,6 @@ import ( "github.com/etf1/kafka-transformer/pkg/instrument" "github.com/etf1/kafka-transformer/pkg/logger" pkg "github.com/etf1/kafka-transformer/pkg/transformer" - confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) // Config is the configuration used by KafkaTransformer diff --git a/pkg/transformer/passthrough.go b/pkg/transformer/passthrough.go index 4cbb2f6..ec5a0d0 100644 --- a/pkg/transformer/passthrough.go +++ b/pkg/transformer/passthrough.go @@ -1,7 +1,7 @@ package transformer import ( - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + "github.com/confluentinc/confluent-kafka-go/kafka" ) type passThrough struct{} diff --git a/pkg/transformer/projector.go b/pkg/transformer/projector.go index 0c08522..4735c54 100644 --- a/pkg/transformer/projector.go +++ b/pkg/transformer/projector.go @@ -1,7 +1,7 @@ package transformer import ( - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + "github.com/confluentinc/confluent-kafka-go/kafka" ) // Projector is an interface which is used by Kafka.Transformer diff --git a/pkg/transformer/transformer.go b/pkg/transformer/transformer.go index 950f50d..4cee5f3 100644 --- a/pkg/transformer/transformer.go +++ b/pkg/transformer/transformer.go @@ -1,7 +1,7 @@ package transformer import ( - "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" + "github.com/confluentinc/confluent-kafka-go/kafka" ) // Transformer is an interface which is used by Kafka.Transformer diff --git a/scripts/check_govet.sh b/scripts/check_govet.sh index 1f9728a..5248082 100755 --- a/scripts/check_govet.sh +++ b/scripts/check_govet.sh @@ -2,4 +2,4 @@ set -euo pipefail -go vet ./... +go vet -tags musl ./... diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0078700..e53ac7a 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -7,12 +7,12 @@ function wait_for_kafka(){ for i in {1..10} do - echo "Waiting for kafka cluster to be ready ..." + echo "Waiting for kafka cluster $1 to be ready ..." # kafkacat has 5s timeout kafkacat -b "${server}" -L > /dev/null 2>&1 && break done } -wait_for_kafka "kafka:29092" +wait_for_kafka $KAFKA_BOOTSTRAP_SERVER make tests \ No newline at end of file