Migrate to go1.15 and confluent-kafka-go v1.5.2 (#10)

* update confluent-kafka-go dependency to v1.5.2 * fix build * fix tests * add logs * Fix worflow * Fix integration test (#9) * use environnement variable to define boostrap server
etf1 · Dec 29, 2020 · 58fc208 · 58fc208
1 parent 882c97a
commit 58fc208
Show file tree

Hide file tree

Showing 33 changed files with 45 additions and 45 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - master
-      - developr
+      - develop
 
 jobs:
 
@@ -17,10 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.14
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-GOLANG_VERSION?=1.14
+GOLANG_VERSION?=1.15
 
 .PHONY: dev.up
 dev.up:
@@ -10,8 +10,8 @@ dev.down:
 
 .PHONY: build
 build:
-	go build -v ./...
-	cd examples && go build -v ./...
+	go build -tags musl -v ./...
+	cd examples && go build -v -tags musl ./...
 
 .PHONY: verify
 verify: build
@@ -21,7 +21,7 @@ verify: build
 
 .PHONY: tests
 tests: verify
-	go test -count=1 -v -tags integration ./...
+	go test -count=1 -v -tags integration -tags musl ./...
 
 .PHONY: tests.docker
 tests.docker:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,6 +5,8 @@ services:
       context: .
       dockerfile: Dockerfile.test
     command: ["bash", "./scripts/run_tests.sh"]
+    environment: 
+      - KAFKA_BOOTSTRAP_SERVER=kafka:29092
     depends_on:
       - kafka
 

diff --git a/examples/custom_collector/main.go b/examples/custom_collector/main.go
@@ -7,8 +7,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer/kafka"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )

diff --git a/examples/custom_collector/projector.go b/examples/custom_collector/projector.go
@@ -3,7 +3,7 @@ package main
 import (
 	"sync/atomic"
 
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	kafka "github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 type customProjector struct {

diff --git a/examples/custom_collector/prometheus.go b/examples/custom_collector/prometheus.go
@@ -4,9 +4,9 @@ import (
 	"log"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/prometheus/client_golang/prometheus"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 type promCollector struct {

diff --git a/examples/custom_projector/main.go b/examples/custom_projector/main.go
@@ -6,8 +6,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer/kafka"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 func main() {

diff --git a/examples/custom_projector/redis_projector.go b/examples/custom_projector/redis_projector.go
@@ -5,8 +5,8 @@ import (
 	"strconv"
 	"time"
 
+	kafka "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/go-redis/redis/v7"
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // RedisProjector will project message to redis by implementing transformer.Projector interface

diff --git a/examples/custom_transformer/header_transformer.go b/examples/custom_transformer/header_transformer.go
@@ -3,8 +3,8 @@ package main
 import (
 	"time"
 
+	kafka "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/logger"
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 type headerTransformer struct {

diff --git a/examples/custom_transformer/main.go b/examples/custom_transformer/main.go
@@ -6,9 +6,9 @@ import (
 	"os/signal"
 	"syscall"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer/kafka"
 	"github.com/sirupsen/logrus"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 func main() {

diff --git a/examples/go.mod b/examples/go.mod
@@ -1,13 +1,13 @@
 module github.com/etf1/kafka-transformer/examples
 
-go 1.14
+go 1.15
 
 require (
+	github.com/confluentinc/confluent-kafka-go v1.5.2
 	github.com/etf1/kafka-transformer v0.0.0-20200327090708-353621d904e9
 	github.com/go-redis/redis/v7 v7.2.0
 	github.com/prometheus/client_golang v1.5.1
 	github.com/sirupsen/logrus v1.4.2
-	gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0
 )
 
 replace github.com/etf1/kafka-transformer => ../
diff --git a/examples/go.sum b/examples/go.sum
@@ -8,6 +8,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E=
+github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -111,10 +113,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI=
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY=
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.0 h1:70Hht0HKadDe6GpSgstEtYrDMtHo3ZqK+3KeHepusaw=
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=

diff --git a/go.mod b/go.mod
@@ -1,5 +1,5 @@
 module github.com/etf1/kafka-transformer
 
-go 1.14
+go 1.15
 
-require gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0
+require github.com/confluentinc/confluent-kafka-go v1.5.2
diff --git a/go.sum b/go.sum
@@ -1,2 +1,2 @@
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI=
-gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY=
+github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E=
+github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
diff --git a/internal/instrument/noop.go b/internal/instrument/noop.go
@@ -4,8 +4,8 @@ import (
 	"log"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 type noop struct {

diff --git a/internal/test/collector.go b/internal/test/collector.go
@@ -5,8 +5,8 @@ import (
 	"testing"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // SliceCollector is a slice implementation of the Collector interface

diff --git a/internal/test/collector_test.go b/internal/test/collector_test.go
@@ -5,8 +5,8 @@ package test
 import (
 	"testing"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer/kafka"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Default case with a simple collector

diff --git a/internal/test/helper.go b/internal/test/helper.go
@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 type dummyTransformer struct{}
@@ -72,7 +72,7 @@ func messages(topic string, count int) []*confluent.Message {
 	messages := make([]*confluent.Message, 0)
 
 	for i := 1; i <= count; i++ {
-		messages = append(messages, message(topic, "message"+string(i)))
+		messages = append(messages, message(topic, fmt.Sprintf("message%v", i)))
 	}
 
 	return messages

diff --git a/internal/test/projector.go b/internal/test/projector.go
@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	"github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 // SliceProjector will simply write the message in a slice

diff --git a/internal/test/transformer.go b/internal/test/transformer.go
@@ -1,8 +1,8 @@
 package test
 
 import (
+	"github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer"
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 type unstableTransformer struct {

diff --git a/internal/test/transformer_test.go b/internal/test/transformer_test.go
@@ -5,8 +5,8 @@ package test
 import (
 	"testing"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/transformer/kafka"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Default case with a simple transformer

diff --git a/internal/transformer/kafka/consumer.go b/internal/transformer/kafka/consumer.go
@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	_instrument "github.com/etf1/kafka-transformer/internal/instrument"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Consumer represents the kafka consumer which will consume messages from a topic

diff --git a/internal/transformer/kafka/producer.go b/internal/transformer/kafka/producer.go
@@ -4,10 +4,10 @@ import (
 	"log"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	_instrument "github.com/etf1/kafka-transformer/internal/instrument"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Producer represents the kafka producer which will produce

diff --git a/internal/transformer/projector.go b/internal/transformer/projector.go
@@ -6,11 +6,11 @@ import (
 	"sync"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	_instrument "github.com/etf1/kafka-transformer/internal/instrument"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
 	pkg "github.com/etf1/kafka-transformer/pkg/transformer"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Projector will project kafka message to kafka topic or somewhere else

diff --git a/internal/transformer/transformer.go b/internal/transformer/transformer.go
@@ -6,10 +6,10 @@ import (
 	"sync"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
 	"github.com/etf1/kafka-transformer/pkg/transformer"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Transformer represents the transformer which will perform the custom transformation

diff --git a/internal/transformer/workers.go b/internal/transformer/workers.go
@@ -6,11 +6,11 @@ import (
 	"sync"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	_instrument "github.com/etf1/kafka-transformer/internal/instrument"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
 	pkg "github.com/etf1/kafka-transformer/pkg/transformer"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Workers is a pool of goroutines used for parallel transformation

diff --git a/pkg/instrument/collector.go b/pkg/instrument/collector.go
@@ -3,7 +3,7 @@ package instrument
 import (
 	"time"
 
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 // Collector allows to specify a collector for all the main actions of the kafka transformer.

diff --git a/pkg/transformer/kafka/kafka.go b/pkg/transformer/kafka/kafka.go
@@ -6,14 +6,14 @@ import (
 	"sync"
 	"time"
 
+	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
 	_instrument "github.com/etf1/kafka-transformer/internal/instrument"
 	_logger "github.com/etf1/kafka-transformer/internal/logger"
 	internal "github.com/etf1/kafka-transformer/internal/transformer"
 	"github.com/etf1/kafka-transformer/internal/transformer/kafka"
 	"github.com/etf1/kafka-transformer/pkg/instrument"
 	"github.com/etf1/kafka-transformer/pkg/logger"
 	pkg "github.com/etf1/kafka-transformer/pkg/transformer"
-	confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
 )
 
 // Config is the configuration used by KafkaTransformer

diff --git a/pkg/transformer/passthrough.go b/pkg/transformer/passthrough.go
@@ -1,7 +1,7 @@
 package transformer
 
 import (
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	"github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 type passThrough struct{}

diff --git a/pkg/transformer/projector.go b/pkg/transformer/projector.go
@@ -1,7 +1,7 @@
 package transformer
 
 import (
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	"github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 // Projector is an interface which is used by Kafka.Transformer

diff --git a/pkg/transformer/transformer.go b/pkg/transformer/transformer.go
@@ -1,7 +1,7 @@
 package transformer
 
 import (
-	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+	"github.com/confluentinc/confluent-kafka-go/kafka"
 )
 
 // Transformer is an interface which is used by Kafka.Transformer

diff --git a/scripts/check_govet.sh b/scripts/check_govet.sh
@@ -2,4 +2,4 @@
 
 set -euo pipefail
 
-go vet ./...
+go vet -tags musl ./...
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
@@ -7,12 +7,12 @@ function wait_for_kafka(){
 
     for i in {1..10}
     do
-        echo "Waiting for kafka cluster to be ready ..."
+        echo "Waiting for kafka cluster $1 to be ready ..."
         # kafkacat has 5s timeout
         kafkacat -b "${server}" -L > /dev/null 2>&1 && break
     done
 }
 
-wait_for_kafka "kafka:29092"
+wait_for_kafka $KAFKA_BOOTSTRAP_SERVER
 
 make tests
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@

		set -euo pipefail

		go vet ./...
		go vet -tags musl ./...