[pull] master from housepower:master

.github/workflows/gh-pages.yml

@@ -0,0 +1,19 @@
+name: gh-pages
+
+on:
+  push:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "15.x"
+      - run: "[[ -z $(git show HEAD -- docs) ]] || npm install"
+      - run: "[[ -z $(git show HEAD -- docs) ]] || npm run docs:build"
+      - run: "[[ -z $(git show HEAD -- docs) ]] || dev/deploy_gh_pages.sh"
+        env:
+          TOKEN_GITHUB: ${{ secrets.TOKEN_GITHUB }}

.github/workflows/lint.yml

@@ -1,28 +1,45 @@
-name: golangci-lint
+# https://github.com/golangci/golangci-lint-action
+name: lint
 on:
   push:
-    tags:
-      - v*
+    tags: [ v* ]
     branches:
       - master
+      - main
   pull_request:
+    branches:
+      - master
+      - main
 jobs:
-  golangci:
-    name: lint
+  golangci-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.32.2
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: v1.51.2
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir
 
           # Optional: golangci-lint command line arguments.
-          args: --issues-exit-code=0 --disable=nakedret,exhaustivestruct,wrapcheck
+          # args: --issues-exit-code=0
+          args: --timeout=3m
 
           # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
+          # only-new-issues: true
+
+          # Optional: if set to true then the all caching functionality will be complete disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true

.github/workflows/tests.yml

@@ -14,9 +14,9 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
-        go-version: ^1.13
+        go-version: '1.21'
       id: go
 
     - name: Check out code into the Go module directory
@@ -31,7 +31,7 @@ jobs:
       run: make build
 
     - name: unittest
-      run: make unittest
+      run: make gotest
 
     - name: benchtest
       run: make benchtest

.gitignore

@@ -1,10 +1,17 @@
 tmp
 sh
 .vscode
+node_modules/
+package-lock.json
+nacos_cache
 .DS_Store
 *.log
 .idea
 *.bin
-.goreleaser.yml
+*.out
+*.pem
+*.p12
 dist
+vendor
+bin/
 

.goreleaser.yml

@@ -12,26 +12,21 @@ builds:
       - CGO_ENABLED=0
     goos:
       - linux
+    goarch:
+      - amd64
+      - arm64
     main: cmd/clickhouse_sinker/main.go
     binary: clickhouse_sinker
   - id: nacos_publish_config
     env:
       - CGO_ENABLED=0
     goos:
       - linux
+    goarch:
+      - amd64
+      - arm64
     main: cmd/nacos_publish_config/main.go
     binary: nacos_publish_config
-dockers:
-  -
-    binaries:
-    - clickhouse_sinker
-    - nacos_publish_config
-    goos: linux
-    goarch: amd64
-    dockerfile: Dockerfile_goreleaser
-    image_templates:
-      - quay.io/housepower/clickhouse_sinker:latest
-      - quay.io/housepower/clickhouse_sinker:{{ .Tag }}
 archives:
   - replacements:
       darwin: Darwin

CHANGELOG.md

@@ -0,0 +1,145 @@
+# Changelog
+
+#### Version 3.0.8 (2023-08-xx)
+
+Improvements:
+
+- "too many simultaneous queries" from ClickHouse will no longer be a fatal error, instead sinker will keep retrying untill complete the write operation
+- Output the offset range per batch
+
+New Features:
+
+- Introduced a task level configuration property "field", which accepts a valid json string and allows additional
+fields to be appened to each input message.
+
+#### Version 3.0.7 (2023-07-19)
+
+Improvements:
+
+- when run sinker in cluster mode, caculate lag failure of certain task should not block the entire rebalance work. Instead, the current task should not be assigned to any running sinker instance.
+- sinker will not crash when handling incompatible map type.
+
+#### Version 3.0.6 (2023-05-19)
+
+Improvements:
+
+- When idenfying new columns, also update the schema of tables that are part of a logical cluster
+- Allow writing non-numeric type metric
+
+#### Version 3.0.5 (2023-05-12)
+
+New Features:
+
+- Introduced two new configuration properties to Sinker:
+  - "ReloadSeriesMapInterval", allows you to manage how frequently the cached seriesMap is reloaded.
+  - "ActiveSeriesRange", allows you to manage the amount of cached information.
+    Refer to the "config.md" file for more info.
+- Add a new http endpoint "/debug/vars" for viewing contents of seriesMap
+
+Deprecations:
+
+- DialTimeout option is deprecated.
+
+#### Version 3.0.4 (2023-04-18)
+
+Improvements:
+
+- Automatically end sinker if it remains inactive in last 10 mins
+
+#### Version 3.0.3 (2023-03-30)
+
+Improvements:
+
+- Have writingpool per shard to avoid ErrAcquireConnTimeout
+- Do not create kafka client everytime when caculate lags
+- Support configuring PlainloginModule in kafka.security section
+
+Bug Fixes:
+
+- Avoid program running into stuck when facing fatalpanic
+
+#### Version 3.0.2 (2023-03-13)
+
+Improvements:
+
+- update sinker grafana dashboard
+- combine nacos log into sinker log
+- update dmseries map when applying new config, reload the records from series table every single day
+- avoid recreating dist tables, alter the table schema instead
+- update clickhouse_sinker_consume_lags metric every 10 secs
+
+#### Version 3.0.1 (2023-03-03)
+
+Bug Fixes:
+
+- Fix nacos publish config error "BUG: got different config"
+- Fix changing "TimeUnit" config property does not trigger config reload
+- Fix illegal "TimeZone" value result in sinker crash
+- Fix wrong parsing result of Decimal type [909](https://github.com/ClickHouse/clickhouse-go/pull/909)
+
+Improvements:
+
+- Metrics from GoCollector and ProcessCollector are now being pushed to metric-push-gateway-addrs
+- Terminate program immediately when receiving one more exit signal
+- Limit the fetch size and poll size based on the BufferSize config property
+
+#### Version 3.0.0 (2023-02-07)
+
+New Features:
+
+- Add support of ingesting multi-value metrics, the metric table will be expanded accordingly
+- Allow specifying the series table name
+- Allow customization of DatabaseName in task level
+
+Improvements:
+
+- Group the tasks by consumerGroup property to reduce number of kafka client, see design.md for details
+
+Deprecation:
+
+- Kafka-go and Sarama are no longer internal options for sinker
+- 
+
+#### Version 2.6.9 (2023-02-07)
+
+Improvements:
+
+- Ignore SIGHUP signal, so that fire up sinker with nohup could work correctly
+- Stop retrying when facing offsets commit error, leave it to the future commitment to sync the offsets
+- Offsets commit error should not result in a process abort
+
+#### Version 2.6.8 (2022-12-10)
+
+New Features:
+
+- Add clickhouse Map type support
+- Small updates to allow TLS connections for AWS MSK, etc.
+  ([169](https://github.com/housepower/clickhouse_sinker/pull/169))
+
+Bug Fixes:
+
+- Fix ClickHouse.Init goroutine leak
+
+#### Version 2.6.7 (2022-12-07)
+
+Improvements:
+
+- Add new sinker metrics to show the wrSeriesQuota status
+- Always allow writing new series to avoid data mismatch between series and metrics table
+
+#### Version 2.6.6 (2022-12-05)
+
+Bug Fixes:
+
+- reset wrSeries timely to avoid failure of writing metric data to clickhouse
+
+#### Version 2.6.5 (2022-11-30)
+
+Bug Fixes:
+
+- Fix the 'segmentation violation' in ch-go package
+- Fix the create table error 'table already exists' when trying to create a distribution table
+
+#### Previous releases
+
+See https://github.com/housepower/clickhouse_sinker/releases