From 86ef429e72b38111a8982b64a388399247d971e3 Mon Sep 17 00:00:00 2001 From: Raja Kolli Date: Thu, 26 Dec 2024 06:56:12 +0000 Subject: [PATCH 1/4] feat : implement concurrent consumers using saperate thread --- .../kafka-boot-concurrent-kafka-consumer.yml | 35 +++ kafka-spring-boot/README.md | 3 +- .../boot-concurrent-kafka-consumer/.gitignore | 2 + .../.mvn/wrapper/maven-wrapper.properties | 19 ++ .../boot-concurrent-kafka-consumer/mvnw | 259 ++++++++++++++++++ .../boot-concurrent-kafka-consumer/mvnw.cmd | 149 ++++++++++ .../boot-concurrent-kafka-consumer/pom.xml | 91 ++++++ .../kafka/ConcurrentKafkaConsumer.java | 12 + .../example/kafka/listener/AppListener.java | 75 +++++ .../processor/ToUpperStringProcessor.java | 83 ++++++ .../src/main/resources/application.properties | 7 + .../kafka/TestConcurrentKafkaConsumer.java | 14 + .../example/kafka/common/ContainerConfig.java | 17 ++ .../example/kafka/common/ProducerConfig.java | 53 ++++ .../kafka/listener/AppListenerTest.java | 85 ++++++ 15 files changed, 903 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/kafka-boot-concurrent-kafka-consumer.yml create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/.gitignore create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/.mvn/wrapper/maven-wrapper.properties create mode 100755 kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw.cmd create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/ConcurrentKafkaConsumer.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/listener/AppListener.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/processor/ToUpperStringProcessor.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/resources/application.properties create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/TestConcurrentKafkaConsumer.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ContainerConfig.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java diff --git a/.github/workflows/kafka-boot-concurrent-kafka-consumer.yml b/.github/workflows/kafka-boot-concurrent-kafka-consumer.yml new file mode 100644 index 00000000..c238b602 --- /dev/null +++ b/.github/workflows/kafka-boot-concurrent-kafka-consumer.yml @@ -0,0 +1,35 @@ +name: boot-concurrent-kafka-consumer + +on: + push: + paths: + - "kafka-spring-boot/boot-concurrent-kafka-consumer/**" + branches: [ main ] + pull_request: + paths: + - "kafka-spring-boot/boot-concurrent-kafka-consumer/**" + types: + - opened + - synchronize + - reopened + +jobs: + build: + + runs-on: ubuntu-latest + defaults: + run: + working-directory: kafka-spring-boot/boot-concurrent-kafka-consumer + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up JDK + uses: actions/setup-java@v4.6.0 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + - name: Build with Maven + run: ./mvnw -B clean verify --file pom.xml diff --git a/kafka-spring-boot/README.md b/kafka-spring-boot/README.md index cb50d544..a2d61667 100644 --- a/kafka-spring-boot/README.md +++ b/kafka-spring-boot/README.md @@ -6,4 +6,5 @@ This folder contains sample kafka producer and consumer using java, spring & Spr | Type | Description | |------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------| | [Spring Boot Kafka Sample](./boot-kafka-sample) | Producing and consuming Messages using Spring Boot | -| [Multiple Producers & Consumers](./spring-boot-multiple-producers-consumers) | Producing and consuming Messages from Multiple Producer and consumers using Spring Boot | +| [Multiple Producers & Consumers](./boot-multiple-producers-consumers) | Producing and consuming Messages from Multiple Producer and consumers using Spring Boot | +| [Concurrent Consumer](./boot-concurrent-kafka-consumer) | Consuming Messages Concurrently using new Thread using Spring Boot | diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/.gitignore b/kafka-spring-boot/boot-concurrent-kafka-consumer/.gitignore new file mode 100644 index 00000000..c39c740f --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/.gitignore @@ -0,0 +1,2 @@ + +/target \ No newline at end of file diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/.mvn/wrapper/maven-wrapper.properties b/kafka-spring-boot/boot-concurrent-kafka-consumer/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..d58dfb70 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw b/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw new file mode 100755 index 00000000..19529ddf --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw.cmd b/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw.cmd new file mode 100644 index 00000000..b150b91e --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml b/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml new file mode 100644 index 00000000..7966d1e9 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.4.1 + + + com.example.kafka + boot-concurrent-kafka-consumer + 0.0.1-SNAPSHOT + boot-concurrent-kafka-consumer + Demo project for Spring Boot Concurrent Consumer + + + 21 + 2.43.0 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.kafka + spring-kafka + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.springframework.kafka + spring-kafka-test + test + + + org.testcontainers + junit-jupiter + test + + + org.testcontainers + kafka + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 2.50.0 + + + + + + + + + compile + + check + + + + + + + + diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/ConcurrentKafkaConsumer.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/ConcurrentKafkaConsumer.java new file mode 100644 index 00000000..54567268 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/ConcurrentKafkaConsumer.java @@ -0,0 +1,12 @@ +package com.example.kafka; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ConcurrentKafkaConsumer { + + public static void main(String[] args) { + SpringApplication.run(ConcurrentKafkaConsumer.class, args); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/listener/AppListener.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/listener/AppListener.java new file mode 100644 index 00000000..b510173f --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/listener/AppListener.java @@ -0,0 +1,75 @@ +package com.example.kafka.listener; + +import com.example.kafka.processor.ToUpperStringProcessor; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.event.EventListener; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.event.ConsumerStoppedEvent; +import org.springframework.stereotype.Component; + +/** + * Listener component for Kafka messages. + * This class is responsible for processing messages received from a Kafka topic. + * It utilizes a dedicated {@link ToUpperStringProcessor} for each thread to handle incoming messages. + */ +@Component +public class AppListener { + + private static final Logger log = LoggerFactory.getLogger(AppListener.class); + + public static final String SPRING_KAFKA_TEST_TOPIC = "test-topic"; + private static final ThreadLocal TO_UPPER_STRING_PROCESSOR_THREAD_LOCAL = + new ThreadLocal<>(); + + private final ObjectProvider toUpperStringProcessorObjectProvider; + + public AppListener(ObjectProvider toUpperStringProcessorObjectProvider) { + this.toUpperStringProcessorObjectProvider = toUpperStringProcessorObjectProvider; + } + + /** + * Listens for messages on the Kafka topic defined by {@code SPRING_KAFKA_TEST_TOPIC}. + * Upon receiving a message, it delegates the processing to a dedicated {@link ToUpperStringProcessor}. + * + * @param consumerRecord The Kafka consumer record containing the received message. + * @param consumer The Kafka consumer instance. + */ + @KafkaListener(topics = SPRING_KAFKA_TEST_TOPIC) + public void onMessage(ConsumerRecord consumerRecord, Consumer consumer) { + log.info("Received new message: {}", consumerRecord); + this.executeOnDedicatedProcessor(consumerRecord, consumer); + } + + /** + * Executes processing of a Kafka message using a thread-specific {@link ToUpperStringProcessor}. + * It ensures that each thread uses its unique instance of StringProcessor. + * + * @param consumerRecord The Kafka consumer record. + * @param consumer The Kafka consumer instance. + */ + private void executeOnDedicatedProcessor( + ConsumerRecord consumerRecord, Consumer consumer) { + String processorId = consumer.groupMetadata().memberId(); + if (TO_UPPER_STRING_PROCESSOR_THREAD_LOCAL.get() == null) { + TO_UPPER_STRING_PROCESSOR_THREAD_LOCAL.set(toUpperStringProcessorObjectProvider.getObject(processorId)); + } + TO_UPPER_STRING_PROCESSOR_THREAD_LOCAL.get().processString(consumerRecord.value()); + } + + /** + * Handles the {@link ConsumerStoppedEvent} to perform clean-up operations. + * This method is called when a Kafka consumer is stopped, and it removes the + * thread-local instance of {@link ToUpperStringProcessor} associated with the consumer. + * + * @param consumerStoppedEvent The event triggered when the Kafka consumer is stopped. + */ + @EventListener + public void onEvent(ConsumerStoppedEvent consumerStoppedEvent) { + log.info("Consumer stopped, closing String processor: {}", consumerStoppedEvent); + TO_UPPER_STRING_PROCESSOR_THREAD_LOCAL.remove(); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/processor/ToUpperStringProcessor.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/processor/ToUpperStringProcessor.java new file mode 100644 index 00000000..b145ba9f --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/java/com/example/kafka/processor/ToUpperStringProcessor.java @@ -0,0 +1,83 @@ +package com.example.kafka.processor; + +import jakarta.annotation.PostConstruct; +import java.util.HashSet; +import java.util.Set; +import java.util.StringJoiner; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +/** + * Processor class for handling string data. + * This class is responsible for processing and storing string values. + * It is scoped as a prototype bean in the Spring context. + */ +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +@Component +public class ToUpperStringProcessor { + + private static final Logger log = LoggerFactory.getLogger(ToUpperStringProcessor.class); + + private final String id; + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + + public ToUpperStringProcessor(String id) { + this.id = id; + } + + /** + * Initializes the ToUpperStringProcessor instance. + * This method logs the creation of a new ToUpperStringProcessor instance with its unique ID. + */ + @PostConstruct + public void init() { + log.info("Initializing new To Upper String Processor with id: {}", this.id); + } + + /** + * Processes and stores a string value in a queue. + * Logs the processing action and handles any potential interruption during the process. + * + * @param value The string value to be processed. + */ + public void processString(String value) { + log.info("Instance {} processing new string {}.", this.id, value); + try { + this.queue.put(value.toUpperCase()); + } catch (InterruptedException exception) { + log.error("Thread has been interrupted.", exception); + Thread.currentThread().interrupt(); + } + } + + /** + * Retrieves the current size of the queue. + * + * @return The number of elements in the queue. + */ + public int queueSize() { + return this.queue.size(); + } + + /** + * Returns a set of distinct strings from the queue. + * This method is useful for retrieving all unique values that have been processed. + * + * @return A Set of distinct strings. + */ + public Set distinctQueuedData() { + return new HashSet<>(this.queue); + } + + @Override + public String toString() { + return new StringJoiner(", ", ToUpperStringProcessor.class.getSimpleName() + "[", "]") + .add("id='" + id + "'") + .toString(); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/resources/application.properties b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/resources/application.properties new file mode 100644 index 00000000..00909a46 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/main/resources/application.properties @@ -0,0 +1,7 @@ +spring.application.name=concurrent-kafka-consumer + +spring.kafka.bootstrap-servers=localhost:9092 +spring.kafka.client-id=concurrent-kafka +spring.kafka.consumer.group-id=concurrent-kafka-consumer +spring.kafka.listener.change-consumer-thread-name=true +spring.kafka.listener.concurrency=3 diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/TestConcurrentKafkaConsumer.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/TestConcurrentKafkaConsumer.java new file mode 100644 index 00000000..cde04fa9 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/TestConcurrentKafkaConsumer.java @@ -0,0 +1,14 @@ +package com.example.kafka; + +import com.example.kafka.common.ContainerConfig; +import com.example.kafka.common.ProducerConfig; +import org.springframework.boot.SpringApplication; + +public class TestConcurrentKafkaConsumer { + + public static void main(String[] args) { + SpringApplication.from(ConcurrentKafkaConsumer::main) + .with(ContainerConfig.class, ProducerConfig.class) + .run(args); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ContainerConfig.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ContainerConfig.java new file mode 100644 index 00000000..d754f9ad --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ContainerConfig.java @@ -0,0 +1,17 @@ +package com.example.kafka.common; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.context.annotation.Bean; +import org.testcontainers.kafka.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +@TestConfiguration(proxyBeanMethods = false) +public class ContainerConfig { + + @Bean + @ServiceConnection + KafkaContainer kafkaContainer() { + return new KafkaContainer(DockerImageName.parse("apache/kafka-native").withTag("3.8.1")); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java new file mode 100644 index 00000000..296ff9e1 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java @@ -0,0 +1,53 @@ +package com.example.kafka.common; + +import com.example.kafka.listener.AppListener; +import com.example.kafka.processor.ToUpperStringProcessor; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import org.apache.kafka.clients.admin.NewTopic; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.config.TopicBuilder; +import org.springframework.kafka.listener.MessageListenerContainer; +import org.springframework.lang.NonNull; + +@TestConfiguration(proxyBeanMethods = false) +public class ProducerConfig { + + @Bean("toUpperStringProcessors") + List toUpperStringProcessors() { + return new ArrayList<>(); + } + + @Bean + BeanPostProcessor beanPostProcessor( + @Qualifier("toUpperStringProcessors") List toUpperStringProcessors) { + return new BeanPostProcessor() { + @Override + public Object postProcessAfterInitialization(@NonNull Object bean, @NonNull String beanName) + throws BeansException { + if (bean instanceof ToUpperStringProcessor toUpperStringProcessor) { + toUpperStringProcessors.add(toUpperStringProcessor); + } + return bean; + } + }; + } + + @Bean + NewTopic springKafkaTestTopic() { + return TopicBuilder.name(AppListener.SPRING_KAFKA_TEST_TOPIC) + .partitions(5) + .build(); + } + + @Bean + Function threadNameSupplier() { + return messageListenerContainer -> messageListenerContainer.getGroupId() + "-" + + List.of(messageListenerContainer.getListenerId().split("-")).getLast(); + } +} diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java new file mode 100644 index 00000000..411fc07f --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java @@ -0,0 +1,85 @@ +package com.example.kafka.listener; + +import static com.example.kafka.listener.AppListener.SPRING_KAFKA_TEST_TOPIC; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import com.example.kafka.common.ContainerConfig; +import com.example.kafka.common.ProducerConfig; +import com.example.kafka.processor.ToUpperStringProcessor; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.core.KafkaTemplate; + +@SpringBootTest( + classes = {ProducerConfig.class, ContainerConfig.class}, + properties = {"spring.kafka.producer.acks=1"}) +class AppListenerTest { + + private static final int NUM_MESSAGES = 1000; + private static final int PARTITION_KEY_DIVISOR = 10; + + @Autowired + private KafkaTemplate kafkaTemplate; + + @Autowired + @Qualifier("toUpperStringProcessors") List toUpperStringProcessors; + + /** + * Tests if all messages are processed by multiple StringProcessor instances. + * Ensures that the KafkaTemplate is not null, sends a number of messages, and + * then verifies that all messages are processed within a specified duration and + * that each StringProcessor instance has a non-empty queue with distinct data. + */ + @Test + void onMessage() { + assertThat(kafkaTemplate).isNotNull(); + CompletableFuture[] futures = sendMessages(); + assertAllMessagesProcessed(futures); + assertStringProcessorsBehaviors(); + } + + /** + * Asserts specific behaviors of the StringProcessor instances. + * Verifies that there are exactly two StringProcessor instances, each with a non-empty queue, + * and that the data in the queues of these two processors are distinct. + */ + private void assertStringProcessorsBehaviors() { + await().untilAsserted(() -> assertThat(toUpperStringProcessors).hasSize(3)); + assertThat(toUpperStringProcessors.get(0).queueSize()).isPositive(); + assertThat(toUpperStringProcessors.get(1).queueSize()).isPositive(); + assertThat(toUpperStringProcessors.get(2).queueSize()).isPositive(); + Set distinctQueuedData = toUpperStringProcessors.get(0).distinctQueuedData(); + distinctQueuedData.addAll(toUpperStringProcessors.get(2).distinctQueuedData()); + assertThat(toUpperStringProcessors.get(1).distinctQueuedData()).doesNotContainAnyElementsOf(distinctQueuedData); + } + + /** + * Sends a predefined number of messages to a Kafka topic. + * Each message has a key derived from a modulus operation and a specific data payload. + * + * @return an array of CompletableFuture objects representing the result of each send operation. + */ + private CompletableFuture[] sendMessages() { + return IntStream.range(0, NUM_MESSAGES) + .mapToObj(value -> "key_" + value % PARTITION_KEY_DIVISOR) + .map(key -> kafkaTemplate.send(SPRING_KAFKA_TEST_TOPIC, key, "data for key: " + key)) + .toArray(CompletableFuture[]::new); + } + + /** + * Asserts that all sent messages are processed within a specific time frame. + * + * @param futures an array of CompletableFuture objects representing the result of each send operation. + */ + private void assertAllMessagesProcessed(CompletableFuture[] futures) { + assertThat(CompletableFuture.allOf(futures)).succeedsWithin(Duration.ofSeconds(10)); + } +} From 20f305165809d8e4e57a8acc48310e04ca71e3c6 Mon Sep 17 00:00:00 2001 From: Raja Kolli Date: Thu, 26 Dec 2024 07:32:59 +0000 Subject: [PATCH 2/4] adds ReadMe and docker compose --- .../boot-concurrent-kafka-consumer/ReadMe.md | 38 +++++++++++++++++ .../docker/docker-compose.yml | 42 +++++++++++++++++++ .../boot-concurrent-kafka-consumer/pom.xml | 3 +- .../example/kafka/common/ProducerConfig.java | 3 +- .../kafka/listener/AppListenerTest.java | 4 +- 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md create mode 100644 kafka-spring-boot/boot-concurrent-kafka-consumer/docker/docker-compose.yml diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md new file mode 100644 index 00000000..9c0b6497 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md @@ -0,0 +1,38 @@ +# Concurrent Kafka Consumer + +This project demonstrates a multi-threaded Kafka consumer approach using Spring Boot, with a dedicated implementation that processes messages concurrently across different threads. + +### Sequence Diagram + + + +### Format code + +```shell +./mvnw spotless:apply +``` + +### Run tests + +```shell +./mvnw clean verify +``` + +### Run locally + +```shell +$ docker-compose -f docker/docker-compose.yml up -d +$ ./mvnw spring-boot:run -Dspring-boot.run.profiles=local +``` + +### Using Testcontainers at Development Time +You can run `TestConcurrentKafkaConsumer.java` from your IDE directly. +You can also run the application using Maven as follows: + +```shell +./mvnw spring-boot:test-run +``` + + +### Useful Links +* Actuator Endpoint: http://localhost:8080/actuator diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/docker/docker-compose.yml b/kafka-spring-boot/boot-concurrent-kafka-consumer/docker/docker-compose.yml new file mode 100644 index 00000000..f4a73b08 --- /dev/null +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/docker/docker-compose.yml @@ -0,0 +1,42 @@ +services: + + broker: + image: apache/kafka-native:3.8.1 + hostname: broker + ports: + - "9092:9092" + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 9092" ] + interval: 30s + timeout: 5s + retries: 3 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT + #Use two listeners with different names, it will force Kafka to communicate with itself via internal + #listener when KAFKA_INTER_BROKER_LISTENER_NAME is set, otherwise Kafka will try to use the advertised listener + KAFKA_LISTENERS: INTERNAL://broker:29092,EXTERNAL_HOST://0.0.0.0:9092,CONTROLLER://broker:9094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:29092,EXTERNAL_HOST://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 32 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_NODE_ID: 1 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker:9094 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_LOG_FLUSH_INTERVAL_MESSAGES: 9223372036854775807 + CLUSTER_ID: 4L6g3nShT-eMCtK--X86sw + + kafdrop: + image: obsidiandynamics/kafdrop + restart: "no" + ports: + - "9000:9000" + environment: + KAFKA_BROKERCONNECT: "broker:29092" + depends_on: + broker: + condition: service_healthy diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml b/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml index 7966d1e9..98b12156 100644 --- a/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/pom.xml @@ -22,7 +22,7 @@ org.springframework.boot - spring-boot-starter + spring-boot-starter-actuator org.springframework.kafka @@ -74,6 +74,7 @@ + diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java index 296ff9e1..70fb72f8 100644 --- a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/common/ProducerConfig.java @@ -3,6 +3,7 @@ import com.example.kafka.listener.AppListener; import com.example.kafka.processor.ToUpperStringProcessor; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.function.Function; import org.apache.kafka.clients.admin.NewTopic; @@ -20,7 +21,7 @@ public class ProducerConfig { @Bean("toUpperStringProcessors") List toUpperStringProcessors() { - return new ArrayList<>(); + return Collections.synchronizedList(new ArrayList<>()); } @Bean diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java index 411fc07f..968c4ced 100644 --- a/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/src/test/java/com/example/kafka/listener/AppListenerTest.java @@ -48,8 +48,8 @@ void onMessage() { /** * Asserts specific behaviors of the StringProcessor instances. - * Verifies that there are exactly two StringProcessor instances, each with a non-empty queue, - * and that the data in the queues of these two processors are distinct. + * Verifies that there are exactly three toUpperStringProcessor instances, each with a non-empty queue, + * and that the data in the queues of these three processors are distinct. */ private void assertStringProcessorsBehaviors() { await().untilAsserted(() -> assertThat(toUpperStringProcessors).hasSize(3)); From 3c785e5027ac069a33b50217f50d4eece991ddec Mon Sep 17 00:00:00 2001 From: Raja Kolli Date: Thu, 26 Dec 2024 07:38:41 +0000 Subject: [PATCH 3/4] updates with sequence diagram --- .../boot-concurrent-kafka-consumer/ReadMe.md | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md index 9c0b6497..1d716d05 100644 --- a/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md @@ -4,6 +4,81 @@ This project demonstrates a multi-threaded Kafka consumer approach using Spring ### Sequence Diagram +```mermaid + +sequenceDiagram + + participant C as ConcurrentKafkaConsumer + participant A as AppListener + participant P as ToUpperStringProcessor + + C->>C: Spring Boot initializes application + C->>A: Kafka messages arrive, triggering onMessage() + A->>A: Retrieve or initialize ThreadLocal + A->>P: processString(value) + P->>P: Converts to uppercase & enqueues result + Note over A,P: Additional messages handled similarly + A->>A: onEvent(ConsumerStoppedEvent) -> Remove ThreadLocal processor +``` + +## Project Structure + +### 1. `ConcurrentKafkaConsumer` +- **Location**: `com.example.kafka` +- **Description**: + - Serves as the Spring Boot application entry point. + - Annotated with `@SpringBootApplication`. + - The `main` method runs the Spring application using `SpringApplication.run`. + +### 2. `AppListener` +- **Location**: `com.example.kafka.listener` +- **Description**: + - A `@Component` that subscribes to a Kafka topic (referenced by `SPRING_KAFKA_TEST_TOPIC`). + - Uses a `ThreadLocal` to manage message processing on a per-thread basis. + - Processes messages in the `onMessage` method and handles resource cleanup in `onEvent` when the consumer stops. + +### 3. `ToUpperStringProcessor` +- **Location**: `com.example.kafka.processor` +- **Description**: + - A `@Component` with prototype scope, ensuring each instance is unique (e.g., one per thread). + - Transforms incoming strings to uppercase and stores them in a `BlockingQueue`. + - Offers methods like `distinctQueuedData()` to retrieve unique processed values. + +## Message Processing Sequence + +1. **Startup and Configuration** + - The `ConcurrentKafkaConsumer` class launches the Spring Boot application. + - Kafka consumer properties are configured in `application.properties`, including concurrency level and bootstrap servers. + +2. **Receiving Messages** + - The `AppListener` class is annotated with `@KafkaListener`, making it an endpoint for Kafka messages. + - When a message arrives, `onMessage` is triggered, and a `ToUpperStringProcessor` is retrieved or created for the current thread. + +3. **Processing** + - The processor converts the message to uppercase and stores it in a thread-safe queue. + - Each thread maintains its own `ToUpperStringProcessor`, eliminating conflicts between parallel consumers. + +4. **Resource Cleanup** + - When a consumer stops, `AppListener`’s `onEvent` method removes the processor from its `ThreadLocal`, preventing resource leaks. + +## Running the Application + +1. **Build and Start** + + ``` + ./mvnw clean package + ./mvnw spring-boot:run + ``` + +2. **Send Kafka Messages** + Send messages to the topic defined in `application.properties` (default is `SPRING_KAFKA_TEST_TOPIC`). +3. **Observe Logs** + Each thread logs its own unique processor ID and message processing. Check the console or log files for details. + +## Key Advantages +- **Increased Throughput**: Multiple threads can consume and process messages in parallel. +- **Isolation**: Each thread has a dedicated processor, avoiding synchronization overload. +- **Scalability**: The concurrency level can be fine-tuned via application properties. ### Format code From a25f76f6ddeedf12bf905b03e30a06b3841e3413 Mon Sep 17 00:00:00 2001 From: Raja Kolli Date: Thu, 26 Dec 2024 07:46:57 +0000 Subject: [PATCH 4/4] updates seq diagram --- .../boot-concurrent-kafka-consumer/ReadMe.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md index 1d716d05..512ee2cc 100644 --- a/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md +++ b/kafka-spring-boot/boot-concurrent-kafka-consumer/ReadMe.md @@ -13,11 +13,21 @@ sequenceDiagram participant P as ToUpperStringProcessor C->>C: Spring Boot initializes application + Note over C: Configure concurrent consumers C->>A: Kafka messages arrive, triggering onMessage() + par Thread 1 A->>A: Retrieve or initialize ThreadLocal A->>P: processString(value) P->>P: Converts to uppercase & enqueues result + and Thread 2 + A->>A: Retrieve or initialize ThreadLocal + A->>P: processString(value) + P->>P: Converts to uppercase & enqueues result + end Note over A,P: Additional messages handled similarly + alt Error occurs + A->>A: Handle error & retry/dead letter + end A->>A: onEvent(ConsumerStoppedEvent) -> Remove ThreadLocal processor ```