diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f4c1fdc..fc4b52e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -18,8 +18,7 @@ jobs:
max-parallel: 100
matrix:
spring_boot_version:
- - 3.4.6
- - 3.5.0
+ - 4.0.6
env:
SPRING_BOOT_VERSION: ${{ matrix.spring_boot_version }}
GRADLE_OPTS: "-Djava.security.egd=file:/dev/./urandom -Dorg.gradle.parallel=true"
@@ -52,29 +51,35 @@ jobs:
ALLOW_ANONYMOUS_LOGIN: "yes"
JVMFLAGS: "-Xmx512m -Xms64m"
kafka1:
- image: wurstmeister/kafka:2.12-2.2.0
+ image: bitnamilegacy/kafka:3.6.0
env:
- KAFKA_BROKER_ID: 1
- KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181/kafka1
- KAFKA_LISTENERS: PLAINTEXT://:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- KAFKA_MESSAGE_MAX_BYTES: "10485760"
- KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
- KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
- KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
- KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_CFG_BROKER_ID: 1
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper1:2181/kafka1
+ KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
+ KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
+ KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_CFG_MESSAGE_MAX_BYTES: "10485760"
+ KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
+ KAFKA_CFG_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
+ KAFKA_CFG_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
+ KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+ KAFKA_HEAP_OPTS: -Xms256m -Xmx768m
kafka2:
- image: wurstmeister/kafka:2.12-2.2.0
+ image: bitnamilegacy/kafka:3.6.0
env:
- KAFKA_BROKER_ID: 1
- KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181/kafka2
- KAFKA_LISTENERS: PLAINTEXT://:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- KAFKA_MESSAGE_MAX_BYTES: "10485760"
- KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
- KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
- KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
- KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_CFG_BROKER_ID: 1
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper1:2181/kafka2
+ KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
+ KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9092
+ KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_CFG_MESSAGE_MAX_BYTES: "10485760"
+ KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
+ KAFKA_CFG_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
+ KAFKA_CFG_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
+ KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+ KAFKA_HEAP_OPTS: -Xms256m -Xmx768m
steps:
- name: "Install packages"
# We do some changes as the main ubuntu repo is currently very slow.
diff --git a/build.common.gradle b/build.common.gradle
index 383d0d5..7234a25 100644
--- a/build.common.gradle
+++ b/build.common.gradle
@@ -75,7 +75,7 @@ dependencies {
}
java {
- if (springBootVersion.startsWith("3.")) {
+ if (springBootVersion.startsWith("3.") || springBootVersion.startsWith("4.")) {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
} else {
@@ -84,6 +84,10 @@ java {
}
withSourcesJar()
withJavadocJar()
+ toolchain {
+ languageVersion = JavaLanguageVersion.of(25)
+ vendor = JvmVendorSpec.ADOPTIUM
+ }
}
compileJava {
diff --git a/build.gradle b/build.gradle
index 0b78d5d..b33ae81 100644
--- a/build.gradle
+++ b/build.gradle
@@ -3,8 +3,12 @@ import com.github.spotbugs.snom.Effort
import org.eclipse.jgit.api.errors.RefAlreadyExistsException
buildscript {
+ repositories {
+ google()
+ mavenCentral()
+ }
if (!project.hasProperty("springBootVersion")) {
- ext.springBootVersion = System.getenv("SPRING_BOOT_VERSION") ?: "3.4.6"
+ ext.springBootVersion = System.getenv("SPRING_BOOT_VERSION") ?: "4.0.6"
}
dependencies {
classpath 'com.google.protobuf:protobuf-gradle-plugin:0.9.4'
@@ -15,11 +19,11 @@ buildscript {
plugins {
id 'idea'
id 'org.springframework.boot' version "$springBootVersion" apply false
- id "com.github.spotbugs" version "6.0.2"
+ id "com.github.spotbugs" version "6.5.5"
id "at.zierler.yamlvalidator" version "1.5.0"
id 'org.ajoberstar.grgit' version '5.2.1'
id 'io.github.gradle-nexus.publish-plugin' version "1.3.0"
- id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
+ id 'com.gradleup.shadow' version '9.4.2' apply false
}
idea.project {
@@ -29,6 +33,7 @@ idea.project {
}
spotbugs {
+ toolVersion = '4.9.8'
effort = Effort.valueOf('MAX')
reportLevel = Confidence.valueOf('DEFAULT')
}
diff --git a/build.libraries.gradle b/build.libraries.gradle
index 19c3ab7..2ed9cb8 100644
--- a/build.libraries.gradle
+++ b/build.libraries.gradle
@@ -1,6 +1,6 @@
ext {
protobufVersion = "3.24.0"
- springBootVersion = "${System.getenv("SPRING_BOOT_VERSION") ?: '3.4.6'}"
+ springBootVersion = "${System.getenv("SPRING_BOOT_VERSION") ?: '4.0.6'}"
libraries = [
// version defined
awaitility : 'org.awaitility:awaitility:4.2.0',
@@ -11,7 +11,7 @@ ext {
jakartaValidationApi : 'jakarta.validation:jakarta.validation-api:3.0.2',
javaxValidationApi : "javax.validation:validation-api:2.0.1.Final",
kafkaStreams : 'org.apache.kafka:kafka-streams:3.2.3',
- lz4Java : 'org.lz4:lz4-java:1.8.0',
+ lz4Java : 'at.yawk.lz4:lz4-java:1.10.1',
protobufJava : "com.google.protobuf:protobuf-java:${protobufVersion}",
semver4j : "com.vdurmont:semver4j:3.1.0",
snappyJava : 'org.xerial.snappy:snappy-java:1.1.10.5',
@@ -32,7 +32,7 @@ ext {
flywayCore : 'org.flywaydb:flyway-core',
flywayMysql : 'org.flywaydb:flyway-mysql',
flywayPostgresql : 'org.flywaydb:flyway-database-postgresql',
- jacksonDatabind : 'com.fasterxml.jackson.core:jackson-databind',
+ jacksonDatabind : 'tools.jackson.core:jackson-databind',
kafkaClients : 'org.apache.kafka:kafka-clients',
lombok : 'org.projectlombok:lombok',
mariadbJavaClient : 'org.mariadb.jdbc:mariadb-java-client',
@@ -40,6 +40,9 @@ ext {
micrometerRegistryPrometheus : 'io.micrometer:micrometer-registry-prometheus',
postgresql : 'org.postgresql:postgresql',
springBootConfigurationProcessor: 'org.springframework.boot:spring-boot-configuration-processor',
+ springBootFlyway : 'org.springframework.boot:spring-boot-flyway',
+ springBootKafka : 'org.springframework.boot:spring-boot-kafka',
+ springBootValidation : 'org.springframework.boot:spring-boot-validation',
springBootStarter : 'org.springframework.boot:spring-boot-starter',
springBootStarterActuator : 'org.springframework.boot:spring-boot-starter-actuator',
springBootStarterJdbc : 'org.springframework.boot:spring-boot-starter-jdbc',
diff --git a/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestController.java b/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestController.java
index be53b69..04810ec 100644
--- a/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestController.java
+++ b/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestController.java
@@ -1,6 +1,5 @@
package com.transferwise.kafka.tkms.demoapp.complextest;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.kafka.tkms.api.ITransactionalKafkaMessageSender;
import com.transferwise.kafka.tkms.api.ITransactionalKafkaMessageSender.SendMessageResult;
import com.transferwise.kafka.tkms.api.TkmsMessage;
@@ -10,6 +9,7 @@
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
+import tools.jackson.databind.ObjectMapper;
@RestController
public class ComplexTestController {
diff --git a/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestMessagesListener.java b/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestMessagesListener.java
index 81e4c9b..9dcf1ed 100644
--- a/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestMessagesListener.java
+++ b/demoapp/src/main/java/com/transferwise/kafka/tkms/demoapp/complextest/ComplexTestMessagesListener.java
@@ -1,6 +1,5 @@
package com.transferwise.kafka.tkms.demoapp.complextest;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.common.baseutils.ExceptionUtils;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -13,6 +12,7 @@
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
+import tools.jackson.databind.ObjectMapper;
@Slf4j
@Component
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index c1962a7..b1b8ef5 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 3499ded..df6a6ad 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,9 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.1-bin.zip
networkTimeout=10000
+retries=0
+retryBackOffMs=500
+validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index aeb74cb..b9bb139 100755
--- a/gradlew
+++ b/gradlew
@@ -1,7 +1,7 @@
#!/bin/sh
#
-# Copyright © 2015-2021 the original authors.
+# Copyright © 2015 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# SPDX-License-Identifier: Apache-2.0
+#
##############################################################################
#
@@ -55,7 +57,7 @@
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
-# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
@@ -83,7 +85,8 @@ done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
@@ -111,7 +114,6 @@ case "$( uname )" in #(
NONSTOP* ) nonstop=true ;;
esac
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
@@ -130,10 +132,13 @@ location of your Java installation."
fi
else
JAVACMD=java
- which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+ if ! command -v java >/dev/null 2>&1
+ then
+ die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
+ fi
fi
# Increase the maximum file descriptors if we can.
@@ -141,7 +146,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
- # shellcheck disable=SC3045
+ # shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
@@ -149,7 +154,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
- # shellcheck disable=SC3045
+ # shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
@@ -166,7 +171,6 @@ fi
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
- CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
@@ -198,16 +202,15 @@ fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-# Collect all arguments for the java command;
-# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-# shell script including quotes and variable substitutions, so put them in
-# double quotes to make sure that they get re-expanded; and
-# * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+# and any embedded shellness will be escaped.
+# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
- -classpath "$CLASSPATH" \
- org.gradle.wrapper.GradleWrapperMain \
+ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"
# Stop when "xargs" is not available.
diff --git a/gradlew.bat b/gradlew.bat
index 6689b85..aa5f10b 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -13,6 +13,8 @@
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
@@ -21,8 +23,8 @@
@rem
@rem ##########################################################################
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
+@rem Set local scope for the variables, and ensure extensions are enabled
+setlocal EnableExtensions
set DIRNAME=%~dp0
if "%DIRNAME%"=="" set DIRNAME=.
@@ -43,13 +45,13 @@ set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
-goto fail
+"%COMSPEC%" /c exit 1
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
@@ -57,36 +59,24 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
-goto fail
+"%COMSPEC%" /c exit 1
:execute
@rem Setup the command line
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
-
-:end
-@rem End local scope for the variables with windows NT shell
-if %ERRORLEVEL% equ 0 goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-set EXIT_CODE=%ERRORLEVEL%
-if %EXIT_CODE% equ 0 set EXIT_CODE=1
-if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
-exit /b %EXIT_CODE%
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
+@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
+@rem which allows us to clear the local environment before executing the java command
+endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
-:omega
+:exitWithErrorLevel
+@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
+"%COMSPEC%" /c exit %ERRORLEVEL%
diff --git a/mise.toml b/mise.toml
new file mode 100644
index 0000000..33b2c9b
--- /dev/null
+++ b/mise.toml
@@ -0,0 +1,2 @@
+[tools]
+java = "temurin-25"
diff --git a/settings.gradle b/settings.gradle
index 2d9d6f4..1eb0245 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,3 +1,6 @@
+plugins {
+ id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
+}
rootProject.name = "tw-tkms"
include ':tw-tkms-starter'
include ':tw-tkms-test-starter'
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
index a54e4dc..1c6e3a0 100644
--- a/spotbugs-exclude.xml
+++ b/spotbugs-exclude.xml
@@ -5,11 +5,7 @@
xsi:schemaLocation="https://github.com/spotbugs/filter/3.0.0 https://raw.githubusercontent.com/spotbugs/spotbugs/3.1.0/spotbugs/etc/findbugsfilter.xsd">
-
-
-
-
-
+
diff --git a/tw-tkms-starter/build.gradle b/tw-tkms-starter/build.gradle
index 82f5228..54d9935 100644
--- a/tw-tkms-starter/build.gradle
+++ b/tw-tkms-starter/build.gradle
@@ -3,7 +3,7 @@ plugins {
id 'idea'
id "com.google.protobuf"
id "docker-compose"
- id 'com.github.johnrengelman.shadow'
+ id 'com.gradleup.shadow'
id 'maven-publish'
id 'signing'
}
@@ -33,6 +33,8 @@ dependencies {
implementation libraries.micrometerCore
implementation libraries.protobufJava
implementation libraries.springBootStarter
+ implementation libraries.springBootFlyway
+ implementation libraries.springBootValidation
implementation libraries.springJdbc
implementation libraries.twBaseUtils
implementation libraries.twContext
@@ -67,6 +69,7 @@ dependencies {
testImplementation libraries.springBootStarterTest
testImplementation libraries.springBootStarterWeb
testImplementation libraries.springKafka
+ testRuntimeOnly libraries.springBootKafka
testRuntimeOnly libraries.mariadbJavaClient
testRuntimeOnly libraries.postgresql
@@ -114,11 +117,7 @@ test {
*/
shadowJar {
dependencies {
- dependencies {
- exclude(dependency {
- it.moduleName != 'protobuf-java'
- })
- }
+ include { it.moduleName == 'protobuf-java' }
}
manifest {
attributes 'Implementation-Version': "$project.version"
diff --git a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/ITransactionalKafkaMessageSender.java b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/ITransactionalKafkaMessageSender.java
index aadc320..e86a758 100644
--- a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/ITransactionalKafkaMessageSender.java
+++ b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/ITransactionalKafkaMessageSender.java
@@ -14,6 +14,8 @@ public interface ITransactionalKafkaMessageSender {
*/
SendMessageResult sendMessage(TkmsMessage message);
+ SendMessageResult sendMessage(SendMessageRequest request);
+
@Data
@Accessors(chain = true)
class SendMessageResult {
@@ -34,7 +36,6 @@ class SendMessageResult {
private TkmsShardPartition shardPartition;
}
- SendMessageResult sendMessage(SendMessageRequest request);
@Data
@Accessors(chain = true)
diff --git a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/TkmsMessage.java b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/TkmsMessage.java
index 14b4536..c7998d4 100644
--- a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/TkmsMessage.java
+++ b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/TkmsMessage.java
@@ -49,7 +49,7 @@ public class TkmsMessage {
/**
* Kafka message's content/payload.
*/
- @SuppressFBWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"})
+ @SuppressFBWarnings("EI_EXPOSE_REP")
@NotNull
private byte[] value;
diff --git a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/helpers/TkmsMessageFactory.java b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/helpers/TkmsMessageFactory.java
index 1698f67..316cb6e 100644
--- a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/helpers/TkmsMessageFactory.java
+++ b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/api/helpers/TkmsMessageFactory.java
@@ -1,11 +1,11 @@
package com.transferwise.kafka.tkms.api.helpers;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.common.baseutils.ExceptionUtils;
import com.transferwise.kafka.tkms.api.TkmsMessage;
import java.nio.charset.StandardCharsets;
import lombok.NonNull;
import org.springframework.beans.factory.annotation.Autowired;
+import tools.jackson.databind.ObjectMapper;
public class TkmsMessageFactory implements ITkmsMessageFactory {
diff --git a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/config/TkmsAutoConfiguration.java b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/config/TkmsAutoConfiguration.java
index 987a80c..f0c0771 100644
--- a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/config/TkmsAutoConfiguration.java
+++ b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/config/TkmsAutoConfiguration.java
@@ -13,8 +13,8 @@
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
-import org.springframework.boot.autoconfigure.validation.ValidationAutoConfiguration;
+import org.springframework.boot.flyway.autoconfigure.FlywayAutoConfiguration;
+import org.springframework.boot.validation.autoconfigure.ValidationAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
diff --git a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/dao/TkmsDao.java b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/dao/TkmsDao.java
index 121f31a..ddab063 100644
--- a/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/dao/TkmsDao.java
+++ b/tw-tkms-starter/src/main/java/com/transferwise/kafka/tkms/dao/TkmsDao.java
@@ -9,7 +9,6 @@
import com.transferwise.kafka.tkms.api.TkmsShardPartition;
import com.transferwise.kafka.tkms.config.TkmsProperties;
import com.transferwise.kafka.tkms.metrics.ITkmsMetricsTemplate;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -195,7 +194,6 @@ protected InputStream serializeMessage(TkmsShardPartition shardPartition, TkmsMe
return ExceptionUtils.doUnchecked(() -> messageSerializer.serialize(shardPartition, message));
}
- @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
protected long keyToLong(KeyHolder keyHolder) {
return keyHolder.getKey().longValue();
}
diff --git a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/EndToEndIntTest.java b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/EndToEndIntTest.java
index 51dc59d..3602567 100644
--- a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/EndToEndIntTest.java
+++ b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/EndToEndIntTest.java
@@ -4,7 +4,6 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.awaitility.Awaitility.await;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.common.baseutils.ExceptionUtils;
import com.transferwise.common.baseutils.UuidUtils;
import com.transferwise.common.baseutils.transactionsmanagement.ITransactionsHelper;
@@ -49,6 +48,7 @@
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.springframework.beans.factory.annotation.Autowired;
+import tools.jackson.databind.ObjectMapper;
@BaseTestEnvironment
@Slf4j
diff --git a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/FlywayConfiguration.java b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/FlywayConfiguration.java
index b0deb18..0373904 100644
--- a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/FlywayConfiguration.java
+++ b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/FlywayConfiguration.java
@@ -1,6 +1,6 @@
package com.transferwise.kafka.tkms.test;
-import org.springframework.boot.autoconfigure.flyway.FlywayMigrationStrategy;
+import org.springframework.boot.flyway.autoconfigure.FlywayMigrationStrategy;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
diff --git a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/KafkaConfiguration.java b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/KafkaConfiguration.java
index 8c93059..4204a4f 100644
--- a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/KafkaConfiguration.java
+++ b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/KafkaConfiguration.java
@@ -2,7 +2,6 @@
import com.transferwise.common.baseutils.ExceptionUtils;
import com.transferwise.kafka.tkms.test.TestProperties.KafkaServer;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -29,7 +28,6 @@ public class KafkaConfiguration implements InitializingBean {
@Autowired
private TestProperties tkmsProperties;
- @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE")
@Override
public void afterPropertiesSet() {
for (var kafkaServer : tkmsProperties.getKafkaServers()) {
@@ -73,6 +71,7 @@ protected void createTopic(final AdminClient adminClient, final String topicName
final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));
createTopicsResult.values().get(topicName).get();
log.info("Created Kafka topic '" + topicName + "' in " + i + ".");
+ awaitTopicMetadataPropagated(adminClient, topicName, partitions);
return;
} catch (InterruptedException | ExecutionException e) {
if (!(e.getCause() instanceof TopicExistsException)) {
@@ -83,4 +82,29 @@ protected void createTopic(final AdminClient adminClient, final String topicName
}
}
+ /**
+ * `createTopics` acknowledges as soon as the topic is created in the controller, but partition leadership is not necessarily propagated to all
+ * brokers yet. TKMS validates topics via the producer's metadata at startup, which fails if it queries before propagation completes. Wait until the
+ * topic is fully describable with a leader for every partition, so topic validation does not flake under load.
+ */
+ protected void awaitTopicMetadataPropagated(final AdminClient adminClient, final String topicName, final int partitions) {
+ for (int i = 0; i < 100; i++) {
+ try {
+ var description = adminClient.describeTopics(Collections.singleton(topicName)).allTopicNames().get().get(topicName);
+ boolean ready = description != null
+ && description.partitions().size() == partitions
+ && description.partitions().stream().allMatch(p -> p.leader() != null && p.leader().id() >= 0);
+ if (ready) {
+ return;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ if (!(e.getCause() instanceof UnknownTopicOrPartitionException)) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ ExceptionUtils.doUnchecked(() -> Thread.sleep(50));
+ }
+ throw new IllegalStateException("Topic '" + topicName + "' metadata did not propagate in time.");
+ }
+
}
diff --git a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/TestApplication.java b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/TestApplication.java
index ac71093..72234c3 100644
--- a/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/TestApplication.java
+++ b/tw-tkms-starter/src/test/java/com/transferwise/kafka/tkms/test/TestApplication.java
@@ -4,7 +4,7 @@
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.flyway.FlywayMigrationStrategy;
+import org.springframework.boot.flyway.autoconfigure.FlywayMigrationStrategy;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
diff --git a/tw-tkms-starter/src/test/resources/docker-compose.yml b/tw-tkms-starter/src/test/resources/docker-compose.yml
index 3b7a4eb..c84b6d8 100644
--- a/tw-tkms-starter/src/test/resources/docker-compose.yml
+++ b/tw-tkms-starter/src/test/resources/docker-compose.yml
@@ -9,55 +9,45 @@ services:
ALLOW_ANONYMOUS_LOGIN: "yes"
JVMFLAGS: -server -Xms25m -Xmx512m -XX:+HeapDumpOnOutOfMemoryError -XX:GCHeapFreeLimit=5 -XX:GCTimeLimit=90 -XX:SoftRefLRUPolicyMSPerMB=5 -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:+ExplicitGCInvokesConcurrent
kafka1:
- image: wurstmeister/kafka:2.12-2.4.1
+ image: bitnamilegacy/kafka:3.6.0
ports:
- - "9092"
+ - "9092:9092"
container_name: "tw_tkms_kafka_1"
environment:
- PORT_COMMAND: "docker port $$(docker ps -q -f name=tw_tkms_kafka_1) 9092/tcp | cut -d: -f2"
- KAFKA_BROKER_ID: 1
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka1
- KAFKA_ADVERTISED_LISTENERS: "EXTERNAL://localhost:_{PORT_COMMAND},INTERNAL://kafka1:9093"
- KAFKA_LISTENERS: EXTERNAL://:9092,INTERNAL://:9093
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
- KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- KAFKA_MESSAGE_MAX_BYTES: 10485760
- KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
- KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
- KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
- KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
- LOG4J_LOGGER_ORG: WARN,STDOUT
- LOG4J_LOGGER_ORG_APACHE_KAFKA: WARN,STDOUT
- LOG4J_LOGGER_KAFKA: WARN,STDOUT
- KAFKA_JVM_PERFORMANCE_OPTS: -server -Xms25m -Xmx512m -XX:+HeapDumpOnOutOfMemoryError -XX:GCHeapFreeLimit=5 -XX:GCTimeLimit=90 -XX:SoftRefLRUPolicyMSPerMB=5 -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:+ExplicitGCInvokesConcurrent
- volumes:
- - /var/run/docker.sock:/var/run/docker.sock
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_CFG_BROKER_ID: 1
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181/kafka1
+ KAFKA_CFG_LISTENERS: EXTERNAL://:9092,INTERNAL://:9093
+ KAFKA_CFG_ADVERTISED_LISTENERS: "EXTERNAL://localhost:9092,INTERNAL://kafka1:9093"
+ KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
+ KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
+ KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_CFG_MESSAGE_MAX_BYTES: 10485760
+ KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
+ KAFKA_CFG_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
+ KAFKA_CFG_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
+ KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+ KAFKA_HEAP_OPTS: -Xms256m -Xmx768m
kafka2:
- image: wurstmeister/kafka:2.12-2.4.1
+ image: bitnamilegacy/kafka:3.6.0
ports:
- - "9092"
+ - "9093:9092"
container_name: "tw_tkms_kafka_2"
environment:
- PORT_COMMAND: "docker port $$(docker ps -q -f name=tw_tkms_kafka_2) 9092/tcp | cut -d: -f2"
- KAFKA_BROKER_ID: 2
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka2
- KAFKA_ADVERTISED_LISTENERS: "EXTERNAL://localhost:_{PORT_COMMAND},INTERNAL://kafka2:9093"
- KAFKA_LISTENERS: EXTERNAL://:9092,INTERNAL://:9093
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
- KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- KAFKA_MESSAGE_MAX_BYTES: 10485760
- KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
- KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
- KAFKA_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
- KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
- LOG4J_LOGGER_ORG: WARN,STDOUT
- LOG4J_LOGGER_ORG_APACHE_KAFKA: WARN,STDOUT
- LOG4J_LOGGER_KAFKA: WARN,STDOUT
- KAFKA_JVM_PERFORMANCE_OPTS: -server -Xms25m -Xmx512m -XX:+HeapDumpOnOutOfMemoryError -XX:GCHeapFreeLimit=5 -XX:GCTimeLimit=90 -XX:SoftRefLRUPolicyMSPerMB=5 -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:+ExplicitGCInvokesConcurrent
- volumes:
- - /var/run/docker.sock:/var/run/docker.sock
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_CFG_BROKER_ID: 2
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181/kafka2
+ KAFKA_CFG_LISTENERS: EXTERNAL://:9092,INTERNAL://:9093
+ KAFKA_CFG_ADVERTISED_LISTENERS: "EXTERNAL://localhost:9093,INTERNAL://kafka2:9093"
+ KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
+ KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
+ KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_CFG_MESSAGE_MAX_BYTES: 10485760
+ KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000
+ KAFKA_CFG_UNCLEAN_LEADER_ELECTION_ENABLE: "true"
+ KAFKA_CFG_LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS: 5
+ KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+ KAFKA_HEAP_OPTS: -Xms256m -Xmx768m
mariadb:
image: mariadb:10.6
ports:
diff --git a/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsRegisteredMessagesCollector.java b/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsRegisteredMessagesCollector.java
index 601101c..cea7a9b 100644
--- a/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsRegisteredMessagesCollector.java
+++ b/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsRegisteredMessagesCollector.java
@@ -1,6 +1,5 @@
package com.transferwise.kafka.tkms.test;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.common.baseutils.ExceptionUtils;
import com.transferwise.kafka.tkms.api.ITkmsEventsListener;
import com.transferwise.kafka.tkms.api.TkmsShardPartition;
@@ -14,6 +13,7 @@
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
+import tools.jackson.databind.ObjectMapper;
public class TkmsRegisteredMessagesCollector implements ITkmsRegisteredMessagesCollector, ITkmsEventsListener {
@@ -26,7 +26,7 @@ public class TkmsRegisteredMessagesCollector implements ITkmsRegisteredMessagesC
private AtomicInteger messagesCount = new AtomicInteger();
- private boolean enabled = true;
+ private volatile boolean enabled = true;
@Override
public void messageRegistered(MessageRegisteredEvent event) {
diff --git a/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsSentMessagesCollector.java b/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsSentMessagesCollector.java
index 6649a82..19972be 100644
--- a/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsSentMessagesCollector.java
+++ b/tw-tkms-test-starter/src/main/java/com/transferwise/kafka/tkms/test/TkmsSentMessagesCollector.java
@@ -1,6 +1,5 @@
package com.transferwise.kafka.tkms.test;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.transferwise.common.baseutils.ExceptionUtils;
import com.transferwise.kafka.tkms.api.ITkmsEventsListener;
import com.transferwise.kafka.tkms.api.TkmsShardPartition;
@@ -14,6 +13,7 @@
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
+import tools.jackson.databind.ObjectMapper;
public class TkmsSentMessagesCollector implements ITkmsSentMessagesCollector, ITkmsEventsListener {