feat(Tracer Bullet)
This commit is contained in:
+5
-3
@@ -1,6 +1,8 @@
|
||||
package at.mocode.infrastructure.eventstore.redis
|
||||
|
||||
import at.mocode.core.domain.event.DomainEvent
|
||||
import at.mocode.core.domain.model.AggregateId
|
||||
import at.mocode.core.domain.model.EventVersion
|
||||
import at.mocode.infrastructure.eventstore.api.ConcurrencyException
|
||||
import at.mocode.infrastructure.eventstore.api.EventSerializer
|
||||
import at.mocode.infrastructure.eventstore.api.EventStore
|
||||
@@ -26,7 +28,7 @@ class RedisEventStore(
|
||||
|
||||
val aggregateId = events.first().aggregateId
|
||||
require(events.all { it.aggregateId == aggregateId }) { "All events must belong to the same aggregate" }
|
||||
require(streamId == aggregateId) { "Stream ID must match aggregate ID" }
|
||||
require(streamId == aggregateId.value) { "Stream ID must match aggregate ID" }
|
||||
|
||||
var currentVersion = getStreamVersion(streamId)
|
||||
|
||||
@@ -59,7 +61,7 @@ class RedisEventStore(
|
||||
|
||||
private fun appendToStreamInternal(event: DomainEvent, streamId: Uuid, currentVersion: Long): Long {
|
||||
val newVersion = currentVersion + 1
|
||||
require(event.version == newVersion) { "Event version ${event.version} does not match expected new version $newVersion" }
|
||||
require(event.version.value == newVersion) { "Event version ${event.version} does not match expected new version $newVersion" }
|
||||
|
||||
val streamKey = getStreamKey(streamId)
|
||||
val allEventsStreamKey = getAllEventsStreamKey()
|
||||
@@ -102,7 +104,7 @@ class RedisEventStore(
|
||||
}
|
||||
} ?: emptyList()
|
||||
|
||||
return events.filter { it.version >= fromVersion && (toVersion == null || it.version <= toVersion) }
|
||||
return events.filter { it.version >= EventVersion(fromVersion) && (toVersion == null || it.version <= EventVersion(toVersion)) }
|
||||
}
|
||||
|
||||
override fun getStreamVersion(streamId: Uuid): Long {
|
||||
|
||||
+19
-18
@@ -2,6 +2,7 @@ package at.mocode.infrastructure.eventstore.redis
|
||||
|
||||
import at.mocode.core.domain.event.BaseDomainEvent
|
||||
import at.mocode.core.domain.event.DomainEvent
|
||||
import at.mocode.core.domain.model.*
|
||||
import at.mocode.infrastructure.eventstore.api.EventSerializer
|
||||
import at.mocode.infrastructure.eventstore.api.EventStore
|
||||
import com.benasher44.uuid.Uuid
|
||||
@@ -85,8 +86,8 @@ class RedisEventStoreIntegrationTest {
|
||||
@Test
|
||||
fun `event publishing and consuming with consumer groups should work`() {
|
||||
val aggregateId = uuid4()
|
||||
val event1 = TestCreatedEvent(aggregateId = aggregateId, version = 1L, name = "Test Entity")
|
||||
val event2 = TestUpdatedEvent(aggregateId = aggregateId, version = 2L, name = "Updated Test Entity")
|
||||
val event1 = TestCreatedEvent(aggregateId = AggregateId(aggregateId), version = EventVersion(1L), name = "Test Entity")
|
||||
val event2 = TestUpdatedEvent(aggregateId = AggregateId(aggregateId), version = EventVersion(2L), name = "Updated Test Entity")
|
||||
|
||||
val latch = CountDownLatch(2)
|
||||
val receivedEvents = mutableListOf<DomainEvent>()
|
||||
@@ -112,34 +113,34 @@ class RedisEventStoreIntegrationTest {
|
||||
|
||||
assertEquals(2, receivedEvents.size)
|
||||
|
||||
val receivedEvent1 = receivedEvents.find { it.version == 1L } as TestCreatedEvent
|
||||
assertEquals(aggregateId, receivedEvent1.aggregateId)
|
||||
val receivedEvent1 = receivedEvents.find { it.version == EventVersion(1L) } as TestCreatedEvent
|
||||
assertEquals(AggregateId(aggregateId), receivedEvent1.aggregateId)
|
||||
assertEquals("Test Entity", receivedEvent1.name)
|
||||
|
||||
val receivedEvent2 = receivedEvents.find { it.version == 2L } as TestUpdatedEvent
|
||||
assertEquals(aggregateId, receivedEvent2.aggregateId)
|
||||
val receivedEvent2 = receivedEvents.find { it.version == EventVersion(2L) } as TestUpdatedEvent
|
||||
assertEquals(AggregateId(aggregateId), receivedEvent2.aggregateId)
|
||||
assertEquals("Updated Test Entity", receivedEvent2.name)
|
||||
}
|
||||
|
||||
data class TestCreatedEvent(
|
||||
override val aggregateId: Uuid,
|
||||
override val version: Long,
|
||||
override val aggregateId: AggregateId,
|
||||
override val version: EventVersion,
|
||||
val name: String,
|
||||
override val eventType: String = "TestCreated",
|
||||
override val eventId: Uuid = uuid4(),
|
||||
override val eventType: EventType = EventType("TestCreated"),
|
||||
override val eventId: EventId = EventId(uuid4()),
|
||||
override val timestamp: Instant = Clock.System.now(),
|
||||
override val correlationId: Uuid? = null,
|
||||
override val causationId: Uuid? = null
|
||||
override val correlationId: CorrelationId? = null,
|
||||
override val causationId: CausationId? = null
|
||||
) : BaseDomainEvent(aggregateId, eventType, version, eventId, timestamp, correlationId, causationId)
|
||||
|
||||
data class TestUpdatedEvent(
|
||||
override val aggregateId: Uuid,
|
||||
override val version: Long,
|
||||
override val aggregateId: AggregateId,
|
||||
override val version: EventVersion,
|
||||
val name: String,
|
||||
override val eventType: String = "TestUpdated",
|
||||
override val eventId: Uuid = uuid4(),
|
||||
override val eventType: EventType = EventType("TestUpdated"),
|
||||
override val eventId: EventId = EventId(uuid4()),
|
||||
override val timestamp: Instant = Clock.System.now(),
|
||||
override val correlationId: Uuid? = null,
|
||||
override val causationId: Uuid? = null
|
||||
override val correlationId: CorrelationId? = null,
|
||||
override val causationId: CausationId? = null
|
||||
) : BaseDomainEvent(aggregateId, eventType, version, eventId, timestamp, correlationId, causationId)
|
||||
}
|
||||
|
||||
+13
-12
@@ -1,6 +1,7 @@
|
||||
package at.mocode.infrastructure.eventstore.redis
|
||||
|
||||
import at.mocode.core.domain.event.BaseDomainEvent
|
||||
import at.mocode.core.domain.model.*
|
||||
import at.mocode.infrastructure.eventstore.api.ConcurrencyException
|
||||
import at.mocode.infrastructure.eventstore.api.EventSerializer
|
||||
import com.benasher44.uuid.Uuid
|
||||
@@ -68,8 +69,8 @@ class RedisEventStoreTest {
|
||||
@Test
|
||||
fun `append and read events should work correctly for new stream`() {
|
||||
val aggregateId = uuid4()
|
||||
val event1 = TestCreatedEvent(aggregateId, 1L, "Test Entity")
|
||||
val event2 = TestUpdatedEvent(aggregateId, 2L, "Updated Test Entity")
|
||||
val event1 = TestCreatedEvent(AggregateId(aggregateId), EventVersion(1L), "Test Entity")
|
||||
val event2 = TestUpdatedEvent(AggregateId(aggregateId), EventVersion(2L), "Updated Test Entity")
|
||||
|
||||
eventStore.appendToStream(listOf(event1, event2), aggregateId, 0)
|
||||
|
||||
@@ -77,21 +78,21 @@ class RedisEventStoreTest {
|
||||
assertEquals(2, events.size)
|
||||
|
||||
val firstEvent = events[0] as TestCreatedEvent
|
||||
assertEquals(1L, firstEvent.version)
|
||||
assertEquals(EventVersion(1L), firstEvent.version)
|
||||
assertEquals("Test Entity", firstEvent.name)
|
||||
|
||||
val secondEvent = events[1] as TestUpdatedEvent
|
||||
assertEquals(2L, secondEvent.version)
|
||||
assertEquals(EventVersion(2L), secondEvent.version)
|
||||
assertEquals("Updated Test Entity", secondEvent.name)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `appending with wrong expected version should throw ConcurrencyException`() {
|
||||
val aggregateId = uuid4()
|
||||
val event1 = TestCreatedEvent(aggregateId, 1L, "Test Entity")
|
||||
val event1 = TestCreatedEvent(AggregateId(aggregateId), EventVersion(1L), "Test Entity")
|
||||
eventStore.appendToStream(listOf(event1), aggregateId, 0) // Stream is now at version 1
|
||||
|
||||
val event2 = TestUpdatedEvent(aggregateId, 2L, "Updated Test Entity")
|
||||
val event2 = TestUpdatedEvent(AggregateId(aggregateId), EventVersion(2L), "Updated Test Entity")
|
||||
assertThrows<ConcurrencyException> {
|
||||
eventStore.appendToStream(listOf(event2), aggregateId, 0)
|
||||
}
|
||||
@@ -99,15 +100,15 @@ class RedisEventStoreTest {
|
||||
|
||||
@Serializable
|
||||
data class TestCreatedEvent(
|
||||
@Transient override val aggregateId: Uuid = uuid4(),
|
||||
@Transient override val version: Long = 0,
|
||||
@Transient override val aggregateId: AggregateId = AggregateId(uuid4()),
|
||||
@Transient override val version: EventVersion = EventVersion(0),
|
||||
val name: String
|
||||
) : BaseDomainEvent(aggregateId, "TestCreated", version)
|
||||
) : BaseDomainEvent(aggregateId, EventType("TestCreated"), version)
|
||||
|
||||
@Serializable
|
||||
data class TestUpdatedEvent(
|
||||
@Transient override val aggregateId: Uuid = uuid4(),
|
||||
@Transient override val version: Long = 0,
|
||||
@Transient override val aggregateId: AggregateId = AggregateId(uuid4()),
|
||||
@Transient override val version: EventVersion = EventVersion(0),
|
||||
val name: String
|
||||
) : BaseDomainEvent(aggregateId, "TestUpdated", version)
|
||||
) : BaseDomainEvent(aggregateId, EventType("TestUpdated"), version)
|
||||
}
|
||||
|
||||
+13
-12
@@ -2,6 +2,7 @@ package at.mocode.infrastructure.eventstore.redis
|
||||
|
||||
import at.mocode.core.domain.event.BaseDomainEvent
|
||||
import at.mocode.core.domain.event.DomainEvent
|
||||
import at.mocode.core.domain.model.*
|
||||
import at.mocode.infrastructure.eventstore.api.EventSerializer
|
||||
import at.mocode.infrastructure.eventstore.api.EventStore
|
||||
import com.benasher44.uuid.Uuid
|
||||
@@ -78,8 +79,8 @@ class RedisIntegrationTest {
|
||||
@Test
|
||||
fun `event publishing and consuming should be fast and reliable`() {
|
||||
val aggregateId = uuid4()
|
||||
val event1 = TestCreatedEvent(aggregateId, 1L, "Test Entity")
|
||||
val event2 = TestUpdatedEvent(aggregateId, 2L, "Updated Test Entity")
|
||||
val event1 = TestCreatedEvent(AggregateId(aggregateId), EventVersion(1L), "Test Entity")
|
||||
val event2 = TestUpdatedEvent(AggregateId(aggregateId), EventVersion(2L), "Updated Test Entity")
|
||||
|
||||
val receivedEvents = mutableListOf<DomainEvent>()
|
||||
eventConsumer.registerEventHandler("TestCreated") { receivedEvents.add(it) }
|
||||
@@ -91,26 +92,26 @@ class RedisIntegrationTest {
|
||||
|
||||
assertEquals(2, receivedEvents.size)
|
||||
|
||||
val receivedEvent1 = receivedEvents.find { it.version == 1L } as TestCreatedEvent
|
||||
assertEquals(aggregateId, receivedEvent1.aggregateId)
|
||||
val receivedEvent1 = receivedEvents.find { it.version == EventVersion(1L) } as TestCreatedEvent
|
||||
assertEquals(AggregateId(aggregateId), receivedEvent1.aggregateId)
|
||||
assertEquals("Test Entity", receivedEvent1.name)
|
||||
|
||||
val receivedEvent2 = receivedEvents.find { it.version == 2L } as TestUpdatedEvent
|
||||
assertEquals(aggregateId, receivedEvent2.aggregateId)
|
||||
val receivedEvent2 = receivedEvents.find { it.version == EventVersion(2L) } as TestUpdatedEvent
|
||||
assertEquals(AggregateId(aggregateId), receivedEvent2.aggregateId)
|
||||
assertEquals("Updated Test Entity", receivedEvent2.name)
|
||||
}
|
||||
|
||||
@Serializable
|
||||
data class TestCreatedEvent(
|
||||
@Transient override val aggregateId: Uuid = uuid4(),
|
||||
@Transient override val version: Long = 0,
|
||||
@Transient override val aggregateId: AggregateId = AggregateId(uuid4()),
|
||||
@Transient override val version: EventVersion = EventVersion(0),
|
||||
val name: String
|
||||
) : BaseDomainEvent(aggregateId, "TestCreated", version)
|
||||
) : BaseDomainEvent(aggregateId, EventType("TestCreated"), version)
|
||||
|
||||
@Serializable
|
||||
data class TestUpdatedEvent(
|
||||
@Transient override val aggregateId: Uuid = uuid4(),
|
||||
@Transient override val version: Long = 0,
|
||||
@Transient override val aggregateId: AggregateId = AggregateId(uuid4()),
|
||||
@Transient override val version: EventVersion = EventVersion(0),
|
||||
val name: String
|
||||
) : BaseDomainEvent(aggregateId, "TestUpdated", version)
|
||||
) : BaseDomainEvent(aggregateId, EventType("TestUpdated"), version)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
FROM openjdk:17-jre-slim
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the gateway JAR file
|
||||
COPY infrastructure/gateway/build/libs/*.jar app.jar
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8080
|
||||
|
||||
# Add health check
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=30s --retries=3 \
|
||||
CMD curl -f http://localhost:8080/actuator/health || exit 1
|
||||
|
||||
# Run the application
|
||||
ENTRYPOINT ["java", "-jar", "app.jar"]
|
||||
@@ -6,7 +6,18 @@ server:
|
||||
spring:
|
||||
application:
|
||||
name: api-gateway
|
||||
security:
|
||||
user:
|
||||
name: admin
|
||||
password: admin
|
||||
cloud:
|
||||
consul:
|
||||
host: localhost
|
||||
port: 8500
|
||||
discovery:
|
||||
register: true
|
||||
health-check-path: /actuator/health
|
||||
health-check-interval: 10s
|
||||
gateway:
|
||||
# HTTP Client-Timeouts für stabile Upstream-Verbindungen
|
||||
httpclient:
|
||||
@@ -22,9 +33,17 @@ spring:
|
||||
# Antwort-Header bereinigen (verhindert doppelte CORS-Header)
|
||||
default-filters:
|
||||
- DedupeResponseHeader=Access-Control-Allow-Credentials Access-Control-Allow-Origin
|
||||
# Aktiviert die automatische Routen-Erstellung basierend auf Consul
|
||||
discovery:
|
||||
locator:
|
||||
enabled: true
|
||||
# Macht Routen-Namen klein (z.B. /members-service/** statt /MEMBERS-SERVICE/**)
|
||||
lower-case-service-id: true
|
||||
# Route definitions with service discovery
|
||||
routes:
|
||||
- id: ping-service-route
|
||||
uri: lb://ping-service
|
||||
predicates:
|
||||
- Path=/api/ping/**
|
||||
filters:
|
||||
- StripPrefix=1
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info
|
||||
|
||||
+3
-1
@@ -71,7 +71,9 @@ class GatewayApplicationTests {
|
||||
class TestRoutes {
|
||||
@Bean
|
||||
fun routeLocator(builder: RouteLocatorBuilder): RouteLocator = builder.routes()
|
||||
.route("test-forward") { r -> r.path("/hello").uri("forward:/internal/hello") }
|
||||
.route("test-forward") {
|
||||
it.path("/hello").uri("forward:/internal/hello")
|
||||
}
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
+89
-21
@@ -1,5 +1,6 @@
|
||||
package at.mocode.infrastructure.messaging.client
|
||||
|
||||
import at.mocode.infrastructure.messaging.config.KafkaConfig
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.kafka.support.serializer.JsonDeserializer
|
||||
@@ -7,42 +8,109 @@ import org.springframework.stereotype.Component
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.kafka.receiver.KafkaReceiver
|
||||
import reactor.kafka.receiver.ReceiverOptions
|
||||
import reactor.util.retry.Retry
|
||||
import java.time.Duration
|
||||
import java.util.Collections
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* A reactive, non-blocking Kafka implementation of the EventConsumer interface.
|
||||
* A reactive, non-blocking Kafka implementation of the EventConsumer interface
|
||||
* with optimized connection pooling, security, and error handling.
|
||||
*/
|
||||
@Component
|
||||
class KafkaEventConsumer(
|
||||
// Wir injizieren die Basis-Konfigurationseigenschaften aus messaging-config
|
||||
private val consumerConfig: Map<String, Any>
|
||||
private val kafkaConfig: KafkaConfig
|
||||
) : EventConsumer {
|
||||
|
||||
private val logger = LoggerFactory.getLogger(KafkaEventConsumer::class.java)
|
||||
|
||||
override fun <T : Any> receiveEvents(topic: String, eventType: Class<T>): Flux<T> {
|
||||
// Für jeden Aufruf wird eine neue, spezifische Konfiguration für diesen Topic erstellt.
|
||||
val receiverOptions = ReceiverOptions.create<String, T>(consumerConfig)
|
||||
.subscription(Collections.singleton(topic))
|
||||
.withValueDeserializer(JsonDeserializer(eventType).trustedPackages("*"))
|
||||
.addAssignListener { partitions ->
|
||||
logger.info("Partitions assigned for topic '{}': {}", topic, partitions)
|
||||
}
|
||||
.addRevokeListener { partitions ->
|
||||
logger.warn("Partitions revoked for topic '{}': {}", topic, partitions)
|
||||
}
|
||||
// Connection pool to reuse KafkaReceiver instances per topic-eventType combination
|
||||
private val receiverCache = ConcurrentHashMap<String, KafkaReceiver<String, Any>>()
|
||||
|
||||
return KafkaReceiver.create(receiverOptions)
|
||||
.receive()
|
||||
override fun <T : Any> receiveEvents(topic: String, eventType: Class<T>): Flux<T> {
|
||||
logger.info("Setting up reactive consumer for topic '{}' with event type '{}'", topic, eventType.simpleName)
|
||||
|
||||
val cacheKey = "${topic}-${eventType.name}"
|
||||
|
||||
// Get or create a cached receiver for this topic-eventType combination
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
val receiver = receiverCache.computeIfAbsent(cacheKey) {
|
||||
createOptimizedReceiver<T>(topic, eventType) as KafkaReceiver<String, Any>
|
||||
} as KafkaReceiver<String, T>
|
||||
|
||||
return receiver.receive()
|
||||
.doOnNext { record ->
|
||||
logger.debug(
|
||||
"Received message from topic-partition {}-{} with offset {}",
|
||||
record.topic(), record.partition(), record.offset()
|
||||
"Received message from topic-partition {}-{} with offset {} for event type '{}'",
|
||||
record.topic(), record.partition(), record.offset(), eventType.simpleName
|
||||
)
|
||||
}
|
||||
.map { it.value() } // Extrahiere nur die deserialisierte Nachricht
|
||||
.doOnError { exception ->
|
||||
logger.error("Error receiving events from topic '{}'", topic, exception)
|
||||
.map { record ->
|
||||
// Manual commit acknowledgment for better control
|
||||
record.receiverOffset().acknowledge()
|
||||
record.value()
|
||||
}
|
||||
.doOnError { exception ->
|
||||
logger.error("Error receiving events from topic '{}' for event type '{}'",
|
||||
topic, eventType.simpleName, exception)
|
||||
}
|
||||
.retryWhen(
|
||||
Retry.backoff(3, Duration.ofSeconds(1))
|
||||
.maxBackoff(Duration.ofSeconds(10))
|
||||
.doBeforeRetry { retrySignal ->
|
||||
logger.warn("Retrying consumer for topic '{}', attempt: {}, error: {}",
|
||||
topic, retrySignal.totalRetries() + 1, retrySignal.failure().message)
|
||||
}
|
||||
.onRetryExhaustedThrow { _, retrySignal ->
|
||||
logger.error("Consumer retry exhausted for topic '{}' after {} attempts",
|
||||
topic, retrySignal.totalRetries())
|
||||
retrySignal.failure()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an optimized KafkaReceiver with secure configuration and performance tuning.
|
||||
*/
|
||||
private fun <T : Any> createOptimizedReceiver(topic: String, eventType: Class<T>): KafkaReceiver<String, T> {
|
||||
// Generate unique group ID for this consumer instance
|
||||
val groupId = "${kafkaConfig.defaultGroupIdPrefix}-${topic}-${eventType.simpleName.lowercase()}"
|
||||
val consumerConfig = kafkaConfig.consumerConfigs(groupId)
|
||||
|
||||
// Create type-safe JSON deserializer with restricted trusted packages
|
||||
val jsonDeserializer = JsonDeserializer(eventType).apply {
|
||||
// Use restricted trusted packages instead of wildcard for security
|
||||
addTrustedPackages(kafkaConfig.trustedPackages)
|
||||
setUseTypeHeaders(false)
|
||||
}
|
||||
|
||||
val receiverOptions = ReceiverOptions.create<String, T>(consumerConfig)
|
||||
.subscription(Collections.singleton(topic))
|
||||
.withValueDeserializer(jsonDeserializer)
|
||||
.addAssignListener { partitions ->
|
||||
logger.info("Consumer '{}' assigned partitions for topic '{}': {}",
|
||||
groupId, topic, partitions.map { "${it.topicPartition().topic()}-${it.topicPartition().partition()}" })
|
||||
}
|
||||
.addRevokeListener { partitions ->
|
||||
logger.warn("Consumer '{}' revoked partitions for topic '{}': {}",
|
||||
groupId, topic, partitions.map { "${it.topicPartition().topic()}-${it.topicPartition().partition()}" })
|
||||
}
|
||||
// Enable commit interval for manual acknowledgment control
|
||||
.commitInterval(Duration.ofSeconds(5))
|
||||
.commitBatchSize(100)
|
||||
|
||||
return KafkaReceiver.create(receiverOptions)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup method to clear cached receivers on application shutdown.
|
||||
* Reactive receivers will be automatically cleaned up when their streams complete.
|
||||
*/
|
||||
@jakarta.annotation.PreDestroy
|
||||
fun cleanup() {
|
||||
logger.info("Cleaning up Kafka consumer cache...")
|
||||
val cacheSize = receiverCache.size
|
||||
receiverCache.clear()
|
||||
logger.info("Kafka consumer cleanup completed. Cleared {} cached receivers", cacheSize)
|
||||
}
|
||||
}
|
||||
|
||||
+92
-13
@@ -5,42 +5,121 @@ import org.springframework.kafka.core.reactive.ReactiveKafkaProducerTemplate
|
||||
import org.springframework.stereotype.Component
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.Mono
|
||||
import reactor.util.retry.Retry
|
||||
import java.time.Duration
|
||||
|
||||
/**
|
||||
* A reactive, non-blocking Kafka implementation of EventPublisher.
|
||||
* A reactive, non-blocking Kafka implementation of EventPublisher with enhanced
|
||||
* error handling, retry mechanisms, and optimized batch processing.
|
||||
*/
|
||||
@Component
|
||||
class KafkaEventPublisher(
|
||||
// KORREKTUR: Verwendung des reaktiven Templates
|
||||
private val reactiveKafkaTemplate: ReactiveKafkaProducerTemplate<String, Any>
|
||||
) : EventPublisher {
|
||||
|
||||
private val logger = LoggerFactory.getLogger(KafkaEventPublisher::class.java)
|
||||
|
||||
companion object {
|
||||
private const val DEFAULT_RETRY_ATTEMPTS = 3L
|
||||
private const val DEFAULT_RETRY_DELAY_SECONDS = 1L
|
||||
private const val DEFAULT_MAX_BACKOFF_SECONDS = 10L
|
||||
private const val DEFAULT_BATCH_CONCURRENCY = 10
|
||||
}
|
||||
|
||||
override fun publishEvent(topic: String, key: String?, event: Any): Mono<Void> {
|
||||
logger.debug("Publishing event to topic '{}' with key '{}'", topic, key)
|
||||
logger.debug("Publishing event to topic '{}' with key '{}', event type: '{}'",
|
||||
topic, key, event::class.simpleName)
|
||||
|
||||
return reactiveKafkaTemplate.send(topic, key, event)
|
||||
.doOnSuccess { result ->
|
||||
val record = result.recordMetadata()
|
||||
logger.info(
|
||||
"Successfully published event to topic-partition {}-{} with offset {}",
|
||||
record.topic(), record.partition(), record.offset()
|
||||
logger.debug(
|
||||
"Successfully published event to topic-partition {}-{} with offset {} (key: '{}')",
|
||||
record.topic(), record.partition(), record.offset(), key
|
||||
)
|
||||
}
|
||||
.doOnError { exception ->
|
||||
logger.error("Failed to publish event to topic '{}' with key '{}'", topic, key, exception)
|
||||
logger.warn("Failed to publish event to topic '{}' with key '{}' - will retry if configured",
|
||||
topic, key, exception)
|
||||
}
|
||||
.then() // Wandelt das Ergebnis in ein Mono<Void> um
|
||||
.retryWhen(createRetrySpec(topic, key))
|
||||
.doOnError { exception ->
|
||||
logger.error("Final failure after retries: Failed to publish event to topic '{}' with key '{}'",
|
||||
topic, key, exception)
|
||||
}
|
||||
.then()
|
||||
}
|
||||
|
||||
override fun publishEvents(topic: String, events: List<Pair<String?, Any>>): Flux<Void> {
|
||||
logger.debug("Publishing {} events to topic '{}'", events.size, topic)
|
||||
// Verwendet Flux.fromIterable, um eine Sequenz von Sende-Operationen zu erstellen
|
||||
if (events.isEmpty()) {
|
||||
logger.debug("No events to publish to topic '{}'", topic)
|
||||
return Flux.empty()
|
||||
}
|
||||
|
||||
logger.info("Publishing {} events to topic '{}' using optimized batch processing", events.size, topic)
|
||||
|
||||
return Flux.fromIterable(events)
|
||||
// .flatMap stellt sicher, dass die Sende-Operationen parallelisiert,
|
||||
// aber dennoch reaktiv (nicht-blockierend) ausgeführt werden.
|
||||
.flatMap { (key, event) ->
|
||||
.index() // Add index for progress tracking
|
||||
.flatMap({ indexedEventPair ->
|
||||
val index = indexedEventPair.t1
|
||||
val eventPair = indexedEventPair.t2
|
||||
val (key, event) = eventPair
|
||||
publishEvent(topic, key, event)
|
||||
.doOnSuccess {
|
||||
if ((index + 1) % 100 == 0L || index == events.size.toLong() - 1) {
|
||||
logger.info("Batch progress: {}/{} events published to topic '{}'",
|
||||
index + 1, events.size, topic)
|
||||
}
|
||||
}
|
||||
.onErrorContinue { error, _ ->
|
||||
logger.error("Error publishing event {} in batch to topic '{}': {}",
|
||||
index + 1, topic, error.message)
|
||||
}
|
||||
}, DEFAULT_BATCH_CONCURRENCY) // Controlled concurrency for better resource management
|
||||
.doOnComplete {
|
||||
logger.info("Completed publishing batch of {} events to topic '{}'", events.size, topic)
|
||||
}
|
||||
.doOnError { error ->
|
||||
logger.error("Batch publishing to topic '{}' failed with error: {}", topic, error.message)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a retry specification with exponential backoff for robust error handling.
|
||||
*/
|
||||
private fun createRetrySpec(topic: String, key: String?): Retry =
|
||||
Retry.backoff(DEFAULT_RETRY_ATTEMPTS, Duration.ofSeconds(DEFAULT_RETRY_DELAY_SECONDS))
|
||||
.maxBackoff(Duration.ofSeconds(DEFAULT_MAX_BACKOFF_SECONDS))
|
||||
.filter { exception ->
|
||||
// Only retry on transient errors (not serialization errors, etc.)
|
||||
isRetryableException(exception)
|
||||
}
|
||||
.doBeforeRetry { retrySignal ->
|
||||
logger.info("Retrying publish to topic '{}' with key '{}', attempt: {}, error: {}",
|
||||
topic, key, retrySignal.totalRetries() + 1,
|
||||
retrySignal.failure().message?.take(100))
|
||||
}
|
||||
.onRetryExhaustedThrow { _, retrySignal ->
|
||||
logger.error("Retry exhausted for topic '{}' with key '{}' after {} attempts",
|
||||
topic, key, retrySignal.totalRetries())
|
||||
retrySignal.failure()
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if an exception is retryable based on its type and characteristics.
|
||||
*/
|
||||
private fun isRetryableException(exception: Throwable): Boolean {
|
||||
return when {
|
||||
exception.message?.contains("timeout", ignoreCase = true) == true -> true
|
||||
exception.message?.contains("connection", ignoreCase = true) == true -> true
|
||||
exception.message?.contains("network", ignoreCase = true) == true -> true
|
||||
exception is java.util.concurrent.TimeoutException -> true
|
||||
exception is java.net.ConnectException -> true
|
||||
exception is java.io.IOException -> true
|
||||
// Don't retry serialization errors or authentication failures
|
||||
exception.message?.contains("serializ", ignoreCase = true) == true -> false
|
||||
exception.message?.contains("auth", ignoreCase = true) == true -> false
|
||||
else -> true // Default to retryable for unknown exceptions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+43
-8
@@ -1,22 +1,57 @@
|
||||
package at.mocode.infrastructure.messaging.client
|
||||
|
||||
import at.mocode.infrastructure.messaging.config.KafkaConfig
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory
|
||||
import org.springframework.kafka.core.reactive.ReactiveKafkaProducerTemplate
|
||||
import reactor.kafka.sender.SenderOptions
|
||||
import java.time.Duration
|
||||
|
||||
/**
|
||||
* Reactive Kafka configuration utilities for creating a ReactiveKafkaProducerTemplate.
|
||||
* Spring Configuration for reactive Kafka components with optimized settings.
|
||||
*/
|
||||
class ReactiveKafkaConfig {
|
||||
@Configuration
|
||||
class ReactiveKafkaConfig(
|
||||
private val kafkaConfig: KafkaConfig
|
||||
) {
|
||||
|
||||
private val logger = LoggerFactory.getLogger(ReactiveKafkaConfig::class.java)
|
||||
|
||||
/**
|
||||
* Create a ReactiveKafkaProducerTemplate using the configuration from the given ProducerFactory.
|
||||
* Creates a Spring Bean for the optimized ReactiveKafkaProducerTemplate.
|
||||
* This template includes enhanced error handling, monitoring, and performance tuning.
|
||||
*/
|
||||
fun reactiveKafkaProducerTemplate(
|
||||
producerFactory: DefaultKafkaProducerFactory<String, Any>
|
||||
): ReactiveKafkaProducerTemplate<String, Any> {
|
||||
@Bean
|
||||
fun reactiveKafkaProducerTemplate(): ReactiveKafkaProducerTemplate<String, Any> {
|
||||
logger.info("Creating optimized ReactiveKafkaProducerTemplate with enhanced configuration")
|
||||
|
||||
val producerFactory = kafkaConfig.producerFactory()
|
||||
val props: Map<String, Any> = producerFactory.configurationProperties
|
||||
val senderOptions: SenderOptions<String, Any> = SenderOptions.create(props)
|
||||
return ReactiveKafkaProducerTemplate(senderOptions)
|
||||
|
||||
val senderOptions = SenderOptions.create<String, Any>(props)
|
||||
// Enhanced sender options for better performance and reliability
|
||||
.maxInFlight(1024) // Increase in-flight requests for better throughput
|
||||
.scheduler(reactor.core.scheduler.Schedulers.boundedElastic()) // Use bounded elastic scheduler
|
||||
.closeTimeout(Duration.ofSeconds(30)) // Give enough time for graceful shutdown
|
||||
.stopOnError(false) // Continue processing even if some messages fail
|
||||
|
||||
return ReactiveKafkaProducerTemplate(senderOptions).apply {
|
||||
// Configure additional properties if needed
|
||||
logger.info("ReactiveKafkaProducerTemplate configured successfully with bootstrap servers: {}",
|
||||
kafkaConfig.bootstrapServers)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a KafkaConfig bean if not already provided.
|
||||
* This allows for external configuration override while providing sensible defaults.
|
||||
*/
|
||||
@Bean
|
||||
fun kafkaConfig(): KafkaConfig {
|
||||
return KafkaConfig().apply {
|
||||
logger.info("Initializing KafkaConfig with bootstrap servers: {}", bootstrapServers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+12
-13
@@ -38,8 +38,8 @@ class KafkaIntegrationTest {
|
||||
}
|
||||
producerFactory = kafkaConfig.producerFactory()
|
||||
|
||||
val reactiveKafkaConfig = ReactiveKafkaConfig()
|
||||
val reactiveTemplate = reactiveKafkaConfig.reactiveKafkaProducerTemplate(producerFactory)
|
||||
val reactiveKafkaConfig = ReactiveKafkaConfig(kafkaConfig)
|
||||
val reactiveTemplate = reactiveKafkaConfig.reactiveKafkaProducerTemplate()
|
||||
kafkaEventPublisher = KafkaEventPublisher(reactiveTemplate)
|
||||
}
|
||||
|
||||
@@ -54,19 +54,18 @@ class KafkaIntegrationTest {
|
||||
val testKey = "test-key"
|
||||
val testEvent = TestEvent("Test Message")
|
||||
|
||||
val consumerProps = mapOf(
|
||||
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG to kafkaContainer.bootstrapServers,
|
||||
ConsumerConfig.GROUP_ID_CONFIG to "test-group-${UUID.randomUUID()}",
|
||||
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG to StringDeserializer::class.java,
|
||||
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG to JsonDeserializer::class.java,
|
||||
ConsumerConfig.AUTO_OFFSET_RESET_CONFIG to "earliest",
|
||||
JsonDeserializer.TRUSTED_PACKAGES to "*",
|
||||
JsonDeserializer.USE_TYPE_INFO_HEADERS to false,
|
||||
JsonDeserializer.VALUE_DEFAULT_TYPE to TestEvent::class.java.name
|
||||
)
|
||||
// Use the same KafkaConfig for consistent and secure configuration
|
||||
val testKafkaConfig = KafkaConfig().apply {
|
||||
bootstrapServers = kafkaContainer.bootstrapServers
|
||||
// For tests, we need to trust the test package
|
||||
trustedPackages = "at.mocode.*"
|
||||
}
|
||||
|
||||
val consumerProps = testKafkaConfig.consumerConfigs("test-group-${UUID.randomUUID()}")
|
||||
|
||||
val jsonValueDeserializer = JsonDeserializer(TestEvent::class.java).apply {
|
||||
addTrustedPackages("*")
|
||||
addTrustedPackages(testKafkaConfig.trustedPackages)
|
||||
setUseTypeHeaders(false)
|
||||
}
|
||||
val receiverOptions = ReceiverOptions.create<String, TestEvent>(consumerProps)
|
||||
.withKeyDeserializer(StringDeserializer())
|
||||
|
||||
+65
-3
@@ -1,13 +1,16 @@
|
||||
package at.mocode.infrastructure.messaging.config
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig
|
||||
import org.apache.kafka.clients.producer.ProducerConfig
|
||||
import org.apache.kafka.common.serialization.StringDeserializer
|
||||
import org.apache.kafka.common.serialization.StringSerializer
|
||||
import org.springframework.kafka.core.DefaultKafkaProducerFactory
|
||||
import org.springframework.kafka.core.ProducerFactory
|
||||
import org.springframework.kafka.support.serializer.JsonDeserializer
|
||||
import org.springframework.kafka.support.serializer.JsonSerializer
|
||||
|
||||
/**
|
||||
* Central Kafka producer configuration used across modules.
|
||||
* Central Kafka configuration used across modules with optimized settings for performance and reliability.
|
||||
*
|
||||
* This class can be instantiated programmatically (as done in tests) or
|
||||
* registered as a Spring @Configuration with @Bean methods in an application context.
|
||||
@@ -20,14 +23,73 @@ class KafkaConfig {
|
||||
var bootstrapServers: String = "localhost:9092"
|
||||
|
||||
/**
|
||||
* Common producer properties with sensible defaults (String keys, JSON values).
|
||||
* Default consumer group ID prefix.
|
||||
*/
|
||||
var defaultGroupIdPrefix: String = "messaging-client"
|
||||
|
||||
/**
|
||||
* Comma-separated list of trusted packages for JSON deserialization security.
|
||||
* Default restricts to application packages only.
|
||||
*/
|
||||
var trustedPackages: String = "at.mocode.*"
|
||||
|
||||
/**
|
||||
* Optimized producer properties with performance tuning and reliability settings.
|
||||
*/
|
||||
fun producerConfigs(): Map<String, Any> = mapOf(
|
||||
ProducerConfig.BOOTSTRAP_SERVERS_CONFIG to bootstrapServers,
|
||||
ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG to StringSerializer::class.java,
|
||||
ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG to JsonSerializer::class.java,
|
||||
// Avoid adding type info headers; keeps payloads simple and interoperable.
|
||||
JsonSerializer.ADD_TYPE_INFO_HEADERS to false
|
||||
JsonSerializer.ADD_TYPE_INFO_HEADERS to false,
|
||||
|
||||
// Performance optimizations
|
||||
ProducerConfig.BATCH_SIZE_CONFIG to 32768, // 32KB batch size for better throughput
|
||||
ProducerConfig.LINGER_MS_CONFIG to 5, // Wait up to 5ms to batch messages
|
||||
ProducerConfig.COMPRESSION_TYPE_CONFIG to "snappy", // Fast compression
|
||||
ProducerConfig.BUFFER_MEMORY_CONFIG to 67108864, // 64MB buffer memory
|
||||
|
||||
// Reliability settings
|
||||
ProducerConfig.ACKS_CONFIG to "all", // Wait for all replicas
|
||||
ProducerConfig.RETRIES_CONFIG to 3, // Retry failed sends
|
||||
ProducerConfig.RETRY_BACKOFF_MS_CONFIG to 1000, // 1 second retry backoff
|
||||
ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG to 30000, // 30 second delivery timeout
|
||||
ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG to 10000, // 10 second request timeout
|
||||
|
||||
// Idempotence for exactly-once semantics
|
||||
ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG to true,
|
||||
ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION to 5
|
||||
)
|
||||
|
||||
/**
|
||||
* Optimized consumer properties with performance tuning and reliability settings.
|
||||
*/
|
||||
fun consumerConfigs(groupId: String? = null): Map<String, Any> = mapOf(
|
||||
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG to bootstrapServers,
|
||||
ConsumerConfig.GROUP_ID_CONFIG to (groupId ?: "${defaultGroupIdPrefix}-${System.currentTimeMillis()}"),
|
||||
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG to StringDeserializer::class.java,
|
||||
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG to JsonDeserializer::class.java,
|
||||
|
||||
// JSON deserialization security
|
||||
JsonDeserializer.TRUSTED_PACKAGES to trustedPackages,
|
||||
JsonDeserializer.USE_TYPE_INFO_HEADERS to false,
|
||||
|
||||
// Performance optimizations
|
||||
ConsumerConfig.FETCH_MIN_BYTES_CONFIG to 1024, // 1KB minimum fetch size
|
||||
ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG to 500, // Max 500ms wait for fetch
|
||||
ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG to 1048576, // 1MB max partition fetch
|
||||
ConsumerConfig.MAX_POLL_RECORDS_CONFIG to 500, // Process up to 500 records per poll
|
||||
|
||||
// Reliability settings
|
||||
ConsumerConfig.AUTO_OFFSET_RESET_CONFIG to "earliest",
|
||||
ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG to false, // Manual commit for better control
|
||||
ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG to 30000, // 30 second session timeout
|
||||
ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG to 3000, // 3 second heartbeat
|
||||
|
||||
// Connection settings
|
||||
ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG to 540000, // 9 minutes idle timeout
|
||||
ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG to 50,
|
||||
ConsumerConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG to 1000
|
||||
)
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user