spring boot kafka generic JSON templateSender - json

Question

I am working with kafka and spring boot and I need to send JSON object to kafka, the point is that I am able to send an object as JSON configuring KafkaTemplate but just for this object.
package com.bankia.apimanager.config;
import com.bankia.apimanager.model.RequestDTO;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;
import org.springframework.kafka.support.serializer.JsonSerializer;
import java.util.HashMap;
import java.util.Map;
#Configuration
public class KafkaConfiguration {
#Value("${spring.kafka.bootstrap-servers}")
private String bootstrapServers;
#Bean
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
return props;
}
#Bean
public ProducerFactory<String, RequestDTO> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
#Bean
public KafkaTemplate<String, RequestDTO> kafkaTemplate() {
return new KafkaTemplate<>(producerFactory());
}
}
package com.bankia.apimanager.controller;
import com.bankia.apimanager.model.RequestDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
#RestController
#RequestMapping("/infrastructure")
public class InfraStructureRequestController {
private final static Logger LOG = LoggerFactory.getLogger( InfraStructureRequestController.class );
private static final String TOPIC = "test";
#Autowired
private KafkaTemplate<String, RequestDTO> sender;
#RequestMapping(value = "/test", method = RequestMethod.GET)
public String postMessage(){
ListenableFuture<SendResult<String, RequestDTO>> future = sender.send(TOPIC, new RequestDTO("Hola","Paco"));
future.addCallback(new ListenableFutureCallback<SendResult<String, RequestDTO>>() {
#Override
public void onSuccess(SendResult<String, RequestDTO> result) {
LOG.info("Sent message with offset=[" + result.getRecordMetadata().offset() + "]");
}
#Override
public void onFailure(Throwable ex) {
LOG.error("Unable to send message due to : " + ex.getMessage());
}
});
return "OK";
}
}
but what about if now I want to send a new DTO object? do I have to declare a new KafkaTemplate<String,NEWOBJECT> and autowire each kafka template declared in configuration for each object? there is another way to be able to just declare one kafkaTemplate in which I can send any type of object and automatically will be serialized in JSON?

Answer 1

I think, you can specify a generic KafkaTemplate<String, Object> and set the producer value serializer to JsonSerializer like this:
#Configuration
public class KafkaConfiguration {
#Value("${spring.kafka.bootstrap-servers}")
private String bootstrapServers;
#Bean
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
return props;
}
#Bean
public ProducerFactory<String, Object> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
#Bean
public KafkaTemplate<String, Object> kafkaTemplate() {
return new KafkaTemplate<>(producerFactory());
}
}

Answer 2

Referring your code:
Value Serializer is correctly defined as JsonSerializer, which will convert objects of any type to JSON.
#Bean
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
return props;
}
Change <String, RequestDTO> to <String, Object> at every place in KafkaConfig & Controller.
Keep in mind that generics remain until compile time (type erasure)
only.

Answer 3

There are two scenario:
Scenario #1
If you want to use KafkaTemplate to send any type(as mentioned in your question) to kafka, so there is no need to declare your own KafkaTemplate bean because Spring boot did this for you in KafkaAutoConfiguration.
package org.springframework.boot.autoconfigure.kafka;
...
#Configuration(proxyBeanMethods = false)
#ConditionalOnClass(KafkaTemplate.class)
#EnableConfigurationProperties(KafkaProperties.class)
#Import({ KafkaAnnotationDrivenConfiguration.class, KafkaStreamsAnnotationDrivenConfiguration.class })
public class KafkaAutoConfiguration {
private final KafkaProperties properties;
public KafkaAutoConfiguration(KafkaProperties properties) {
this.properties = properties;
}
#Bean
#ConditionalOnMissingBean(KafkaTemplate.class)
public KafkaTemplate<?, ?> kafkaTemplate(ProducerFactory<Object, Object> kafkaProducerFactory,
ProducerListener<Object, Object> kafkaProducerListener,
ObjectProvider<RecordMessageConverter> messageConverter) {
KafkaTemplate<Object, Object> kafkaTemplate = new KafkaTemplate<>(kafkaProducerFactory);
messageConverter.ifUnique(kafkaTemplate::setMessageConverter);
kafkaTemplate.setProducerListener(kafkaProducerListener);
kafkaTemplate.setDefaultTopic(this.properties.getTemplate().getDefaultTopic());
return kafkaTemplate;
}
}
**Some Note**:
This config class has been annotated with #ConditionalOnClass(KafkaTemplate.class) that means: (from spring docs--->) #Conditional that only matches when the specified classes are on the classpath.
kafkaTemplate bean method is annotated with
#ConditionalOnMissingBean(KafkaTemplate.class) that means: (from spring docs ---->) #Conditional that only matches when no beans meeting the specified requirements are already contained in the BeanFactory.
Important! In pure java world, KafkaTemplate<?, ?> is not subtype of for example: KafkaTemplate<String, RequestDTO> so you can't to do this:
KafkaTemplate<?, ?> kf1 = ...;
KafkaTemplate<String, RequestDTO> kf2 = kf1; // Compile time error
because java parameterized types are invariant as mentioned in Effective Java third edition item 31. But is spring world that is ok and will be injected to your own service. You need only to specify your own generic type on your kafkaTemplate properties.
For example:
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
#Service
public class KafkaService {
#Autowired
private KafkaTemplate<Integer, String> kafkaTemplate1;
#Autowired
private KafkaTemplate<Integer, RequestDTO> KafkaTemplate2;
}
Scenario #2
If you need to restrict value type of kafka record then you need to specify your own kafka bean something like this:
#Configuration(proxyBeanMethods = false)
#ConditionalOnClass(KafkaTemplate.class)
#EnableConfigurationProperties(CorridorTracingConfiguration.class)
public class CorridorKafkaAutoConfiguration {
#Bean
#ConditionalOnMissingBean(KafkaTemplate.class)
public KafkaTemplate<?, AbstractMessage> kafkaTemplate(ProducerFactory<Object, AbstractMessage> kafkaProducerFactory,
ProducerListener<Object, AbstractMessage> kafkaProducerListener,
ObjectProvider<RecordMessageConverter> messageConverter) {
KafkaTemplate<Object, AbstractMessage> kafkaTemplate = new KafkaTemplate<>(kafkaProducerFactory);
messageConverter.ifUnique(kafkaTemplate::setMessageConverter);
kafkaTemplate.setProducerListener(kafkaProducerListener);
kafkaTemplate.setDefaultTopic(this.properties.getTemplate().getDefaultTopic());
return kafkaTemplate;
}
Now this can be injected only to
KafkaTemplate<String, AbstractMessage> kafkaTemplate, the key type can be anything else instead of String. But you can send any sub type of AbstractMessage to kafka via it.
An example usage:
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
#Service
public class KafkaService {
#Autowired
private KafkaTemplate<String, AbstractMessage> kafkaTemplate;
public void makeTrx(TrxRequest trxRequest) {
kafkaTemplate.send("fraud-request", trxRequest.fromAccountNumber(), new FraudRequest(trxRequest));
}
}
#Accessors(chain = true)
#Getter
#Setter
#EqualsAndHashCode(callSuper = true)
#ToString(callSuper = true)
public class FraudRequest extends AbstractMessage {
private float amount;
private String fromAccountNumber;
private String toAccountNumber;
...
}
To restrict the key of kafka message follow the same (above) way