How to write nested XML to database using spring batch? - mysql

Question

I have an XML file data . and my target to put it in mysql database with a table called forecast containing just four columns (area_id, parameter_id, datetime, and value).
I managed to read the data from all the XML data but couldn't write the data into the database yet. How to write data that has been read in the logging class to the database?
below, there are several classes that I created with a spring batch configuration
Area.java
#XmlRootElement(name="area")
#XmlAccessorType(XmlAccessType.FIELD)
public class Area {
#XmlAttribute(name = "id")
private String area_id;
#XmlElement(name = "parameter")
private List<Parameter> area=null;
public List<Parameter> getArea(){
return area;
}
public void setArea(List<Parameter> area){
this.area = area;
}
public Area(){
}
public Area(String area_id) {
this.area_id = area_id;
}
#Override
public String toString() {
return "Area{" +
"area_id='" + area_id + '\'' +
", area=" + area +
'}';
}
}
TimeRange.java
#XmlRootElement(name="timerange")
#XmlAccessorType(XmlAccessType.FIELD)
public class TimeRange{
#XmlElement(name = "value")
String value;
#XmlAttribute(name = "datetime")
String datetime;
public TimeRange(){
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getDatetime() {
return datetime;
}
public void setDatetime(String datetime) {
this.datetime = datetime;
}
#Override
public String toString() {
final StringBuffer sb = new StringBuffer("TimeRange{");
sb.append("value='").append(value).append('\'');
sb.append(", datetime='").append(datetime).append('\'');
sb.append('}');
return sb.toString();
}
}
Parameter.java
#XmlRootElement(name="parameter")
#XmlAccessorType(XmlAccessType.FIELD)
public class Parameter{
#XmlAttribute(name="id")
private String parameter_id;
#XmlElement(name="timerange")
public List<TimeRange> parameter;
public List<TimeRange> getParameter(){
return parameter;
}
public void setParameter(List<TimeRange> parameter){
this.parameter = parameter;
}
public Parameter(){
}
public Parameter(String parameter_id) {
this.parameter_id = parameter_id;
}
#Override
public String toString() {
final StringBuffer sb = new StringBuffer("Parameter{");
sb.append("parameter_id='").append(parameter_id).append('\'');
sb.append(", parameter=").append(parameter);
sb.append('}');
return sb.toString();
}
}
Config.java
#Configuration
public class SpringBatchExampleJobConfig {
#Bean
public ItemReader<Area> areaItemReader() {
Jaxb2Marshaller areaMarshaller = new Jaxb2Marshaller();
areaMarshaller.setClassesToBeBound(Area.class);
return new StaxEventItemReaderBuilder<Area>()
.name("areaReader")
.resource(new ClassPathResource("data/DigitalForecast-SulawesiTengah.xml"))
.addFragmentRootElements("area")
.unmarshaller(areaMarshaller)
.build();
}
#Bean
public ItemWriter<Area> areaItemWriter(){
return new LoggingAreaItemWriter();
}
/**
* Creates a bean that represents the only step of our batch job.
* #param areaItemReader
* #param step1BuilderFactory
* #param areaItemWriter
* #return
*/
#Bean
public Step Step1(ItemReader<Area> areaItemReader,
ItemWriter<Area> areaItemWriter,
StepBuilderFactory step1BuilderFactory) {
return step1BuilderFactory.get("Step1")
.<Area, Area>chunk(1)
.reader(areaItemReader)
.writer(areaItemWriter)
.build();
}
/**
* Creates a bean that represents our example batch job.
* #param Step1
* #param jobBuilderFactory
* #return
*/
#Bean
public Job areaJob(Step Step1,
JobBuilderFactory jobBuilderFactory) {
return jobBuilderFactory.get("areaJob")
.incrementer(new RunIdIncrementer())
.start(Step1)
.build();
}
}
and the logging class I use to write readable data results
LoggingAreaItemWriter.java
public class LoggingAreaItemWriter implements ItemWriter<Area> {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggingAreaItemWriter.class);
#Override
public void write(List<? extends Area> list) throws Exception {
LOGGER.info("Writing area: {}", list);
}
}
and the result
Writing area: [Area{area_id='501520', area=[Parameter{parameter_id='hu', parameter=[TimeRange{value='95', datetime='202102010000'}, TimeRange{value='80', datetime='202102010600'}, TimeRange{value='90', datetime='202102011200'}, TimeRange{value='95', datetime='202102011800'}, TimeRange{value='85', datetime='202102020000'}, TimeRange{value='60', datetime='202102020600'}, TimeRange{value='90', datetime='202102021200'}, TimeRange{value='90', datetime='202102021800'}, TimeRange{value='85', datetime='202102030000'}, TimeRange{value='55', datetime='202102030600'}, TimeRange{value='95', datetime='202102031200'}, TimeRange{value='95', datetime='202102031800'}]}, Parameter{parameter_id='humax', parameter=[TimeRange{value='95', datetime='202102011200'}, TimeRange{value='95', datetime='202102021200'}, TimeRange{value='95', datetime='202102031200'}]}, Parameter{parameter_id='tmax', parameter=[TimeRange{value='91.4', datetime='202102011200'}, TimeRange{value='91.4', datetime='202102021200'}, TimeRange{value='91.4', datetime='202102031200'}]}, Parameter{parameter_id='humin', parameter=[TimeRange{value='80', datetime='202102011200'}, TimeRange{value='60', datetime='202102021200'}, TimeRange{value='55', datetime='202102031200'}]}, Parameter{parameter_id='tmin', parameter=[TimeRange{value='75.2', datetime='202102011200'}, TimeRange{value='73.4', datetime='202102021200'}, TimeRange{value='73.4', datetime='202102031200'}]}, Parameter{parameter_id='t', parameter=[TimeRange{value='75.2', datetime='202102010000'}, TimeRange{value='91.4', datetime='202102010600'}, TimeRange{value='75.2', datetime='202102011200'}, TimeRange{value='73.4', datetime='202102011800'}, TimeRange{value='77', datetime='202102020000'}, TimeRange{value='91.4', datetime='202102020600'}, TimeRange{value='77', datetime='202102021200'}, TimeRange{value='75.2', datetime='202102021800'}, TimeRange{value='78.8', datetime='202102030000'}, TimeRange{value='91.4', datetime='202102030600'}, TimeRange{value='77', datetime='202102031200'}, TimeRange{value='73.4', datetime='202102031800'}]}, Parameter{parameter_id='weather', parameter=[TimeRange{value='1', datetime='202102010000'}, TimeRange{value='1', datetime='202102010600'}, TimeRange{value='1', datetime='202102011200'}, TimeRange{value='3', datetime='202102011800'}, TimeRange{value='1', datetime='202102020000'}, TimeRange{value='1', datetime='202102020600'}, TimeRange{value='1', datetime='202102021200'}, TimeRange{value='3', datetime='202102021800'}, TimeRange{value='3', datetime='202102030000'}, TimeRange{value='1', datetime='202102030600'}, TimeRange{value='0', datetime='202102031200'}, TimeRange{value='3', datetime='202102031800'}]}, Parameter{parameter_id='wd', parameter=[TimeRange{value='29230', datetime='202102010000'}, TimeRange{value='4500', datetime='202102010600'}, TimeRange{value='2230', datetime='202102011200'}, TimeRange{value='22500', datetime='202102011800'}, TimeRange{value='22500', datetime='202102020000'}, TimeRange{value='4500', datetime='202102020600'}, TimeRange{value='15730', datetime='202102021200'}, TimeRange{value='22500', datetime='202102021800'}, TimeRange{value='27000', datetime='202102030000'}, TimeRange{value='000', datetime='202102030600'}, TimeRange{value='000', datetime='202102031200'}, TimeRange{value='27000', datetime='202102031800'}]}, Parameter{parameter_id='ws', parameter=[TimeRange{value='1.028888888', datetime='202102010000'}, TimeRange{value='10.28888888', datetime='202102010600'}, TimeRange{value='1.028888888', datetime='202102011200'}, TimeRange{value='1.028888888', datetime='202102011800'}, TimeRange{value='1.028888888', datetime='202102020000'}, TimeRange{value='10.28888888', datetime='202102020600'}, TimeRange{value='1.028888888', datetime='202102021200'}, TimeRange{value='1.028888888', datetime='202102021800'}, TimeRange{value='1.028888888', datetime='202102030000'}, TimeRange{value='0', datetime='202102030600'}, TimeRange{value='0', datetime='202102031200'}, TimeRange{value='2.57222222', datetime='202102031800'}]}]}]

Answer 1

Please add a JDBC writer instead of logging writer like below.
#Bean
public JdbcBatchItemWriter<UnifiedInvoiceDTO> summaryWriter() {
JdbcBatchItemWriter<UnifiedInvoiceDTO> databaseItemWriter = new JdbcBatchItemWriter<>();
databaseItemWriter.setDataSource(dataSource);
databaseItemWriter.setSql("YOUR SQL");
ItemPreparedStatementSetter<UnifiedInvoiceDTO> invoicePreparedStatementSetter = new InvoiceSummarySetter();
databaseItemWriter.setItemPreparedStatementSetter(invoicePreparedStatementSetter);
return databaseItemWriter;
}
public class InvoiceSummarySetter implements ItemPreparedStatementSetter<UnifiedInvoiceDTO> {
Logger logger = LoggerFactory.getLogger(InvoiceSummarySetter.class);
#Override
public void setValues(UnifiedInvoiceDTO invoiceDTO, PreparedStatement preparedStatement) throws SQLException {
preparedStatement.setString(1, invoiceDTO.getTypeInvInfo().getAccountNumber());
}

Answer 2

Your XML input has a hierarchical format:
<area id="1">
<parameter id="11">
<timerange datetime="202102220000">
<value>85</value>
</timerange>
</>
<parameter id="12">
<timerange datetime="202102220600">
<value>95</value>
</timerange>
</>
</area>
<area id="2">
<parameter id="21">
<timerange datetime="202102220000">
<value>85</value>
</timerange>
</>
<parameter id="22">
<timerange datetime="202102220600">
<value>95</value>
</timerange>
</>
</area>
while your output has a flat format area_id,parameter_id,datetime,value:
1,11,202102220000,85
1,12,202102220600,95
2,21,202102220000,85
2,22,202102220600,95
So a single input XML item should be written as multiple database records. In this case, you need to "flatmap" your items before writing them to the database. This can be done with an item processor. See the following posts for more details on how to do that:
Save object with List as csv file FlatFileItemWriter
Flatten processing result in spring batch