Reading Very Complex JSON using Spring Batch - json

Question

My objective is to read a very complex JSON using Spring Batch. Below is the sample JSON.
{
"order-info" : {
"order-number" : "Test-Order-1"
"order-items" : [
{
"item-id" : "4144769310"
"categories" : [
"ABCD",
"DEF"
],
"item_imag" : "http:// "
"attributes: {
"color" : "red"
},
"dimensions" : {
},
"vendor" : "abcd",
},
{
"item-id" : "88888",
"categories" : [
"ABCD",
"DEF"
],
.......
I understand that I would need to create a Custom ItemReader to parse this JSON.
Kindly provide me some pointers. I am really clueless.
I am now not using CustomItemReader. I am using Java POJOs. My JsonItemReader is as per below:
#Bean
public JsonItemReader<Trade> jsonItemReader() {
ObjectMapper objectMapper = new ObjectMapper();
JacksonJsonObjectReader<Trade> jsonObjectReader =
new JacksonJsonObjectReader<>(Trade.class);
jsonObjectReader.setMapper(objectMapper);
return new JsonItemReaderBuilder<Trade>()
.jsonObjectReader(jsonObjectReader)
.resource(new ClassPathResource("search_data_1.json"))
.name("tradeJsonItemReader")
.build();
}
The exception which I now get is :
java.lang.IllegalStateException: The Json input stream must start with an array of Json objects
From similar posts in this forum I understand that I need to use JsonObjectReader. "You can implement it to read a single json object and use it with the JsonItemReader (either at construction time or using the setter)".
How can I do this either # construction time or using setter? Please share some code snippet for the same.
The delegate of MultiResourceItemReader should still be a JsonItemReader. You just need to use a custom JsonObjectReader with the JsonItemReader instead of JacksonJsonObjectReader. Visually, this would be: MultiResourceItemReader -- delegates to --> JsonItemReader -- uses --> your custom JsonObjectReader.
Could you please share a code snippet for the above?

Answer 1

JacksonJsonItemReader is meant to parse from a root node that is already and array node, so it expects your json to start with '['.
If you desire to parse a complex object - in this case, one that have many parent nodes/properties before it gets to the array - you should write a reader. It is really simple to do it and you can follow JacksonJsonObjectReader's structure. Here follows and example of a generic reader for complex object with respective unit tests.
The unit test
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.springframework.core.io.ByteArrayResource;
import com.example.batch_experiment.dataset.Dataset;
import com.example.batch_experiment.dataset.GenericJsonObjectReader;
import com.example.batch_experiment.json.InvalidArrayNodeException;
import com.example.batch_experiment.json.UnreachableNodeException;
import com.fasterxml.jackson.databind.ObjectMapper;
#RunWith(BlockJUnit4ClassRunner.class)
public class GenericJsonObjectReaderTest {
GenericJsonObjectReader<Dataset> reader;
#Before
public void setUp() {
reader = new GenericJsonObjectReader<Dataset>(Dataset.class, "results");
}
#Test
public void shouldRead_ResultAsRootNode() throws Exception {
reader.open(new ByteArrayResource("{\"result\":{\"results\":[{\"id\":\"a\"}]}}".getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
}
#Test
public void shouldIgnoreUnknownProperty() throws Exception {
String jsonStr = "{\"result\":{\"results\":[{\"id\":\"a\", \"aDifferrentProperty\":0}]}}";
reader.open(new ByteArrayResource(jsonStr.getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
}
#Test
public void shouldIgnoreNullWithoutQuotes() throws Exception {
String jsonStr = "{\"result\":{\"results\":[{\"id\":\"a\",\"name\":null}]}}";
try {
reader.open(new ByteArrayResource(jsonStr.getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
#Test
public void shouldThrowException_OnNullNode() throws Exception {
boolean exceptionThrown = false;
try {
reader.open(new ByteArrayResource("{}".getBytes()) {});
} catch (UnreachableNodeException e) {
exceptionThrown = true;
}
Assert.assertTrue(exceptionThrown);
}
#Test
public void shouldThrowException_OnNotArrayNode() throws Exception {
boolean exceptionThrown = false;
try {
reader.open(new ByteArrayResource("{\"result\":{\"results\":{}}}".getBytes()) {});
} catch (InvalidArrayNodeException e) {
exceptionThrown = true;
}
Assert.assertTrue(exceptionThrown);
}
#Test
public void shouldReadObjectValue() {
try {
reader.setJsonParser(new ObjectMapper().createParser("{\"id\":\"a\"}"));
Dataset dataset = reader.read();
Assert.assertNotNull(dataset);
Assert.assertEquals("a", dataset.getId());
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
}
And the reader:
import java.io.IOException;
import java.io.InputStream;
import java.util.logging.Logger;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.json.JsonObjectReader;
import org.springframework.core.io.Resource;
import com.example.batch_experiment.json.InvalidArrayNodeException;
import com.example.batch_experiment.json.UnreachableNodeException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
/*
* This class follows the structure and functions similar to JacksonJsonObjectReader, with
* the difference that it expects a object as root node, instead of an array.
*/
public class GenericJsonObjectReader<T> implements JsonObjectReader<T>{
Logger logger = Logger.getLogger(GenericJsonObjectReader.class.getName());
ObjectMapper mapper = new ObjectMapper();
private JsonParser jsonParser;
private InputStream inputStream;
private ArrayNode targetNode;
private Class<T> targetType;
private String targetPath;
public GenericJsonObjectReader(Class<T> targetType, String targetPath) {
super();
this.targetType = targetType;
this.targetPath = targetPath;
}
public JsonParser getJsonParser() {
return jsonParser;
}
public void setJsonParser(JsonParser jsonParser) {
this.jsonParser = jsonParser;
}
public ArrayNode getDatasetNode() {
return targetNode;
}
/*
* JsonObjectReader interface has an empty default method and must be implemented in this case to set
* the mapper and the parser
*/
#Override
public void open(Resource resource) throws Exception {
logger.info("Opening json object reader");
this.inputStream = resource.getInputStream();
JsonNode jsonNode = this.mapper.readTree(this.inputStream).findPath(targetPath);
if (!jsonNode.isMissingNode()) {
this.jsonParser = startArrayParser(jsonNode);
logger.info("Reader open with parser reference: " + this.jsonParser);
this.targetNode = (ArrayNode) jsonNode; // for testing purposes
} else {
logger.severe("Couldn't read target node " + this.targetPath);
throw new UnreachableNodeException();
}
}
#Override
public T read() throws Exception {
try {
if (this.jsonParser.nextToken() == JsonToken.START_OBJECT) {
T result = this.mapper.readValue(this.jsonParser, this.targetType);
logger.info("Object read: " + result.hashCode());
return result;
}
} catch (IOException e) {
throw new ParseException("Unable to read next JSON object", e);
}
return null;
}
/**
* Creates a new parser from an array node
*/
private JsonParser startArrayParser(JsonNode jsonArrayNode) throws IOException {
JsonParser jsonParser = this.mapper.getFactory().createParser(jsonArrayNode.toString());
if (jsonParser.nextToken() == JsonToken.START_ARRAY) {
return jsonParser;
} else {
throw new InvalidArrayNodeException();
}
}
#Override
public void close() throws Exception {
this.inputStream.close();
this.jsonParser.close();
}
}