Keep tags order using SnakeYAML - json

I'm trying to translate yaml files to json, but the translation re-orders the tags...
Ex, YAML source:
zzzz:
b: 456
a: dfff
aa:
s10: "dddz"
s3: eeee
bbb:
- b1
- a2
snakeYAML produces:
{
"aa": {
"s3": "eeee",
"s10":"dddz"
},
"bbb":[
"b1",
"a2"
],
"zzzz": {
"a": "dfff",
"b":456
}
}

Create following class in your code, this is a tweaked version from the SnakeYAML source that uses LinkedHashMap and LinkedHashSet that keep the insertion order instead of TreeMap and TreeSet which auto-sort them.
import java.beans.FeatureDescriptor;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.*;
import org.yaml.snakeyaml.error.YAMLException;
import org.yaml.snakeyaml.introspector.*;
import org.yaml.snakeyaml.util.PlatformFeatureDetector;
public class CustomPropertyUtils extends PropertyUtils {
private final Map<Class<?>, Map<String, Property>> propertiesCache = new HashMap<Class<?>, Map<String, Property>>();
private final Map<Class<?>, Set<Property>> readableProperties = new HashMap<Class<?>, Set<Property>>();
private BeanAccess beanAccess = BeanAccess.DEFAULT;
private boolean allowReadOnlyProperties = false;
private boolean skipMissingProperties = false;
private PlatformFeatureDetector platformFeatureDetector;
public CustomPropertyUtils() {
this(new PlatformFeatureDetector());
}
CustomPropertyUtils(PlatformFeatureDetector platformFeatureDetector) {
this.platformFeatureDetector = platformFeatureDetector;
/*
* Android lacks much of java.beans (including the Introspector class, used here), because java.beans classes tend to rely on java.awt, which isn't
* supported in the Android SDK. That means we have to fall back on FIELD access only when SnakeYAML is running on the Android Runtime.
*/
if (platformFeatureDetector.isRunningOnAndroid()) {
beanAccess = BeanAccess.FIELD;
}
}
protected Map<String, Property> getPropertiesMap(Class<?> type, BeanAccess bAccess) {
if (propertiesCache.containsKey(type)) {
return propertiesCache.get(type);
}
Map<String, Property> properties = new LinkedHashMap<String, Property>();
boolean inaccessableFieldsExist = false;
switch (bAccess) {
case FIELD:
for (Class<?> c = type; c != null; c = c.getSuperclass()) {
for (Field field : c.getDeclaredFields()) {
int modifiers = field.getModifiers();
if (!Modifier.isStatic(modifiers) && !Modifier.isTransient(modifiers)
&& !properties.containsKey(field.getName())) {
properties.put(field.getName(), new FieldProperty(field));
}
}
}
break;
default:
// add JavaBean properties
try {
for (PropertyDescriptor property : Introspector.getBeanInfo(type)
.getPropertyDescriptors()) {
Method readMethod = property.getReadMethod();
if ((readMethod == null || !readMethod.getName().equals("getClass"))
&& !isTransient(property)) {
properties.put(property.getName(), new MethodProperty(property));
}
}
} catch (IntrospectionException e) {
throw new YAMLException(e);
}
// add public fields
for (Class<?> c = type; c != null; c = c.getSuperclass()) {
for (Field field : c.getDeclaredFields()) {
int modifiers = field.getModifiers();
if (!Modifier.isStatic(modifiers) && !Modifier.isTransient(modifiers)) {
if (Modifier.isPublic(modifiers)) {
properties.put(field.getName(), new FieldProperty(field));
} else {
inaccessableFieldsExist = true;
}
}
}
}
break;
}
if (properties.isEmpty() && inaccessableFieldsExist) {
throw new YAMLException("No JavaBean properties found in " + type.getName());
}
System.out.println(properties);
propertiesCache.put(type, properties);
return properties;
}
private static final String TRANSIENT = "transient";
private boolean isTransient(FeatureDescriptor fd) {
return Boolean.TRUE.equals(fd.getValue(TRANSIENT));
}
public Set<Property> getProperties(Class<? extends Object> type) {
return getProperties(type, beanAccess);
}
public Set<Property> getProperties(Class<? extends Object> type, BeanAccess bAccess) {
if (readableProperties.containsKey(type)) {
return readableProperties.get(type);
}
Set<Property> properties = createPropertySet(type, bAccess);
readableProperties.put(type, properties);
return properties;
}
protected Set<Property> createPropertySet(Class<? extends Object> type, BeanAccess bAccess) {
Set<Property> properties = new LinkedHashSet<>();
Collection<Property> props = getPropertiesMap(type, bAccess).values();
for (Property property : props) {
if (property.isReadable() && (allowReadOnlyProperties || property.isWritable())) {
properties.add(property);
}
}
return properties;
}
public Property getProperty(Class<? extends Object> type, String name) {
return getProperty(type, name, beanAccess);
}
public Property getProperty(Class<? extends Object> type, String name, BeanAccess bAccess) {
Map<String, Property> properties = getPropertiesMap(type, bAccess);
Property property = properties.get(name);
if (property == null && skipMissingProperties) {
property = new MissingProperty(name);
}
if (property == null) {
throw new YAMLException(
"Unable to find property '" + name + "' on class: " + type.getName());
}
return property;
}
public void setBeanAccess(BeanAccess beanAccess) {
if (platformFeatureDetector.isRunningOnAndroid() && beanAccess != BeanAccess.FIELD) {
throw new IllegalArgumentException(
"JVM is Android - only BeanAccess.FIELD is available");
}
if (this.beanAccess != beanAccess) {
this.beanAccess = beanAccess;
propertiesCache.clear();
readableProperties.clear();
}
}
public void setAllowReadOnlyProperties(boolean allowReadOnlyProperties) {
if (this.allowReadOnlyProperties != allowReadOnlyProperties) {
this.allowReadOnlyProperties = allowReadOnlyProperties;
readableProperties.clear();
}
}
public boolean isAllowReadOnlyProperties() {
return allowReadOnlyProperties;
}
/**
* Skip properties that are missing during deserialization of YAML to a Java
* object. The default is false.
*
* #param skipMissingProperties
* true if missing properties should be skipped, false otherwise.
*/
public void setSkipMissingProperties(boolean skipMissingProperties) {
if (this.skipMissingProperties != skipMissingProperties) {
this.skipMissingProperties = skipMissingProperties;
readableProperties.clear();
}
}
public boolean isSkipMissingProperties() {
return skipMissingProperties;
}
}
Then, create your Yaml instance like this:
DumperOptions options = new DumperOptions();
CustomPropertyUtils customPropertyUtils = new CustomPropertyUtils();
Representer customRepresenter = new Representer();
customRepresenter.setPropertyUtils(customPropertyUtils);
Yaml yaml = new Yaml(customRepresenter, options);
Profit!

The keeping of property order depends on java implementation and is not guaranteed.
In order to control the yaml generation you will need to implement your CustomRepresenter overriding the getProperties , see the example below:
package io.github.rockitconsulting.test.rockitizer.configuration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.introspector.Property;
import org.yaml.snakeyaml.representer.Representer;
/**
* Custom implementation of {#link Representer} and {#link Comparator}
* to keep the needed order of javabean properties of model classes,
* thus generating the understandable yaml
*
*/
public class ConfigurationModelRepresenter extends Representer {
public ConfigurationModelRepresenter() {
super();
}
public ConfigurationModelRepresenter(DumperOptions options) {
super(options);
}
protected Set<Property> getProperties(Class<? extends Object> type) {
Set<Property> propertySet;
if (typeDefinitions.containsKey(type)) {
propertySet = typeDefinitions.get(type).getProperties();
}
propertySet = getPropertyUtils().getProperties(type);
List<Property> propsList = new ArrayList<>(propertySet);
Collections.sort(propsList, new BeanPropertyComparator());
return new LinkedHashSet<>(propsList);
}
class BeanPropertyComparator implements Comparator<Property> {
public int compare(Property p1, Property p2) {
if (p1.getType().getCanonicalName().contains("util") && !p2.getType().getCanonicalName().contains("util")) {
return 1;
} else if(p2.getName().endsWith("Name")|| p2.getName().equalsIgnoreCase("name")) {
return 1;
} else {
return -1;
} // returning 0 would merge keys
}
}
}
The below snippet shows the usage of the newly created class to generate the yaml structure:
DumperOptions options = new DumperOptions();
ConfigurationModelRepresenter customRepresenter = new ConfigurationModelRepresenter();
Yaml yaml = new Yaml(customRepresenter, options);
StringWriter writer = new StringWriter();
yaml.dump(suite, writer);
FileWriter fw = new FileWriter(rootPathTestSrc + "config_gen.yml", false);
fw.write(writer.toString());
fw.close();
This approach is much cleaner, comparing to the one suggested above.

Related

MySQL JSON_EXTRACT value of property based on criteria

Suppose a JSON column called blob in a MySQL 5.7 database table called "Thing" with the following content:
[
{
"id": 1,
"value": "blue"
},
{
"id": 2,
"value": "red"
}
]
Is it possible to select all records from Thing where the blob contains an object within the array where the id is some dynamic value and the value is also some dynamic value.
E.g. "give me all Things where the blob contains an object whose id is 2 and value is 'red'"
Not sure how to form the WHERE clause below:
SET #id = 2;
SET #value1 = 'red';
SET #value2 = 'blue';
-- with equals?
SELECT *
FROM Thing
WHERE JSON_EXTRACT(blob, '$[*].id ... equals #id ... and .value') = #value1;
-- with an IN clause?
SELECT *
FROM Thing
WHERE JSON_EXTRACT(blob, '$[*].id ... equals #id ... and .value') IN (#value1, #value2);
They said it couldn't be done. They told me I was a fool. Lo and behold, I have done it!
Here are some helper functions to teach Hibernate how to perform JSON functions against a MySQL 5.7 backend, and an example use case. It's confusing, for sure, but it works.
Context
This contrived example is of a Person entity which can have many BioDetails, which is a question/answer type (but is more involved than that). The example within below essentially is searching within two JSON payloads, grabbing JSON values from one to build JSON paths within which to search in the other. In the end, you can pass in a complex structure of AND'd or OR'd criteria, which will be applied against a JSON blob and return only the resulting rows which match.
E.g. give my all Person entities where their age is > 30 and their favorite color is blue or orange. Given that those key/value pairs are stored in a JSON blob, you can find those matched using the example code below.
JSON-search classes and example repo
Classes below use Lombok for brevity.
Classes to allow specification of search criteria
SearchCriteriaContainer
#Data
#EqualsAndHashCode
public class SearchCriteriaContainer
{
private List<SearchCriterion> criteria;
private boolean and;
}
SearchCriterion
#Data
#EqualsAndHashCode(callSuper = true)
public class SearchCriterion extends SearchCriteriaContainer
{
private String field;
private List<String> values;
private SearchOperator operator;
private boolean not = false;
}
SearchOperator
#RequiredArgsConstructor
public enum SearchOperator
{
EQUAL("="),
LESS_THAN("<"),
LESS_THAN_OR_EQUAL("<="),
GREATER_THAN(">"),
GREATER_THAN_OR_EQUAL(">="),
LIKE("like"),
IN("in"),
IS_NULL("is null");
private final String value;
#JsonCreator
public static SearchOperator fromValue(#NotBlank String value)
{
return Stream
.of(SearchOperator.values())
.filter(o -> o.getValue().equals(value))
.findFirst()
.orElseThrow(() ->
{
String message = String.format("Could not find %s with value: %s", SearchOperator.class.getName(), value);
return new IllegalArgumentException(message);
});
}
#JsonValue
public String getValue()
{
return this.value;
}
#Override
public String toString()
{
return value;
}
}
Helper class which is used to call JSON functions
#RequiredArgsConstructor
public class CriteriaBuilderHelper
{
private final CriteriaBuilder criteriaBuilder;
public Expression<String> concat(Expression<?>... values)
{
return criteriaBuilder.function("CONCAT", String.class, values);
}
public Expression<String> substringIndex(Expression<?> value, String delimiter, int count)
{
return substringIndex(value, criteriaBuilder.literal(delimiter), criteriaBuilder.literal(count));
}
public Expression<String> substringIndex(Expression<?> value, Expression<String> delimiter, Expression<Integer> count)
{
return criteriaBuilder.function("SUBSTRING_INDEX", String.class, value, delimiter, count);
}
public Expression<String> jsonUnquote(Expression<?> jsonValue)
{
return criteriaBuilder.function("JSON_UNQUOTE", String.class, jsonValue);
}
public Expression<String> jsonExtract(Expression<?> jsonDoc, Expression<?> path)
{
return criteriaBuilder.function("JSON_EXTRACT", String.class, jsonDoc, path);
}
public Expression<String> jsonSearchOne(Expression<?> jsonDoc, Expression<?> value, Expression<?>... paths)
{
return jsonSearch(jsonDoc, "one", value, paths);
}
public Expression<String> jsonSearch(Expression<?> jsonDoc, Expression<?> value, Expression<?>... paths)
{
return jsonSearch(jsonDoc, "all", value, paths);
}
public Expression<String> jsonSearch(Expression<?> jsonDoc, String oneOrAll, Expression<?> value, Expression<?>... paths)
{
if (!"one".equals(oneOrAll) && !"all".equals(oneOrAll))
{
throw new RuntimeException("Parameter 'oneOrAll' must be 'one' or 'all', not: " + oneOrAll);
}
else
{
final var expressions = new ArrayList<>(List.of(
jsonDoc,
criteriaBuilder.literal(oneOrAll),
value,
criteriaBuilder.nullLiteral(String.class)));
if (paths != null)
{
expressions.addAll(Arrays.asList(paths));
}
return criteriaBuilder.function("JSON_SEARCH", String.class, expressions.toArray(Expression[]::new));
}
}
}
Utility to turn SearchCriteria into MySQL JSON function calls
SearchHelper
public class SearchHelper
{
private static final Pattern pathSeparatorPattern = Pattern.compile("\\.");
public static String getKeyPart(String key)
{
return pathSeparatorPattern.split(key)[0];
}
public static String getPathPart(String key)
{
final var parts = pathSeparatorPattern.split(key);
final var path = new StringBuilder();
for (var i = 1; i < parts.length; i++)
{
if (i > 1)
{
path.append(".");
}
path.append(parts[i]);
}
return path.toString();
}
public static Optional<Predicate> getCriteriaPredicate(SearchCriteriaContainer container, CriteriaBuilder cb, Path<String> bioDetailJson, Path<String> personJson)
{
final var predicates = new ArrayList<Predicate>();
if (container != null && container.getCriteria() != null && container.getCriteria().size() > 0)
{
final var h = new CriteriaBuilderHelper(cb);
container.getCriteria().forEach(ac ->
{
final var groupingOnly = ac.getField() == null && ac.getOperator() == null;
// a criterion can be used for grouping other criterion, and might not have a field/operator/value
if (!groupingOnly)
{
final var key = getKeyPart(ac.getField());
final var path = getPathPart(ac.getField());
final var bioDetailQuestionKeyPathEx = h.jsonUnquote(h.jsonSearchOne(bioDetailJson, cb.literal(key), cb.literal("$[*].key")));
final var bioDetailQuestionIdPathEx = h.concat(h.substringIndex(bioDetailQuestionKeyPathEx, ".", 1), cb.literal(".id"));
final var questionIdEx = h.jsonUnquote(h.jsonExtract(bioDetailJson, bioDetailQuestionIdPathEx));
final var answerPathEx = h.substringIndex(h.jsonUnquote(h.jsonSearchOne(personJson, questionIdEx, cb.literal("$[*].questionId"))), ".", 1);
final var answerValuePathEx = h.concat(answerPathEx, cb.literal("." + path));
final var answerValueEx = h.jsonUnquote(h.jsonExtract(personJson, answerValuePathEx));
switch (ac.getOperator())
{
case IN:
{
final var inEx = cb.in(answerValueEx);
if (ac.getValues() == null || ac.getValues().size() == 0)
{
throw new RuntimeException("No values provided for 'IN' criteria for field: " + ac.getField());
}
else
{
ac.getValues().forEach(inEx::value);
}
predicates.add(inEx);
break;
}
case IS_NULL:
{
predicates.add(cb.isNull(answerValueEx));
break;
}
default:
{
if (ac.getValues() == null || ac.getValues().size() == 0)
{
throw new RuntimeException("No values provided for '" + ac.getOperator() + "' criteria for field: " + ac.getField());
}
else
{
ac.getValues().forEach(value ->
{
final var valueEx = cb.literal(value);
switch (ac.getOperator())
{
case EQUAL:
{
predicates.add(cb.equal(answerValueEx, valueEx));
break;
}
case LESS_THAN:
{
predicates.add(cb.lessThan(answerValueEx, valueEx));
break;
}
case LESS_THAN_OR_EQUAL:
{
predicates.add(cb.lessThanOrEqualTo(answerValueEx, valueEx));
break;
}
case GREATER_THAN:
{
predicates.add(cb.greaterThan(answerValueEx, valueEx));
break;
}
case GREATER_THAN_OR_EQUAL:
{
predicates.add(cb.greaterThanOrEqualTo(answerValueEx, valueEx));
break;
}
case LIKE:
{
predicates.add(cb.like(answerValueEx, valueEx));
break;
}
default:
throw new RuntimeException("Unsupported operator during snapshot search: " + ac.getOperator());
}
});
}
}
}
}
// iterate nested criteria
getAnswerCriteriaPredicate(ac, cb, bioDetailJson, personJson).ifPresent(predicates::add);
});
return Optional.of(container.isAnd()
? cb.and(predicates.toArray(Predicate[]::new))
: cb.or(predicates.toArray(Predicate[]::new)));
}
else
{
return Optional.empty();
}
}
}
Example JPA Specification repository / search method
ExampleRepository
#Repository
public interface PersonRepository extends JpaSpecificationExecutor<Person>
{
default Page<Person> search(PersonSearchDirective directive, Pageable pageable)
{
return findAll((person, query, cb) ->
{
final var bioDetail = person.join(Person_.bioDetail);
final var bioDetailJson = bioDetail.get(BioDetailEntity_.bioDetailJson);
final var personJson = person.get(Person_.personJson);
final var predicates = new ArrayList<>();
SearchHelper
.getCriteriaPredicate(directive.getSearchCriteria(), cb, bioDetailJson, personJson)
.ifPresent(predicates::add);
return cb.and(predicates.toArray(Predicate[]::new));
}, pageable);
}
}

mybatis clientGenerated couldn't call back

i code a plugin for mybatis generator, but in clientGenerated of extends plugin method doesn't work.
please helpe me ~ ^_^
code is in under:
public class MapperAnnotationPlugin extends PluginAdapter {
private final static Map<String, String> ANNOTATION_IMPORTS;
static {
ANNOTATION_IMPORTS = new HashMap<>();
ANNOTATION_IMPORTS.put("#Mapper", "org.apache.ibatis.annotations.Mapper");
ANNOTATION_IMPORTS.put("#Repository", "org.springframework.stereotype.Repository");
}
private List<String> annotationList;
#Override
public void initialized(IntrospectedTable introspectedTable) {
super.initialized(introspectedTable);
this.annotationList = new ArrayList<>();
Properties properties = this.getProperties();
boolean findMapper = false;
for (Object key : properties.keySet()) {
String keyStr = key.toString().trim();
if (keyStr.startsWith("#Mapper")) {
findMapper = true;
}
if (StringUtility.isTrue(properties.getProperty(key.toString()))) {
annotationList.add(keyStr);
}
}
if (!findMapper) {
annotationList.add(0, "#Mapper");
}
}
#Override
public boolean clientGenerated(Interface interfaze, IntrospectedTable introspectedTable) {
super.clientGenerated(interfaze, introspectedTable);
for (String annotation : annotationList) {
if ("#Mapper".equals(annotation)) {
if (introspectedTable.getTargetRuntime() == IntrospectedTable.TargetRuntime.MYBATIS3) {
interfaze.addImportedType(new FullyQualifiedJavaType(ANNOTATION_IMPORTS.get(annotation)));
interfaze.addAnnotation(annotation);
}
} else if (Objects.nonNull(ANNOTATION_IMPORTS.get(annotation))) {
logger.info(PluginConst.TEACHEE_PLUGIN + "添加" + annotation);
interfaze.addImportedType(new FullyQualifiedJavaType(ANNOTATION_IMPORTS.get(annotation)));
interfaze.addAnnotation(annotation);
}
}
return true;
}
}
in second method, in debug, it had not go this method, so what could i do in next step
Same problem when I maked a plugin for mybatis-generator-plugin. The method of clientGenerated didnot be callback. It is my way, call from another method which include Interface object.
#Override
public boolean clientCountByExampleMethodGenerated(Method method, Interface interfaze, IntrospectedTable introspectedTable) {
this.clientGenerated(interfaze, null, introspectedTable);
return false;
}
Both generator-core/generator-plugin version is 1.4.0, work fine now.
https://github.com/mybatis/generator/releases/tag/mybatis-generator-1.4.0

write regex in JsonFormat pattern

#JsonFormat(shape = JsonFormat.Shape.STRING, pattern ="yyyy-MM-dd'T'HH:mm:ss.SSS")
is it possible to write regex in pattern? I could not
pattern ="yyyy-MM-dd'T'HH:mm:ss.SSS(Z?)"
I want to make Z as optional
any links suggestions?
I ended up creating custom deserializer based on LocalDateDeserializer.INSTANCE and moved the regex there.
After registering the deserializer the object mapper as a custom module the #JsonFormat annotation is no longer required:
#Bean
public ObjectMapper createObjectMapper() {
return new ObjectMapper()
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
.registerModule(new JavaTimeModule())
.registerModule(new CustomTimeModule());
}
and defined the deserializer in the CustomTimeModule
class CustomTimeModule extends SimpleModule {
public CustomTimeModule() {
super();
addDeserializer(LocalDate.class, CustomLocalDateDeserializer.INSTANCE);
}
}
and finally the regex part, in my case was cutting of the optional non-standard time zone that i was sometimes getting after the date, but could be easily extended to match your case:
public class CustomLocalDateDeserializer extends JSR310DateTimeDeserializerBase<LocalDate> {
private static final long serialVersionUID = 1L;
private static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE;
public static final CustomLocalDateDeserializer INSTANCE = new CustomLocalDateDeserializer();
private CustomLocalDateDeserializer() {
this(DEFAULT_FORMATTER);
}
public CustomLocalDateDeserializer(DateTimeFormatter dtf) {
super(LocalDate.class, dtf);
}
#Override
protected JsonDeserializer<LocalDate> withDateFormat(DateTimeFormatter dtf) {
return new CustomLocalDateDeserializer(dtf);
}
#Override
public LocalDate deserialize(JsonParser parser, DeserializationContext context) throws IOException
{
if (parser.hasToken(JsonToken.VALUE_STRING)) {
String string = parser.getText().trim();
if (string.length() == 0) {
return null;
}
// >>>>>>> regex part comes here <<<<<<<
string = parser.getText().trim().substring(0, 10);
// >>>>>>> regex part comes here <<<<<<<
// as per [datatype-jsr310#37], only check for optional (and, incorrect...) time marker 'T'
// if we are using default formatter
try {
return LocalDate.parse(string, _formatter);
} catch (DateTimeException e) {
return _handleDateTimeException(context, e, string);
}
}
if (parser.isExpectedStartArrayToken()) {
JsonToken t = parser.nextToken();
if (t == JsonToken.END_ARRAY) {
return null;
}
if (context.isEnabled(DeserializationFeature.UNWRAP_SINGLE_VALUE_ARRAYS)
&& (t == JsonToken.VALUE_STRING || t==JsonToken.VALUE_EMBEDDED_OBJECT)) {
final LocalDate parsed = deserialize(parser, context);
if (parser.nextToken() != JsonToken.END_ARRAY) {
handleMissingEndArrayForSingle(parser, context);
}
return parsed;
}
if (t == JsonToken.VALUE_NUMBER_INT) {
int year = parser.getIntValue();
int month = parser.nextIntValue(-1);
int day = parser.nextIntValue(-1);
if (parser.nextToken() != JsonToken.END_ARRAY) {
throw context.wrongTokenException(parser, handledType(), JsonToken.END_ARRAY,
"Expected array to end");
}
return LocalDate.of(year, month, day);
}
context.reportInputMismatch(handledType(),
"Unexpected token (%s) within Array, expected VALUE_NUMBER_INT",
t);
}
if (parser.hasToken(JsonToken.VALUE_EMBEDDED_OBJECT)) {
return (LocalDate) parser.getEmbeddedObject();
}
// 06-Jan-2018, tatu: Is this actually safe? Do users expect such coercion?
if (parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
return LocalDate.ofEpochDay(parser.getLongValue());
}
return _handleUnexpectedToken(context, parser, "Expected array or string.");
}

How to perform partial object serialization providing "paths" using Newtonsoft JSON.NET

I have a situation in which I have a very large C# object, however, I only need to return a handful of properties (which can be on nested objects), allow for client-side JavaScript to modify those properties and then send the resulting object back to the server in order to perform in-place partial de-serialization.
The idea is to re-use some very large existing business objects, but be intelligent about only serializing and sending only those properties back to the client application for modification (to keep the amount of data transferred at a minimum).
I basically have an XML file where I pre-define all of the bindings using a "path syntax" which would indicate only those properties I need to serialize. So, I could use something like "WorkOrder.UserField1" or "WorkOrder.Client.Name".
I have tried using a custom contract resolver to determine whether or not a property should be serialized; however, it doesn't seem that I have information as to the "path" (in other words, other properties in the object model up the chain) in order to determine if the property should or should not be serialized.
I have also tried using a custom JsonTextWriter, but it doesn't seem that I can override the methods necessary to keep track of the path, even though there is a Path property available. Is there something perhaps simple that I am overlooking in order to be able to view the path hierarchy of a property being serialized and determine if it should be serialized by looking up the path in a table and making the decision?
The basic difficulty here is that Json.NET is a contract-based serializer which creates a contract for each type to be serialized, then (de)serializes according to the contract. If a type appears in multiple locations in the object hierarchy, the same contract applies. But you want to selectively include properties for a given type depending on its location in the hierarchy, which conflicts with the basic "one type one contract" design.
One quick way to work around this is to serialize to a JObject, then use JToken.SelectTokens() to select only the JSON data you want to return, removing everything else. Since SelectTokens has full support for JSONPath query syntax, you can selectively include using array and property wildcards or other filters, for instance:
"$.FirstLevel[*].Bar"
includes all properties named "Bar" in all array members of a property named "FirstLevel" of the root object.
This should reduce your network usage as desired, but won't save any processing time on the server.
Removal can be accomplished with the following extension methods:
public static partial class JsonExtensions
{
public static TJToken RemoveAllExcept<TJToken>(this TJToken obj, IEnumerable<string> paths) where TJToken : JToken
{
if (obj == null || paths == null)
throw new NullReferenceException();
var keepers = new HashSet<JToken>(paths.SelectMany(path => obj.SelectTokens(path)), ObjectReferenceEqualityComparer<JToken>.Default);
var keepersAndParents = new HashSet<JToken>(keepers.SelectMany(t => t.AncestorsAndSelf()), ObjectReferenceEqualityComparer<JToken>.Default);
// Keep any token that is a keeper, or a child of a keeper, or a parent of a keeper
// I.e. if you have a path ""$.A.B" and it turns out that B is an object, then everything
// under B should be kept.
foreach (var token in obj.DescendantsAndSelfReversed().Where(t => !keepersAndParents.Contains(t) && !t.AncestorsAndSelf().Any(p => keepers.Contains(p))))
token.RemoveFromLowestPossibleParent();
// Return the object itself for fluent style programming.
return obj;
}
public static string SerializeAndSelectTokens<T>(T root, string[] paths, Formatting formatting = Formatting.None, JsonSerializerSettings settings = null)
{
var obj = JObject.FromObject(root, JsonSerializer.CreateDefault(settings));
obj.RemoveAllExcept(paths);
var json = obj.ToString(formatting);
return json;
}
public static TJToken RemoveFromLowestPossibleParent<TJToken>(this TJToken node) where TJToken : JToken
{
if (node == null)
return null;
JToken toRemove;
var property = node.Parent as JProperty;
if (property != null)
{
// Also detach the node from its immediate containing property -- Remove() does not do this even though it seems like it should
toRemove = property;
property.Value = null;
}
else
{
toRemove = node;
}
if (toRemove.Parent != null)
toRemove.Remove();
return node;
}
public static IEnumerable<JToken> DescendantsAndSelfReversed(this JToken node)
{
if (node == null)
throw new ArgumentNullException();
return RecursiveEnumerableExtensions.Traverse(node, t => ListReversed(t as JContainer));
}
// Iterate backwards through a list without throwing an exception if the list is modified.
static IEnumerable<T> ListReversed<T>(this IList<T> list)
{
if (list == null)
yield break;
for (int i = list.Count - 1; i >= 0; i--)
yield return list[i];
}
}
public static partial class RecursiveEnumerableExtensions
{
// Rewritten from the answer by Eric Lippert https://stackoverflow.com/users/88656/eric-lippert
// to "Efficient graph traversal with LINQ - eliminating recursion" http://stackoverflow.com/questions/10253161/efficient-graph-traversal-with-linq-eliminating-recursion
// to ensure items are returned in the order they are encountered.
public static IEnumerable<T> Traverse<T>(
T root,
Func<T, IEnumerable<T>> children)
{
yield return root;
var stack = new Stack<IEnumerator<T>>();
try
{
stack.Push((children(root) ?? Enumerable.Empty<T>()).GetEnumerator());
while (stack.Count != 0)
{
var enumerator = stack.Peek();
if (!enumerator.MoveNext())
{
stack.Pop();
enumerator.Dispose();
}
else
{
yield return enumerator.Current;
stack.Push((children(enumerator.Current) ?? Enumerable.Empty<T>()).GetEnumerator());
}
}
}
finally
{
foreach (var enumerator in stack)
enumerator.Dispose();
}
}
}
/// <summary>
/// A generic object comparerer that would only use object's reference,
/// ignoring any <see cref="IEquatable{T}"/> or <see cref="object.Equals(object)"/> overrides.
/// </summary>
public class ObjectReferenceEqualityComparer<T> : IEqualityComparer<T> where T : class
{
// Adapted from this answer https://stackoverflow.com/a/1890230
// to https://stackoverflow.com/questions/1890058/iequalitycomparert-that-uses-referenceequals
// By https://stackoverflow.com/users/177275/yurik
private static readonly IEqualityComparer<T> _defaultComparer;
static ObjectReferenceEqualityComparer() { _defaultComparer = new ObjectReferenceEqualityComparer<T>(); }
public static IEqualityComparer<T> Default { get { return _defaultComparer; } }
#region IEqualityComparer<T> Members
public bool Equals(T x, T y)
{
return ReferenceEquals(x, y);
}
public int GetHashCode(T obj)
{
return System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj);
}
#endregion
}
And then use them like:
public class TestClass
{
public static void Test()
{
var root = new RootObject
{
FirstLevel1 = new FirstLevel
{
SecondLevel1 = new List<SecondLevel> { new SecondLevel { A = "a11", B = "b11", Third1 = new ThirdLevel { Foo = "Foos11", Bar = "Bars11" }, Third2 = new List<ThirdLevel> { new ThirdLevel { Foo = "FooList11", Bar = "BarList11" } } } },
SecondLevel2 = new List<SecondLevel> { new SecondLevel { A = "a12", B = "b12", Third1 = new ThirdLevel { Foo = "Foos12", Bar = "Bars12" }, Third2 = new List<ThirdLevel> { new ThirdLevel { Foo = "FooList12", Bar = "BarList12" } } } },
},
FirstLevel2 = new FirstLevel
{
SecondLevel1 = new List<SecondLevel> { new SecondLevel { A = "a21", B = "b21", Third1 = new ThirdLevel { Foo = "Foos21", Bar = "Bars21" }, Third2 = new List<ThirdLevel> { new ThirdLevel { Foo = "FooList21", Bar = "BarList21" } } } },
SecondLevel2 = new List<SecondLevel> { new SecondLevel { A = "a22", B = "b22", Third1 = new ThirdLevel { Foo = "Foos22", Bar = "Bars22" }, Third2 = new List<ThirdLevel> { new ThirdLevel { Foo = "FooList22", Bar = "BarList22" } } } },
}
};
Assert.IsTrue(JObject.FromObject(root).DescendantsAndSelf().OfType<JValue>().Count() == 24); // No assert
var paths1 = new string[]
{
"$.FirstLevel2.SecondLevel1[*].A",
"$.FirstLevel1.SecondLevel2[*].Third2[*].Bar",
};
Test(root, paths1, 2);
var paths3 = new string[]
{
"$.FirstLevel1.SecondLevel2[*].Third2[*].Bar",
};
Test(root, paths3, 1);
var paths4 = new string[]
{
"$.*.SecondLevel2[*].Third2[*].Bar",
};
Test(root, paths4, 2);
}
static void Test<T>(T root, string [] paths, int expectedCount)
{
var json = JsonExtensions.SerializeAndSelectTokens(root, paths, Formatting.Indented);
Console.WriteLine("Result using paths: {0}", JsonConvert.SerializeObject(paths));
Console.WriteLine(json);
Assert.IsTrue(JObject.Parse(json).DescendantsAndSelf().OfType<JValue>().Count() == expectedCount); // No assert
}
}
public class ThirdLevel
{
public string Foo { get; set; }
public string Bar { get; set; }
}
public class SecondLevel
{
public ThirdLevel Third1 { get; set; }
public List<ThirdLevel> Third2 { get; set; }
public string A { get; set; }
public string B { get; set; }
}
public class FirstLevel
{
public List<SecondLevel> SecondLevel1 { get; set; }
public List<SecondLevel> SecondLevel2 { get; set; }
}
public class RootObject
{
public FirstLevel FirstLevel1 { get; set; }
public FirstLevel FirstLevel2 { get; set; }
}
Note that there is an enhancement request Feature request: ADD JsonProperty.ShouldSerialize(object target, string path) #1857 that would enable this sort of functionality more easily.
Demo fiddles here and here.
The much easier implementation (comparing to the accepted answer) is presented here:
public static class JsonExtensions
{
public static TJToken RemoveAllExcept<TJToken>(this TJToken token, IEnumerable<string> paths) where TJToken : JContainer
{
HashSet<JToken> nodesToRemove = new(ReferenceEqualityComparer.Instance);
HashSet<JToken> nodesToKeep = new(ReferenceEqualityComparer.Instance);
foreach (var whitelistedToken in paths.SelectMany(token.SelectTokens))
TraverseTokenPath(whitelistedToken, nodesToRemove, nodesToKeep);
//In that case neither path from paths has returned any token
if (nodesToKeep.Count == 0)
{
token.RemoveAll();
return token;
}
nodesToRemove.ExceptWith(nodesToKeep);
foreach (var notWhitelistedNode in nodesToRemove)
notWhitelistedNode.Remove();
return token;
}
private static void TraverseTokenPath(JToken value, ISet<JToken> nodesToRemove, ISet<JToken> nodesToKeep)
{
JToken? immediateValue = value;
do
{
nodesToKeep.Add(immediateValue);
if (immediateValue.Parent is JObject or JArray)
{
foreach (var child in immediateValue.Parent.Children())
if (!ReferenceEqualityComparer.Instance.Equals(child, value))
nodesToRemove.Add(child);
}
immediateValue = immediateValue.Parent;
} while (immediateValue != null);
}
}
For most cases this can be achieved by a simple single line extension method
public static string ToJson<T>(this T self, string path) => $#"{{""{path}"":{JObject.FromObject(self)[path]?.ToString(Formatting.None)}}}";
This is only valid for extracting an object nested under the root object but is easily adapted with a separate parameter to specify the output path if needed
Thanks to #dbc answer as a good solution, but like he said, it doesn't affect the performance. Sometimes the data loaded from database has numerous references and only ignoring ReferenceLoopHandling is not enough for serialization; hence the serialized data becomes very large and takes a lot of ram in server, and this is caused by repetition of serializing a single object. In this situation, it's better to make a limited jobject from data straightly, rather than making a jobject and then exclude the unwanted paths from it. This can be done with a little customization of database pure data and a ContractResolver. Let's assume all the database entities inherit from a class or interface like DbModel (this is necessary in this solution). Then by a special ContractResolver, serialization of objects can be limited. A sample is like below:
class TypeName
{
public Type Type { get; set; }
public string Name { get; set; }
}
class MyContractResolver : DefaultContractResolver
{
private List<List<TypeName>> allTypeNames = new List<List<TypeName>>();
public MyContractResolver(Type parentType, string[] includePaths)
{
foreach (var includePath in includePaths)
{
List<TypeName> typeNames = new List<TypeName>() { new TypeName() { Type = parentType } };
var pathChilderen = includePath.Split('.');
for(int i = 0; i < pathChilderen.Length; i++)
{
var propType = typeNames[i].Type.GetProperties().FirstOrDefault(c => c.Name == pathChilderen[i]).PropertyType;
if (propType.GetInterface(nameof(IEnumerable)) != null && propType != typeof(String))
{
propType = propType.GetGenericArguments().Single();
}
typeNames.Add(new TypeName() { Name = pathChilderen[i], Type = propType });
}
allTypeNames.Add(typeNames);
}
}
protected override IList<JsonProperty> CreateProperties(Type type, MemberSerialization memberSerialization)
{
IList<JsonProperty> properties = base.CreateProperties(type, memberSerialization);
// only serializer properties that are in include paths
List<JsonProperty> excludeProperties = new List<JsonProperty>();
foreach (var property in properties)
{
if (typeof(DbModel).IsAssignableFrom(property.PropertyType) || (property.PropertyType.GetInterface(nameof(IEnumerable)) != null && property.PropertyType != typeof(String)))
{
Console.WriteLine(property.PropertyType.ToString());
var exclude = true;
foreach (var typeNames in allTypeNames)
{
var index = typeNames.FindIndex(c => c.Name == property.PropertyName && c.Type == property.PropertyType);
if (index > 0)
{
if (typeNames[index - 1].Type == type)
{
exclude = false;
goto EndSearch;
}
}
}
EndSearch:
if (exclude)
excludeProperties.Add(property);
}
}
properties = properties.Where(c => excludeProperties.All(d => d.PropertyName != c.PropertyName)).ToList();
return properties;
}
}
This class can be used like this:
// return Ok(data);
var jObject = JObject.FromObject(data,
JsonSerializer.CreateDefault(new JsonSerializerSettings()
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
Converters = new List<JsonConverter>()
{
new ValidationProblemDetailsConverter(),
new ProblemDetailsConverter(),
new StringEnumConverter()
},
ContractResolver = new MyContractResolver(typeof(Foo), new[] { "bar", "baz.qux" })
}));
return Ok(jObject);
In this example Foo is the class of main object to return, and bar and baz are properties that are going to be serialized (they are loaded from database too). In addition qux is one of the baz properties that is loaded from database and has to be serialized. In this example all the other properties of each model that are not entities of database (so are not inherited from DbModel) are serialized and all the entities of database that exist in original data but not in the including paths, are ignored to be serialized.

gson flat json to nested objects needs serializer/deserializer?

I have some JSON coming in (I don't have any control or ability to change the structure and/or naming within the JSON...important to keep in mind in this question) that has a "flat" structure similar to this:
{
"name": "...",
"email": "...",
"box_background_color": "...",
"box_border_color": "...",
"box_text_color": "...",
...
}
Now, I can just create a simple object that keeps everything flat, like so:
public class Settings {
#SerializedName("name")
private String _name;
#SerializedName("email")
private String _emailAddress;
#SerializedName("box_background_color")
private String _boxBackgroundColor;
#SerializedName("box_border_color")
private String _boxBorderColor;
#SerializedName("box_text_color")
private String _boxTextColor;
...
}
However, I want everything associated with box settings to be in it's own class (BoxSettings). This is more like what I want:
public class Settings {
#SerializedName("name")
private String _name;
#SerializedName("email")
private String _emailAddress;
private BoxSettings _boxSettings
...
}
public class BoxSettings {
#SerializedName("box_background_color")
private String _boxBackgroundColor;
#SerializedName("box_border_color")
private String _boxBorderColor;
#SerializedName("box_text_color")
private String _boxTextColor;
...
}
I know that if the JSON was structured such that the box settings were nested then it would be easy to accomplish what I want, however, I don't have the ability to change the structure of the JSON, so please don't suggest that (I would do it if I could).
My question is this: Is creating an entire TypeAdapter the only way to accomplish what I want or can I still accomplish most of this with annotations? If it is not the only way, how else can I accomplish this without changing the JSON at all?
The following is an example of what I mean by "creating an entire TypeAdapter":
public class SettingsTypeAdapter implements JsonDeserializer<Settings>, JsonSerializer<Settings> {
#Override
public JsonElement serialize(Settings src, Type typeOfSrc, JsonSerializationContext context) {
// Add _name
// Add _emailAddress
// Add BoxSettings._boxBackgroundColor
// Add BoxSettings._boxBorderColor
// Add BoxSettings._boxTextColor
return jsonElement;
}
#Override
public Settings deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
// Read _name
// Read _emailAddress
// Read BoxSettings._boxBackgroundColor
// Read BoxSettings._boxBorderColor
// Read BoxSettings._boxTextColor
return settings;
}
}
The TypeAdapter is not the only way, but in this case would be the best way since you can associate the adapter with a Gson instance (or whatever library you are using) and have all your mapping code there.
Another way is to use JAVA reflection. I've used a version of the below code in my projects before but never with JSON and never with nested objects (mostly when there was no other choice or if i wanted to map a SQL result set to a Java object without calling resultSet.get... a lot of times).
This will work in this case.
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import org.json.JSONObject;
public class Main {
public static void main(String[] args) {
try {
String json = "{\"name\": \"test name\", \"email\": \"email#email.com\", \"box_background_color\": \"red\", \"box_border_color\": \"orange\", \"box_text_color\": \"white\", \"test3_var2\":3}";
JSONObject jsonObject = new JSONObject(json);
System.out.println(jsonObject);
System.out.println();
/*
* need to parse JSON into a map of String, Object
*/
Map<String, Object> mapAll = new HashMap<String, Object>();
Iterator<String> iter = jsonObject.keys();
while (iter.hasNext()) {
String key = (String) iter.next();
Object value = jsonObject.get(key);
mapAll.put(key, value);
System.out.println(key + "::::" + value);
}
System.out.println();
/*
* use the mapper to generate the objects
*/
MyMapper<TestClass1> myMapper = new MyMapper<TestClass1>();
TestClass1 result = myMapper.mapToObject(mapAll, TestClass1.class);
System.out.println(result);
} catch (Exception e) {
e.printStackTrace();
}
}
}
class MyMapper<T> {
#SuppressWarnings("unchecked")
public T mapToObject(Map<String, Object> flatStructure, Class<T> objectClass) {
T result = null;
Field[] fields = null;
try {
// new base object
result = objectClass.newInstance();
// get all of its fields
fields = objectClass.getDeclaredFields();
for (Field field : fields) {
// normal variable
if (field.isAnnotationPresent(MyColumn.class)) {
String variableKey = field.getAnnotation(MyColumn.class).variableKey();
setJavaFieldValue(result, field.getName(), flatStructure.get(variableKey));
}
// variable that is an object and itself has to be mapped
else if (field.isAnnotationPresent(MyInnerColumn.class)) {
String startsWith = field.getAnnotation(MyInnerColumn.class).startsWith();
// reduce the map to only have attributes that are related to this field
Map<String, Object> reducedMap = reduceMap(startsWith, flatStructure);
// make sure that there are attributes for the inner object
if (reducedMap != null) {
// map the inner object
MyMapper<T> myMapper = new MyMapper<T>();
T t2 = myMapper.mapToObject(reducedMap, (Class<T>) field.getType());
// set the mapped object to the base objecct
setJavaFieldValue(result, field.getName(), t2);
}
} else {
// no annotation on the field so ignored
}
}
} catch (Exception e) {
e.printStackTrace();
}
return result;
}
private Map<String, Object> reduceMap(String startsWith, Map<String, Object> mapToReduce) {
Map<String, Object> result = new HashMap<String, Object>();
for (Map.Entry<String, Object> entry : mapToReduce.entrySet()) {
if (entry.getKey().toLowerCase().startsWith(startsWith.toLowerCase())) {
result.put(entry.getKey(), entry.getValue());
}
}
return result.size() == 0 ? null : result;
}
private void setJavaFieldValue(Object object, String fieldName, Object fieldValue) {
try {
Field field = object.getClass().getDeclaredField(fieldName);
boolean fieldAccess = field.isAccessible();
// make the field accessible
field.setAccessible(true);
field.set(object, fieldValue);
// put it back to the way it was
field.setAccessible(fieldAccess);
} catch (Exception e) {
e.printStackTrace();
}
}
}
/*
* Annotation for a regular variable / field
*/
#Target(ElementType.FIELD)
#Retention(RetentionPolicy.RUNTIME)
#interface MyColumn {
// the variable's JSON key
String variableKey() default "";
}
/*
* Annotation for an inner / nested variable / field
*/
#Target(ElementType.FIELD)
#Retention(RetentionPolicy.RUNTIME)
#interface MyInnerColumn {
/*
* JSON keys that start with this string will be
* associated with this nested field
*/
String startsWith() default "";
}
class TestClass1 {
#MyColumn(variableKey = "name")
private String _name;
#MyColumn(variableKey = "email")
private String _emailAddress;
#MyInnerColumn(startsWith = "box_")
private TestClass2 innerClass;
#MyInnerColumn(startsWith = "test3_")
private TestClass3 innerClass2;
#Override
public String toString() {
return "TestClass1 [_name=" + _name + ", _emailAddress=" + _emailAddress + ", innerClass=" + innerClass + ", innerClass2=" + innerClass2 + "]";
}
}
class TestClass2 {
#MyColumn(variableKey = "box_background_color")
private String _boxBackgroundColor;
#MyColumn(variableKey = "box_border_color")
private String _boxBorderColor;
#MyColumn(variableKey = "box_text_color")
private String _boxTextColor;
#Override
public String toString() {
return "TestClass2 [_boxBackgroundColor=" + _boxBackgroundColor + ", _boxBorderColor=" + _boxBorderColor
+ ", _boxTextColor=" + _boxTextColor + "]";
}
}
class TestClass3 {
#MyColumn(variableKey = "test3_var1")
private String _test3Var1;
#MyColumn(variableKey = "test3_var2")
private int _test3Var2;
#Override
public String toString() {
return "TestClass3 [_test3Var1=" + _test3Var1 + ", _test3Var2=" + _test3Var2 + "]";
}
}
Output
{"box_background_color":"red","box_text_color":"white","test3_var2":3,"name":"test name","email":"email#email.com","box_border_color":"orange"}
box_background_color::::red
box_text_color::::white
test3_var2::::3
name::::test name
email::::email#email.com
box_border_color::::orange
TestClass1 [_name=test name, _emailAddress=email#email.com, innerClass=TestClass2 [_boxBackgroundColor=red, _boxBorderColor=orange, _boxTextColor=white], innerClass2=TestClass3 [_test3Var1=null, _test3Var2=3]]