I'd like to have tooling to perform certain validations on JSON. Explanation with examples:
Given JSON fragment:
{
"optionsMinValue": 0
"optionsMaxValue": 56
"options": [
{
"name": "name1",
"value": 0
},
{
"name": "name2",
"value": 1
},
{
"name": "name3",
"value": 56
}
]
}
Validation examples:
Given the fragment above, the validation of optionsMaxValue should
pass.
Given the fragment above, if optionsMaxValue is changed to 55, then
the validation should fail.
Added bonus validation:
Check whether an item is included in the options array for every integer between optionsMinValue and optionsMaxValue. In other words, in the given fragment the array should contain 57 items with an item for each value from 0 to 56.
Existing tooling:
Does tooling exist that can be used relatively easily to perform these sorts of checks?
First thought is that something like json-schema validation could be done. It has been a few years since I looked at that as an option, so my hope is that tooling has emerged that is a homerun on this.
Ajv JSON schema validator - github link
const schema = {
type: "object",
properties: {
name: {type: "string"},
value: {type: "number", minimum: 0, maximum: 55},
},
required: ["name", "value"],
additionalProperties: false,
}
const option = {
"name": "name1",
"value": 0
},
const validate = ajv.compile(schema)
const valid = validate(data)
if (!valid) console.log(validate.errors)
<script src="https://cdnjs.cloudflare.com/ajax/libs/ajv/4.4.0/ajv.min.js"></script>
Joi package is best for these kind of validations
following Joi schema can be used to solve your requirement
Joi.object({
optionsMinValue: Joi.number().min(0).max(30).required(),
optionsMaxValue: Joi.number().min(56).max(100).required(),
options: Joi.array().items(
Joi.object({
name: Joi.string().required(),
value: Joi.number().min(0).max(56).required(),
})
),
});
Following is a sample code that works for your scenario
const inputData = {
optionsMinValue: 0,
optionsMaxValue: 56,
options: [
{
name: "name1",
value: 0,
},
{
name: "name2",
value: 1,
},
{
name: "name3",
value: 56,
},
],
};
const Joi = joi; // for node.js use - const Joi = require("joi");
// Schema for validation
const schema = Joi.object({
optionsMinValue: Joi.number().min(0).max(30).required(),
optionsMaxValue: Joi.number().min(56).max(100).required(),
options: Joi.array().items(
Joi.object({
name: Joi.string().required(),
value: Joi.number().min(0).max(56).required(),
})
),
});
const runValidation = (schema, inputData) => {
const validationResult = Joi.compile(schema)
.prefs({ errors: { label: "key" }, abortEarly: false })
.validate(inputData);
if (validationResult.error) {
// Validation failed
console.log("Error, validation failed");
// Set error message to string
const errorMessage = validationResult.error.details
.map((details) => details.message)
.join(", ");
console.log("failure reason - ", errorMessage);
return;
}
console.log("validation passed");
};
runValidation(schema, inputData);
<script src="https://cdn.jsdelivr.net/npm/joi#17.6.0/dist/joi-browser.min.js"></script>
Even if you use an existing tool, you should write validation rules for that tool. Since you are not an expert in any of these tools, it may be easier to write a few lines of code in your preferred language. For example, in JavaScript it might look like this:
function validateJson(jsonToValidate, maxValue = 56) {
if (jsonToValidate.optionsMaxValue !== maxValue) {
console.log("Failure on optionsMaxValue.");
return false;
}
if (jsonToValidate.options.length !== maxValue+1) {
console.log("Incorrect number of items.");
return false;
}
let values = jsonToValidate.options.map(a => a.value).sort();
if (values[0] !== 0 || values[maxValue] !== maxValue) {
console.log("Values out of desired sequence.");
return false;
}
let sum = values.reduce((a, b) => a + b, 0);
if (sum !== maxValue * (maxValue + 1) / 2) {
console.log("Values out of desired sequence.");
return false;
}
console.log("Validation PASSED.");
return true;
}
Let's try with truncated json object:
let jsonSample = {
"optionsMinValue": 0,
"optionsMaxValue": 2,
"options": [{
"name": "name1",
"value": 0
},
{
"name": "name2",
"value": 1
},
{
"name": "name3",
"value": 2
}
]
};
function validateJson(jsonToValidate, maxValue = 56) {
if (jsonToValidate.optionsMaxValue !== maxValue) {
console.log("Failure on optionsMaxValue.");
return false;
}
if (jsonToValidate.options.length !== maxValue+1) {
console.log("Incorrect number of items.");
return false;
}
let values = jsonToValidate.options.map(a => a.value).sort();
if (values[0] !== 0 || values[maxValue] !== maxValue) {
console.log("Values out of desired sequence.");
return false;
}
let sum = values.reduce((a, b) => a + b, 0);
if (sum !== maxValue * (maxValue + 1) / 2) {
console.log("Values out of desired sequence.");
return false;
}
console.log("Validation PASSED.");
return true;
}
validateJson(jsonSample, 2);
We are working on a Middleware platform where we are required to respond to consumer with a JSON data in a particular format.
The Data we get from south bound API is a key value pair and this needs to be mapped to an understandable format for the consumer
We tried json-path, ObjectMapper but none of them is giving us the expected result for transforming
Respnse from backend API
{
"details": [
{
"name": "x.y.z.name","value": "TR-54695"
},
{
"name": "a.b.c.standards","value": "DOCSIS"
},
{
"name": "x.x.x.hversion","value": "10"
},
{
"name": "x.x.x.sversion","value": "9.1.116V"
},
{
"name": "x.x.x.uptime","value": "8000"
},
{
"name": "x.x.x.accessallowed","value": "true"
},
]
}
To be transformed to
{
"myData": {
"myInfo": {
"productClass": "TR-54695",
"supportedStandards": "DOCSIS",
"hardwareVersion": "10",
"softwareVersion": "9.1.116V",
"modemMacAddress": "",
"upTime": "8000",
"modemNetworkAccessAllowed": true
}
}
}
Do not like manual work, so here generated demo using 2 functions.
Mind ticking accept button under voting in case you like some answer.
function translate(src, mapping) {
var dst = { "myData": { "myInfo": { "modemMacAddress": "" } } }
//in case order matters:
dst = { "myData": { "myInfo": { "productClass": "", "supportedStandards": "", "hardwareVersion": "", "softwareVersion": "", "modemMacAddress": "", "upTime": "", "modemNetworkAccessAllowed": undefined } } }
var trueFalse = { "false": false, "true": true };
src = src.details;
for (var i = 0; i < src.length; i++) {
dst.myData.myInfo[mapping[src[i].name]] = trueFalse[src[i].value] || src[i].value;
}
return dst;
}
function generateMapping(src, dst) {
src = src.details;
var backLinks = {}, rename2 = {};
for (var i = 0; i < src.length; i++) {
backLinks[src[i].value] = src[i].name;
}
dst = dst.myData.myInfo;
for (var i in dst) {
rename2[backLinks[dst[i]]] = i;
}
return rename2;
}
var src = {
"details": [
{ "name": "x.y.z.name", "value": "TR-54695" },
{ "name": "a.b.c.standards", "value": "DOCSIS" },
{ "name": "x.x.x.hversion", "value": "10" },
{ "name": "x.x.x.sversion", "value": "9.1.116V" },
{ "name": "x.x.x.uptime", "value": "8000" },
{ "name": "x.x.x.accessallowed", "value": "true" },
]
}
var dst = {
"myData": {
"myInfo": {
"productClass": "TR-54695",
"supportedStandards": "DOCSIS",
"hardwareVersion": "10",
"softwareVersion": "9.1.116V",
"modemMacAddress": "",
"upTime": "8000",
"modemNetworkAccessAllowed": true
}
}
}
var mapping = generateMapping(src, dst);
// var mapping = {
// "x.y.z.name": "productClass",
// "a.b.c.standards": "supportedStandards",
// "x.x.x.hversion": "hardwareVersion",
// "x.x.x.sversion": "softwareVersion",
// "undefined": "modemMacAddress",
// "x.x.x.uptime": "upTime",
// "x.x.x.accessallowed": "modemNetworkAccessAllowed"
// }
var result = translate(src, mapping);
console.log(JSON.stringify(result, null, 2));
console.log(JSON.stringify(mapping, null, 2));
You can use below code and use codesandbox link (check console output ) for exact response and this link for key:value pair.
let response = {
details: [
{
name: "x.y.z.name",
value: "TR-54695"
},
{
name: "a.b.c.standards",
value: "DOCSIS"
},
{
name: "x.x.x.hversion",
value: "10"
},
{
name: "x.x.x.sversion",
value: "9.1.116V"
},
{
name: "x.x.x.uptime",
value: "8000"
},
{
name: "x.x.x.accessallowed",
value: "true"
}
]
};
// convert function for key value pair
function convertResponse(responseData) {
let output = { myData: { myInfo: {} } };
let outputRef = output.myData.myInfo;
responseData.forEach(element => {
outputRef[element.name] = element.value
});
return output;
}
// OR convert Function for getting exact same output
function convertResponse(responseData) {
let output = { myData: { myInfo: {} } };
let outputRef = output.myData.myInfo;
responseData.forEach(element => {
if (element.name === "x.y.z.name") {
outputRef.productClass = element.value;
} else if (element.name === "a.b.c.standards") {
outputRef.supportedStandards = element.value;
} else if (element.name === "x.x.x.hversion") {
outputRef.hardwareVersion = element.value;
} else if (element.name === "x.x.x.sversion") {
outputRef.softwareVersion = element.value;
} else if (element.name === "x.x.x.uptime") {
outputRef.upTime = element.value;
} else if (element.name === "x.x.x.accessallowed") {
outputRef.modemNetworkAccessAllowed = element.value;
}
});
return output;
}
//Function Call
console.log(convertResponse(response.details));
I have a JSON which come from spark:
val df = spark.read.parquet("hdfs://xxx-namespace/20190311")
val jsonStr = df.schema.json
jsonStr is like this:
{
"type":"struct",
"fields":[
{
"name":"alm_dm_list",
"type":{
"type":"array",
"elementType":"integer",
"containsNull":true
},
"nullable":true,
"metadata":{
}
},
{
"name":"data_batt_sc_volt_lowest",
"type":"double",
"nullable":true,
"metadata":{
}
},
{
"name":"veh_dcdcst",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_temp_data",
"type":{
"type":"array",
"elementType":{
"type":"struct",
"fields":[
{
"name":"esd_temp_probe_cnt",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_temp_probe_list",
"type":{
"type":"array",
"elementType":"integer",
"containsNull":true
},
"nullable":true,
"metadata":{
}
},
{
"name":"esd_temp_subsys_seq",
"type":"integer",
"nullable":true,
"metadata":{
}
}
]
},
"containsNull":true
},
"nullable":true,
"metadata":{
}
},
{
"name":"esd_volt_data",
"type":{
"type":"array",
"elementType":{
"type":"struct",
"fields":[
{
"name":"esd_curr",
"type":"double",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_frame_sc_cnt",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_frame_sc_list",
"type":{
"type":"array",
"elementType":"double",
"containsNull":true
},
"nullable":true,
"metadata":{
}
},
{
"name":"esd_frame_start",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_sc_cnt",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_volt",
"type":"double",
"nullable":true,
"metadata":{
}
},
{
"name":"esd_volt_subsys_seq",
"type":"integer",
"nullable":true,
"metadata":{
}
}
]
},
"containsNull":true
},
"nullable":true,
"metadata":{
}
},
{
"name":"dm_data",
"type":{
"type":"array",
"elementType":{
"type":"struct",
"fields":[
{
"name":"dm_ctl_dc_curr",
"type":"double",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_ctl_temp",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_ctl_volt",
"type":"double",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_seq",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_spd",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_st",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_temp",
"type":"integer",
"nullable":true,
"metadata":{
}
},
{
"name":"dm_torq",
"type":"integer",
"nullable":true,
"metadata":{
}
}
]
},
"containsNull":true
},
"nullable":true,
"metadata":{
}
}]
}
I want to get a scheme based on the JSON string, so i need to parse it and get a struct like this:
val schema = new StructType()
.add("alm_dm_list", ArrayType(IntegerType, true), true)
.add("data_batt_sc_volt_lowest", DoubleType, true)
.add("veh_dcdcst", IntegerType, true)
.add("esd_temp_data", ArrayType(new StructType()
.add("esd_temp_probe_cnt", IntegerType, true)
.add("esd_temp_probe_list", ArrayType(IntegerType, true), true)
.add("esd_temp_subsys_seq", IntegerType, true)
), true)
.add("esd_volt_data", ArrayType(new StructType()
.add("esd_curr", DoubleType, true)
.add("esd_frame_sc_cnt", IntegerType, true)
.add("esd_frame_sc_list", ArrayType(DoubleType, true), true)
.add("esd_frame_start", IntegerType, true)
.add("esd_sc_cnt", IntegerType, true)
.add("esd_volt", DoubleType, true)
.add("esd_volt_subsys_seq", IntegerType, true)
), true)
.add("dm_data", ArrayType(new StructType()
.add("dm_ctl_dc_curr", DoubleType, true)
.add("dm_ctl_temp", IntegerType, true)
.add("dm_ctl_volt", DoubleType, true)
.add("dm_seq", IntegerType, true)
.add("dm_spd", IntegerType, true)
.add("dm_st", IntegerType, true)
.add("dm_temp", IntegerType, true)
.add("dm_torq", IntegerType, true)
), true)
JSON::Infer maybe helpful, but there are nested structures in my JSON, which is complex for me, any suggestions will help.
In case you do not care about identing (possible, but not so important, what about this ?) does not work in IE - guess template strings are only problem(?):
'use strict';
function run() {
buildRecursive(2);
res = res.substr(0, res.length-1) + ';';
console.log(res);
}
var typeRename = {
"array": "ArrayType",
"double": "DoubleType",
"integer": "IntegerType"
}, elementRename = {
"integer": "IntegerType",
"double": "DoubleType"
}
var res = "val schema = ";
function repeatIndent(no) {
var retVal = '';
while(no--) retVal += ' ';
return retVal;
}
function buildRow(indent, params) {
return repeatIndent(indent) + params.join('');
}
function buildRecursive(indent) {
var lev = it.ReadArray(), indentStep = 1;
if(lev.type == "struct") {
res += "new StructType()\n";
var under = lev.fields;
while(under && under.node) {
it.SetCurrent(under.node);
buildRecursive(indent + indentStep);
under = under.next;
}
} else if (lev.name) {
if(lev.type instanceof JNode) {
it.SetCurrent(lev.type.node);
var lev2 = it.ReadArray();
it.DepthFirst();
var elementType = it.FindKey("elementType");
if(elementType.value instanceof Object) {
res += buildRow(indent, ['.add("', lev.name, '", ', typeRename[lev2.type], '(']);
var here = it.Current;
it.SetCurrent(elementType.node);
buildRecursive(indent + indentStep);
it.SetCurrent(here);
res += buildRow(indent,['), ', lev.nullable, ')\n']);
} else res += buildRow(indent, ['.add("', lev.name, '", ', typeRename[lev2.type], '(',
elementRename[elementType.value], ', ', lev2.containsNull, '), ', lev.nullable, ')\n']);
} else {
res += buildRow(indent, ['.add("', lev.name, '", ', typeRename[lev.type], ', ', lev.nullable, ')\n']);
}
}
}
// My JSON iterator
var JNode = (function (jsNode) {
function JNode(_parent, _pred, _key, _value) {
this.parent = _parent;
this.pred = _pred;
this.node = null;
this.next = null;
this.key = _key;
this.value = _value;
}
return JNode;
})();
var JIterator = (function (json) {
var root, current, maxLevel = -1;
function JIterator(json, parent) {
if (parent === undefined) parent = null;
var pred = null, localCurrent;
for (var child in json) {
var obj = json[child] instanceof Object;
if(json instanceof Array) child = parseInt(child); // non-associative array
if (!root) root = localCurrent = new JNode(parent, null, child, json[child]);
else {
localCurrent = new JNode(parent, pred, child, obj ? ((json[child] instanceof Array) ? [] : {}) : json[child]);
}
if (pred) pred.next = localCurrent;
if (parent && parent.node == null) parent.node = localCurrent;
pred = localCurrent;
if (obj) {
var memPred = pred;
JIterator(json[child], pred);
pred = memPred;
}
}
if (this) {
current = root;
this.Level = 0;
}
}
JIterator.prototype.Current = function () { return current; }
JIterator.prototype.SetCurrent = function (newCurrent) { current = newCurrent; }
JIterator.prototype.Parent = function () {
var retVal = current.parent;
if (retVal == null) return false;
this.Level--;
return current = retVal;
}
JIterator.prototype.Pred = function () {
var retVal = current.pred;
if (retVal == null) return false;
return current = retVal;
}
JIterator.prototype.Node = function () {
var retVal = current.node;
if (retVal == null) return false;
this.Level++;
return current = retVal;
}
JIterator.prototype.Next = function () {
var retVal = current.next;
if (retVal == null) return false;
return current = retVal;
}
JIterator.prototype.Key = function () { return current.key; }
JIterator.prototype.KeyDots = function () { return (typeof(current.key) == "number")?"":(current.key+':'); }
JIterator.prototype.Value = function () { return current.value; }
JIterator.prototype.Reset = function () {
current = root;
this.Level = 0;
}
JIterator.prototype.RawPath = function () {
var steps = [], level = current;
do {
if (level != null && level.value instanceof Object) {
steps.push(level.key + (level.value instanceof Array ? "[]" : "{}"));
} else {
if (level != null) steps.push(level.key);
else break;
}
level = level.parent;
} while (level != null);
var retVal = "";
retVal = steps.reverse();
return retVal;
}
JIterator.prototype.Path = function () {
var steps = [], level = current;
do {
if (level != null && level.value instanceof Object) {
var size = 0;
var items = level.node;
if(typeof(level.key) == "number") steps.push('[' + level.key + ']');
else {
while(items) {
size++;
items = items.next;
}
var type = (level.value instanceof Array ? "[]" : "{}");
var prev = steps[steps.length-1];
if(prev && prev[0] == '[') {
var last = prev.length-1;
if(prev[last] == ']') {
last--;
if(!isNaN(prev.substr(1, last))) {
steps.pop();
size += '.' + prev.substr(1, last);
}
}
}
steps.push(level.key + type[0] + size + type[1]);
}
} else {
if (level != null) {
if(typeof(level.key) == "number") steps.push('[' + level.key + ']');
else steps.push(level.key);
}
else break;
}
level = level.parent;
} while (level != null);
var retVal = "";
retVal = steps.reverse();
return retVal;
}
JIterator.prototype.DepthFirst = function () {
if (current == null) return 0; // exit sign
if (current.node != null) {
current = current.node;
this.Level++;
if (maxLevel < this.Level) maxLevel = this.Level;
return 1; // moved down
} else if (current.next != null) {
current = current.next;
return 2; // moved right
} else {
while (current != null) {
if (current.next != null) {
current = current.next;
return 3; // returned up & moved next
}
this.Level--;
current = current.parent;
}
}
return 0; // exit sign
}
JIterator.prototype.BreadthFirst = function () {
if (current == null) return 0; // exit sign
if (current.next) {
current = current.next;
return 1; // moved right
} else if (current.parent) {
var level = this.Level, point = current;
while (this.DepthFirst() && level != this.Level);
if (current) return 2; // returned up & moved next
do {
this.Reset();
level++;
while (this.DepthFirst() && level != this.Level);
if (current) return 3; // returned up & moved next
} while (maxLevel >= level);
return current != null ? 3 : 0;
} else if (current.node) {
current = current.node;
return 3;
} else if (current.pred) {
while (current.pred) current = current.pred;
while (current && !current.node) current = current.next;
if (!current) return null;
else return this.DepthFirst();
}
}
JIterator.prototype.ReadArray = function () {
var retVal = {};
var item = current;
do {
if(item.value instanceof Object) {
if(item.value.length == 0) retVal[item.key] = item.node;
else retVal[item.key] = item;
} else retVal[item.key] = item.value;
item = item.next;
} while (item != null);
return retVal;
}
JIterator.prototype.FindKey = function (key) {
var pos = current;
while(current && current.key != key) this.DepthFirst();
if(current.key == key) {
var retVal = current;
current = pos;
return retVal;
} else {
current = pos;
return null;
}
}
return JIterator;
})();
var json = {
"type": "struct",
"fields": [
{
"name": "alm_dm_list",
"type": {
"type": "array",
"elementType": "integer",
"containsNull": true
},
"nullable": true,
"metadata": {}
},
{
"name": "data_batt_sc_volt_lowest",
"type": "double",
"nullable": true,
"metadata": {}
},
{
"name": "veh_dcdcst",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "esd_temp_data",
"type": {
"type": "array",
"elementType": {
"type": "struct",
"fields": [
{
"name": "esd_temp_probe_cnt",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "esd_temp_probe_list",
"type": {
"type": "array",
"elementType": "integer",
"containsNull": true
},
"nullable": true,
"metadata": {}
},
{
"name": "esd_temp_subsys_seq",
"type": "integer",
"nullable": true,
"metadata": {}
}
]
},
"containsNull": true
},
"nullable": true,
"metadata": {}
},
{
"name": "esd_volt_data",
"type": {
"type": "array",
"elementType": {
"type": "struct",
"fields": [
{
"name": "esd_curr",
"type": "double",
"nullable": true,
"metadata": {}
},
{
"name": "esd_frame_sc_cnt",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "esd_frame_sc_list",
"type": {
"type": "array",
"elementType": "double",
"containsNull": true
},
"nullable": true,
"metadata": {}
},
{
"name": "esd_frame_start",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "esd_sc_cnt",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "esd_volt",
"type": "double",
"nullable": true,
"metadata": {}
},
{
"name": "esd_volt_subsys_seq",
"type": "integer",
"nullable": true,
"metadata": {}
}
]
},
"containsNull": true
},
"nullable": true,
"metadata": {}
},
{
"name": "dm_data",
"type": {
"type": "array",
"elementType": {
"type": "struct",
"fields": [
{
"name": "dm_ctl_dc_curr",
"type": "double",
"nullable": true,
"metadata": {}
},
{
"name": "dm_ctl_temp",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "dm_ctl_volt",
"type": "double",
"nullable": true,
"metadata": {}
},
{
"name": "dm_seq",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "dm_spd",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "dm_st",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "dm_temp",
"type": "integer",
"nullable": true,
"metadata": {}
},
{
"name": "dm_torq",
"type": "integer",
"nullable": true,
"metadata": {}
}
]
},
"containsNull": true
},
"nullable": true,
"metadata": {}
}
]
};
var it = new JIterator(json);
run();