ExpressJS set the Depth of JSON Parsing - json

I want to set the depth of JSON parsing in Express middleware express.json().
For example, if I would set the option to parse the depth=1, then
'{ "email": { "$ne": "user#example.com" } }'
will be parsed to
{ email: "[object Object]" }
-- or --
When I set depth=2, then
'{ "email": { "$ne": "user#example.com" } }'
will be parsed to
{ email: { '$ne': 'user#example.com' } }
And so on,
In this case, there will be no issue of default depth, as the developer will be aware of how many nesting they will allow while development.
PS: It will prevent the application from being vulnerable to NoSQL Injection.

Just write you own middleware:
const get_depth = (obj) => {
let depth = 0
for(const key in obj) {
if( obj[key] instanceof Object ) {
depth = Math.max(get_depth(obj[key]), depth)
}
}
return depth+1
}
const depth_limit = 2
const limit_depth = function(req, res, next) {
if( get_depth(req.body) > depth_limit ) throw new Error("Possible NoSQL Injection")
next()
}
app.use(limit_depth)
Or, if you prefer "[object Object]":
let limit_depth = (obj, current_depth, limit) => {
for(const key in obj) {
if( obj[key] instanceof Object ) {
if( current_depth+1 === limit ) {
obj[key] = "[object Object]" // or something similar
}
else limit_depth(obj[key], current_depth+1, limit)
}
}
}
app.use(function(req, res, next) { limit_depth(req.body, 0, depth_limit); next() })

I write down the query, Maximum 6-8 depth goes. when use lookup inside the lookup.
const [result] = await Collection.aggregate([
{ $match:statusObj },
{
$project:{
_id:1,
name:1
}
},
{
$lookup:{
from:"articles",
let: { "cat_id":"$_id"},
pipeline:[
{
$match:{
$expr:{
$and: [
{ $eq: ["$category_id", "$$cat_id"] },
{ $eq: ["$isDeleted", false] },
{ $eq: ["$type", type] }
]
}
}
},
{
$lookup:{
from:"view_articles",
let: { "article_id":"$_id"},
pipeline:[
{
$match:{
$expr:{
$and: [
{ $eq: ["$article_id", "$$article_id"] },
{ $eq: ["$isDeleted", false] }
]
}
}
}
],
as:"viewCount"
}
},
{
$addFields:{
noOfViewCount : { $size:"$viewCount"}
}
} ],
as:"articleCategoryData"
}
},
{
$addFields: {
postCount: {$size:"$articleCategoryData" },
tempsArray: { $map:
{
input: "$articleCategoryData",
as: "tempData",
in: { $add: "$$tempData.noOfViewCount" }
}
},
},
},
{
$addFields: {
viewCount:{ $sum:"$tempsArray" }
},
},
{
$project:{
_id: 1,
name: 1,
postCount: 1,
viewCount: 1
}
},
{
$facet: {
count: [
{
$count: "total"
}
],
result: [{ $match: {} }, { $skip: skipRecord }, { $limit: limit }]
}
}
]);
you can set depth to 10. If you feel JSON is coming wrong then increase it :)

In case anyone who doesn't want to change the value of req.body, can use this function from here
function serializer(payload: any, cdepth: number, options: Options): void {
const main: any = {}
const maxDepth = typeof options.maxNestingLevel == 'number' ? (options.maxNestingLevel == 0 ? 1 : options.maxNestingLevel) : 1
for (const key in payload) {
// check for object
if (payload[key] instanceof Object) {
// check if depth is limited, replace if needed
if (cdepth === maxDepth) {
main[key] = options.replaceWith
} else {
// serialize the nested
main[key] = serializer(payload[key], cdepth + 1, options)
}
} else {
// add to main object if not to be checked
main[key] = payload[key]
}
}
return main
}

Related

TypeScript: find values of certain keys in JSON

I have a JSON that contains a lot of data, here's an example of it:
{
"type":"doc",
"content":[
{
"type":"paragraph",
"content":[
{
"text":"this is a simple page, about a simple umbrella.",
"type":"text"
}
]
},
{
"type":"paragraph",
"content":[
{
"text":"you can use this text to find the umbrella page.",
"type":"text"
}
]
},
{
"type":"paragraph",
"content":[
{
"text":"do you like it?",
"type":"text"
}
]
}
}
I want to extract the value of text key, no matter where the key is located. I'm trying to go over the keys using Object.keys but it only returns the top-level keys:
for (let x of Object.keys(someJson)) {
console.log(x);
}
How can I find all the values of text in this JSON, no matter where in the JSON it is?
You can use JSON.stringify trick, you can intercept all keys from it
function find(obj: object, key: string) {
const ret: any[] = [];
JSON.stringify(obj, (_, nested) => {
if (nested && nested[key]) {
ret.push(nested[key]);
}
return nested;
});
return ret;
};
...
const o = {
key: '123',
a: {
key: 'hello',
b: [
{
c: {
key: 123,
},
},
],
},
};
it('123', () => {
console.log(JSON.stringify(find(o, 'key'))); // ["123","hello",123]
});
if you want for generic JSON just call this function and pass your object :
function printText(obj){
if(Array.isArray(obj)){
for(const o of obj){
printText(o);
}
}else if(typeof obj === "object"){
if (obj){
for(const o of Object.keys(obj)){
if(o==="text"){
console.log(obj.text);
}else{
printText(obj[o]);
}
}
}
}
}

JSON variable block

I am having json template for elasticsearch query:
var getTemplate = function (agg, filter_term) {
var template = {
"size": 0,
track_total_hits: true,
query: {
bool: {
must: [
{
"query_string": {
"query": filter_term
}
},
aggs: {
"nested" : { "value_count" : { "field" : agg } },
agg: {
terms: {
field: agg,
size: 10,
order: {
"_count": "desc"
}
}
}
}
};
return template;
Sometimes, I want to skip the block with query_string so I want to pass in in the function call. So instead of this:
const callTerminated = agg_filter.getTemplate( 'bbbbb', 'aaa');
Do that:
const callTerminated = agg_filter.getTemplate( 'bbbbb', {"query_string": {"query": aaa }});
Or:
const callTerminated = agg_filter.getTemplate( 'bbbbb', "");
But how to change the template query? It wants comma after variable but when I skip query_string, I don't need comma.
New json template:
var getTemplate = function (agg, filter_term) {
var template = {
"size": 0,
track_total_hits: true,
query: {
bool: {
must: [
filter_term //here it wants comma
aggs: {
"nested" : { "value_count" : { "field" : agg } },
agg: {
terms: {
field: agg,
size: 10,
order: {
"_count": "desc"
}
}
}
}
};
return template;
So how to do it?

JSON Transformation to required format

We are working on a Middleware platform where we are required to respond to consumer with a JSON data in a particular format.
The Data we get from south bound API is a key value pair and this needs to be mapped to an understandable format for the consumer
We tried json-path, ObjectMapper but none of them is giving us the expected result for transforming
Respnse from backend API
{
"details": [
{
"name": "x.y.z.name","value": "TR-54695"
},
{
"name": "a.b.c.standards","value": "DOCSIS"
},
{
"name": "x.x.x.hversion","value": "10"
},
{
"name": "x.x.x.sversion","value": "9.1.116V"
},
{
"name": "x.x.x.uptime","value": "8000"
},
{
"name": "x.x.x.accessallowed","value": "true"
},
]
}
To be transformed to
{
"myData": {
"myInfo": {
"productClass": "TR-54695",
"supportedStandards": "DOCSIS",
"hardwareVersion": "10",
"softwareVersion": "9.1.116V",
"modemMacAddress": "",
"upTime": "8000",
"modemNetworkAccessAllowed": true
}
}
}
Do not like manual work, so here generated demo using 2 functions.
Mind ticking accept button under voting in case you like some answer.
function translate(src, mapping) {
var dst = { "myData": { "myInfo": { "modemMacAddress": "" } } }
//in case order matters:
dst = { "myData": { "myInfo": { "productClass": "", "supportedStandards": "", "hardwareVersion": "", "softwareVersion": "", "modemMacAddress": "", "upTime": "", "modemNetworkAccessAllowed": undefined } } }
var trueFalse = { "false": false, "true": true };
src = src.details;
for (var i = 0; i < src.length; i++) {
dst.myData.myInfo[mapping[src[i].name]] = trueFalse[src[i].value] || src[i].value;
}
return dst;
}
function generateMapping(src, dst) {
src = src.details;
var backLinks = {}, rename2 = {};
for (var i = 0; i < src.length; i++) {
backLinks[src[i].value] = src[i].name;
}
dst = dst.myData.myInfo;
for (var i in dst) {
rename2[backLinks[dst[i]]] = i;
}
return rename2;
}
var src = {
"details": [
{ "name": "x.y.z.name", "value": "TR-54695" },
{ "name": "a.b.c.standards", "value": "DOCSIS" },
{ "name": "x.x.x.hversion", "value": "10" },
{ "name": "x.x.x.sversion", "value": "9.1.116V" },
{ "name": "x.x.x.uptime", "value": "8000" },
{ "name": "x.x.x.accessallowed", "value": "true" },
]
}
var dst = {
"myData": {
"myInfo": {
"productClass": "TR-54695",
"supportedStandards": "DOCSIS",
"hardwareVersion": "10",
"softwareVersion": "9.1.116V",
"modemMacAddress": "",
"upTime": "8000",
"modemNetworkAccessAllowed": true
}
}
}
var mapping = generateMapping(src, dst);
// var mapping = {
// "x.y.z.name": "productClass",
// "a.b.c.standards": "supportedStandards",
// "x.x.x.hversion": "hardwareVersion",
// "x.x.x.sversion": "softwareVersion",
// "undefined": "modemMacAddress",
// "x.x.x.uptime": "upTime",
// "x.x.x.accessallowed": "modemNetworkAccessAllowed"
// }
var result = translate(src, mapping);
console.log(JSON.stringify(result, null, 2));
console.log(JSON.stringify(mapping, null, 2));
You can use below code and use codesandbox link (check console output ) for exact response and this link for key:value pair.
let response = {
details: [
{
name: "x.y.z.name",
value: "TR-54695"
},
{
name: "a.b.c.standards",
value: "DOCSIS"
},
{
name: "x.x.x.hversion",
value: "10"
},
{
name: "x.x.x.sversion",
value: "9.1.116V"
},
{
name: "x.x.x.uptime",
value: "8000"
},
{
name: "x.x.x.accessallowed",
value: "true"
}
]
};
// convert function for key value pair
function convertResponse(responseData) {
let output = { myData: { myInfo: {} } };
let outputRef = output.myData.myInfo;
responseData.forEach(element => {
outputRef[element.name] = element.value
});
return output;
}
// OR convert Function for getting exact same output
function convertResponse(responseData) {
let output = { myData: { myInfo: {} } };
let outputRef = output.myData.myInfo;
responseData.forEach(element => {
if (element.name === "x.y.z.name") {
outputRef.productClass = element.value;
} else if (element.name === "a.b.c.standards") {
outputRef.supportedStandards = element.value;
} else if (element.name === "x.x.x.hversion") {
outputRef.hardwareVersion = element.value;
} else if (element.name === "x.x.x.sversion") {
outputRef.softwareVersion = element.value;
} else if (element.name === "x.x.x.uptime") {
outputRef.upTime = element.value;
} else if (element.name === "x.x.x.accessallowed") {
outputRef.modemNetworkAccessAllowed = element.value;
}
});
return output;
}
//Function Call
console.log(convertResponse(response.details));

if value changes in function, add into new array

I`am filtering a search query by clicking at a checkbox. I have different check boxes. All uses the same function to add and remove my query build.
I am storing the values in a array and format them for my query build.
I'am using Elasticsearch for the query and angular7.
onChangeLocations(value: string, checked: boolean) {
if (checked) {
this.selectedLocations.push(value);
this.selectedTopics.push({
'match_phrase': { '134_facet_locations': value },
});
this.selectedTopics.push({
'match_phrase': { '135_facet_locations': value },
});
this.selectedTopics.push({
'match_phrase': { '136_facet_locations': value },
});
} else {
this.selectedTopics = this.selectedTopics.filter((topic: any) => {
return topic.match_phrase['134_facet_locations'] !== value;
});
this.selectedTopics = this.selectedTopics.filter((topic: any) => {
return topic.match_phrase['135_facet_locations'] !== value;
});
this.selectedTopics = this.selectedTopics.filter((topic: any) => {
return topic.match_phrase['136_facet_locations'] !== value;
});
this.selectedLocations = this.selectedLocations.filter((facet: any) => {
return facet !== value;
});
}
}
this is my Output that works for OR querys.
[
{
"bool": {
"should": [
{
"match_phrase": {
"134_facet_locations": "Deutschland"
}
},
{
"match_phrase": {
"135_facet_locations": "Deutschland"
}
},
{
"match_phrase": {
"136_facet_locations": "Deutschland"
}
},
{
"match_phrase": {
"134_facet_locations": "Basel"
}
},
{
"match_phrase": {
"135_facet_locations": "Basel"
}
},
{
"match_phrase": {
"136_facet_locations": "Basel"
}
}
]
}
}
]
I Need an Output for and querys that must look like this:
{
"should": [
{
"bool": {
"should": [
{
"match_phrase": {
"134_facet_locations": "Basel"
}
},
{
"match_phrase": {
"135_facet_locations": "Basel"
}
},
{
"match_phrase": {
"136_facet_locations": "Basel"
}
}
]
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"134_facet_locations": "Deutschland"
}
},
{
"match_phrase": {
"135_facet_locations": "Deutschland"
}
},
{
"match_phrase": {
"136_facet_locations": "Deutschland"
}
}
]
}
}
]
}
I know how to add them once into an array. But i dont know how to build this complex json query in a good way.
This is my suggestions for the answer. I must check if value changed. if it changed add topics to new "bool should" array. Something like this.

Observable from a RESTful paged collection

On one hand, I have a RESTful HAL HATEOAS collection which looks like this :
{
"page": 1,
"limit": 10,
"pages": 18,
"total": 174,
"_links": {
"self": { "href": "/users?page=1&limit=10" },
"first": { "href": "/users?page=1&limit=10" },
"last": { "href": "/users?page=18&limit=10" },
"next": { "href": "/users?page=2&limit=10" }
},
"_embedded": {
"users": [
{
"name": "bob",
"_links": { "self": { "href": "/users/1" } }
},
...
]
}
}
On the other hand, I have an Angular 2 app.
public getUsers(uri: string = this.baseURI): Observable<User> {
return this.http.get(uri)
.map(res => res.json()._embedded.users as User[])
.flatMap(d => d) // Transform the flux of arrays in flux of users
.catch(this.handleError);
} // Get only the 10th first users
What I'm trying to do have an observable of Users which will append data while _links.next != null
Modified service
public getUsers(uri: string = this.baseURI): Observable<User> {
return this.http.get(uri)
.do(res => {
const uri = JSON.parse(res._body)._links.next.href;
this.nextUri = uri ? uri : null;
})
.map(res => res.json()._embedded.users as User[])
.flatMap(d => d) // Transform the flux of arrays in flux of users
.catch(this.handleError);
}
Recursive function
loadAll(uri: string) {
read(uri)
.subscribe(
user => {
this.stockedUsers.push(user);
},
error => console.log(error),
() => {
if (this.nextUri) {
this.loadAll(this.nextUri);
}
}
);
}
Does someone know how to achieve this properly ?
I want to keep thes advantages of the RxJS flux.
UPDATE/ANSWER
Silly me ! I think I answered myself. Maybe this will help others :
public read(uri: string = this.baseURI): Observable<User> {
return Observable.create(observer => this.iteratePages(observer, uri));
}
private iteratePages(observer: Observer<User>, uri): void {
if (uri == null) { return observer.complete(); }
this.http.get(uri).subscribe(res => {
const data = res.json();
for (const user of data._embedded.users) {
observer.next(user as User);
}
const nextUri = (data._links && data._links.next) ? data._links.next.href : null;
this.iteratePages(observer, nextUri);
});
}