Splitting Json to multiple jsons in NIFI - json

I have the below json file which I want to split in NIFI
Input:
[ {
"id" : 123,
"ticket_id" : 345,
"events" : [ {
"id" : 3322,
"type" : "xyz"
}, {
"id" : 6675,
"type" : "abc",
"value" : "sample value",
"field_name" : "subject"
}, {
"id" : 9988,
"type" : "abc",
"value" : [ "text_file", "json_file" ],
"field_name" : "tags"
}]
}]
and my output should be 3 different jsons like below:
{
"id" : 123,
"ticket_id" : 345,
"events.id" :3322,
"events.type":xyz
}
{
"id" : 123,
"ticket_id" : 345,
"events.id" :6675,
"events.type":"abc",
"events.value": "sample value"
"events.field_name":"subject"
}
{
"id" : 123,
"ticket_id" : 345,
"events.id" :9988,
"events.type":"abc",
"events.value": "[ "text_file", "json_file" ]"
"events.field_name":"tags"
}
I want to know can we do it using splitjson? I mean can splitjson split the json based on the array of json objects present inside the json?
Please let me know if there is a way to achieve this.

If you want 3 different flow files, each containing one JSON object from the array, you should be able to do it with SplitJson using a JSONPath of $ and/or $.*

Using reduce function:
function split(json) {
return json.reduce((acc, item) => {
const events = item.events.map((evt) => {
const obj = {id: item.id, ticket_id: item.ticket_id};
for (const k in evt) {
obj[`events.${k}`] = evt[k];
}
return obj;
});
return [...acc, ...events];
}, []);
}
const input = [{"id":123,"ticket_id":345,"events":[{"id":3322,"type":"xyz"},{"id":6675,"type":"abc","value":"sample value","field_name":"subject"},{"id":9988,"type":"abc","value":["text_file","json_file"],"field_name":"tags"}]}];
const res = split(input);
console.log(res);

Related

Extract values from dictionary and create list of tuples in Robot Framework

I am trying to extract values from a dictionary and return as list of tuples in Robot Framework. Would you suggest how to go about it?
my JSON content looks like this :
{
"_embedded" : {
"products" : [ {
"id" : "BMHY2IZB",
"Name" : "ANR",
"securityType" : "type1",
"_links" : {
"self" : {
"href" : "https://test.com/v1/products/BMHY2IZB"
},
"relatedproducts" : {
"href" : "https://test.com/v1/products/BMHY2IZB/related"
}
}
}, {
"id" : "FXDNZBW",
"Name" : "STREPLC",
"securityType" : "ANV",
"_links" : {
"self" : {
"href" : "https://test.com/v1/products/FXDNZBW"
},
"relatedProducts" : {
"href" : "https://test.com/v1/products/FXDNZBW/related"
}
}
} ]
},
"page" : {
"size" : 20,
"totalElements" : 2,
"totalPages" : 1,
"number" : 0
}
}
And with the below code from Robot Framework:
${fileload} = get file ../../Resources/Sample.json
${json}= to json ${fileload}
${PRD}= get from dictionary ${json} _embedded
${products}= get from dictionary ${PRD} products
${PRDlist} = create list
: FOR ${product} in #{products}
\ append to list ${PRDlist} ${product}
log to console ${PRDlist}
I get a response like this :
[{'id': 'BMHY2IZB', 'Name': 'ANR', 'securityType': 'type1', '_links':
{'self': {'href': 'https://test.com/v1/products/BMHY2IZB'},
'relatedproducts': {'href': 'https://test.com/v1/products/BMHY2
IZB/related'}}}, {'id': 'FXDNZBW', 'Name': 'STREPLC', 'securityType':
'ANV',
'_links': {'self': {'href': 'https://test.com/v1/products/FXDNZBW'},
'relatedProducts': {'href':
'https://test.com/v1/products/FXDNZBW/related'}}}]
But I wanted selected values returned as list of tuples :
[{'BMHY2IZB','ANR','type1'},{'FXDNZBW','STREPLC','ANV'}]
This seem to work :
import os
import collections
def APIResponse(dict):
prds = dict.get('_embedded')
products = prds.get('products')
l2 = []
for i in range(len(products)):
v1= products[i].get('id')
v2= products[i].get('Name')
v3= products[i].get('securityType')
l1 = (v1,v2,v3)
l2.append(l1)
return l2

Search inside JSON with Elastic

I have an index/type in ES which has the following type of records:
body "{\"Status\":\"0\",\"Time\":\"2017-10-3 16:39:58.591\"}"
type "xxxx"
source "11.2.21.0"
The body field is a JSON.So I want to search for example the records that have in their JSON body Status:0.
Query should look something like this(it doesn't work):
GET <host>:<port>/index/type/_search
{
"query": {
"match" : {
"body" : "Status:0"
}
}
}
Any ideas?
You have to change the analyser settings of your index.
For the JSON pattern you presented you will need to have a char_filter and a tokenizer which remove the JSON elements and then tokenize according to your needs.
Your analyser should contain a tokenizer and a char_filter like these ones here:
{
"tokenizer" : {
"type": "pattern",
"pattern": ","
},
"char_filter" : [ {
"type" : "mapping",
"mappings" : [ "{ => ", "} => ", "\" => " ]
} ],
"text" : [ "{\"Status\":\"0\",\"Time\":\"2017-10-3 16:39:58.591\"}" ]
}
Explanation: the char_filter will remove the characters: { } ". The tokenizer will tokenize by the comma.
These can be tested using the Analyze API. If you execute the above JSON against this API you will get these tokens:
{
"tokens" : [ {
"token" : "Status:0",
"start_offset" : 2,
"end_offset" : 13,
"type" : "word",
"position" : 0
}, {
"token" : "Time:2017-10-3 16:39:58.591",
"start_offset" : 15,
"end_offset" : 46,
"type" : "word",
"position" : 1
} ]
}
The first token ("Status:0") which is retrieved by the Analyze API is the one you were using in your search.

Mongolite group by/aggregate on JSON object

I have a json document like this on my mongodb collection:
Updated document:
{
"_id" : ObjectId("59da4aef8c5d757027a5a614"),
"input" : "hi",
"output" : "Hi. How can I help you?",
"intent" : "[{\"intent\":\"greeting\",\"confidence\":0.8154089450836182}]",
"entities" : "[]",
"context" : "{\"conversation_id\":\"48181e58-dd51-405a-bb00-c875c01afa0a\",\"system\":{\"dialog_stack\":[{\"dialog_node\":\"root\"}],\"dialog_turn_counter\":1,\"dialog_request_counter\":1,\"_node_output_map\":{\"node_5_1505291032665\":[0]},\"branch_exited\":true,\"branch_exited_reason\":\"completed\"}}",
"user_id" : "50001",
"time_in" : ISODate("2017-10-08T15:57:32.000Z"),
"time_out" : ISODate("2017-10-08T15:57:35.000Z"),
"reaction" : "1"
}
I need to perform group by on intent.intent field and I'm using Rstudio with mongolite library.
What I have tried is :
pp = '[{"$unwind": "$intent"},{"$group":{"_id":"$intent.intent", "count": {"$sum":1} }}]'
stats <- chat$aggregate(
pipeline=pp,
options = '{"allowDiskUse":true}'
)
print(stats)
But it's not working, output for above code is
_id count
1 NA 727
If intent attribute type is string and keep the object as string.
We can split it to array with \" and use third item of array.
db.getCollection('test1').aggregate([
{ "$project": { intent_text : { $arrayElemAt : [ { $split: ["$intent", "\""] } ,3 ] } } },
{ "$group": {"_id": "$intent_text" , "count": {"$sum":1} }}
])
Result:
{
"_id" : "greeting",
"count" : 1.0
}

Jmeter - How to remove id from given json

I'm comparing 2 jsons with JSR223 Assertion and I'm willing to remove the ids from all levels from the response json:
{
"id" : 52906,
"name1" : "559812 Company name1",
"name2" : "559812 Company name2",
"country" : "DE",
"interests" : {
"id" : 848675,
"description" : false
},
"emails" : [ {
"id" : 904881,
"address" : "559812#gmail.com"
} ],
...
I'm using the following Groovy code:
def slurper2 = new JsonSlurper();
def jsonResponse = slurper2.parseText(prev.getResponseDataAsString());
jsonResponse.rows.findAll { it.remove("id") };
But it doesn't work - Please advise.
I don't really understand where you got this rows bit as I don't see any JSON Array named "rows" in your response.
If you want to remove all "id" attributes you can use the following approach:
def response = prev.getResponseDataAsString()
def responseWithoutIds = response.replaceAll("\"id\"[ ]*:[^,}\\]]*[,]?", "")
// do what you need with the modified response, i.e. store it into a JMeter Variable
vars.put("responseWithoutIds", responseWithoutIds)
Demo:
References:
String.replaceAll() method JavaDoc
Pattern class JavaDoc
Apache Groovy - Why and How You Should Use It
import groovy.json.*;
def s='''{
"id" : 52906,
"name1" : "559812 Company name1",
"name2" : "559812 Company name2",
"country" : "DE",
"interests" : {
"id" : 848675,
"description" : false
},
"emails" : [ {
"id" : 904881,
"address" : "559812#gmail.com"
} ]
}
'''
def removeAttr(node, attr){
if(node instanceof Map){
node.remove(attr)
node.each{ k,v-> removeAttr(v, attr) }
}else if(node instanceof List){
node.each{ i-> removeAttr(i, attr) }
}
}
def slurper2 = new JsonSlurper();
def jsonResponse = slurper2.parseText(s);
removeAttr(jsonResponse,"id")
println JsonOutput.prettyPrint(JsonOutput.toJson(jsonResponse))

Angularjs: Filtering JSON

The structure of the json response I have to work with is giving me all sorts of trouble.
I would like to filter children elements based on the entityType field.
For example I want only element of Folder type.
JSON:
{
"id" : "df1d2550-1442-41b8-9588-785e229c5728",
"path" : "",
"name" : "",
"entityType" : "Folder",
"children" : {
"child1" : {
"id" : "02427ae5-364e-47d0-8998-0876c596d586",
"name" : "child1",
"entityType" : "Book"
},
"child2" : {
"id" : "2bcef8b3-d3a3-410e-a481-a69ec7dce24d",
"name" : "child2",
"entityType" : "Letter"
},
"child3" : {
"id" : "12ee8334-d596-4b59-a09d-c286117f3966",
"name" : "child2",
"entityType" : "Book"
},
"Art" : {
"id" : "e3f2980e-433c-4eaa-b444-ed9702949ffc",
"name" : "Art",
"entityType" : "Folder"
},
"English" : {
"id" : "8fe0f14a-6f76-41aa-9ab3-3e63cd5a900b",
"name" : "English",
"entityType" : "Folder"
}
},
"properties" : { },
"ancestors" : [ ]
};
The JS code:
$scope.isFolder = function(item) {
return item.entityType === "Folder";
};
HTML
<div ng-repeat="item in library.children | filter:isFolder">
<pre>{{item.name}} - {{item.entityType}}</pre>
</div>
This code will display all the children when I only want 2.
Any idea what I am doing wrong ?
Plunker
Thanks
The reason your filter doesn't work is that Filter works on arrays but you have an object literal.
So you can either convert your object literal into an array like:
{
"id" : "df1d2550-1442-41b8-9588-785e229c5728",
"path" : "",
"name" : "",
"entityType" : "Folder",
"children" : [
{
"id" : "02427ae5-364e-47d0-8998-0876c596d586",
"name" : "child1",
"entityType" : "Book"
},
{
"id" : "2bcef8b3-d3a3-410e-a481-a69ec7dce24d",
"name" : "child2",
"entityType" : "Letter"
},
{
"id" : "12ee8334-d596-4b59-a09d-c286117f3966",
"name" : "child2",
"entityType" : "Book"
},
{
"id" : "e3f2980e-433c-4eaa-b444-ed9702949ffc",
"name" : "Art",
"entityType" : "Folder"
},
{
"id" : "8fe0f14a-6f76-41aa-9ab3-3e63cd5a900b",
"name" : "English",
"entityType" : "Folder"
}
],
"properties" : { },
"ancestors" : [ ]
};
You can use below code to convert children's properties to an array:
var arr = [];
for (var key in library.children) {
if (obj.hasOwnProperty(key)) {
obj[key]["key"] = key
arr.push(obj[key]);
}
};
library.children = arr;
Or create your own filter than takes in the object literal as callum linington do.
I went through the documentation. Although it is not as explicit as you would expect, looks like the filter function will only work on Array elements and not objects.
In your example, library.children is an object, not an array. Hence, the filter function is never called (which you can check by adding a console statement inside the function).
The specific part in the documentation which mentions that the filter function is called on Array can be found here. You will find it in the "Argument" section, "expression" row - look for the function description. It says that the function is called on Arrays and not Objects, which is your scenario.
So you can easily solve this by writing your own filter here is your adapted plnkr
Simple filter:
app.filter('byKey', function () {
return function (array, key, value) {
var take = [];
angular.forEach(array, function (e) {
if (e[key] === value){
take.push(e);
}
});
return take;
}
});
Simple usage:
<div ng-repeat="item in library.children | byKey: 'entityType' : 'Folder'">
<pre>{{item.name}} - {{item.entityType}}</pre>
</div>
The breakdown:
return function (input, arg1.....argn) {
}
The above return function in the filter is what drives the filter, the input is what you specify on the left side of the | (pipe), in your case: item in library.children.
It's the right side that confuses people. The first string should be the filter name, in my case byKey as that is what I have named it. Each time you use : you are allowing yourself to pass an arg.
So, I have specified this:
return function (array, key, value) {}
The first argument - array, is specified by the left of the |. The remaining args are specified each time you use :, in your case
byKey : 'entityType' : 'Folder'
byKey = filter, 'entityType' = key, 'Folder' = the value of the key