Need help in Append JSON file using ContentMerge in Apache NIFi - json

I'm trying to merge a JSON file which has multiple objects. Below is my Oringinal JSON file.
{
"applicant": {
"full-name": "Tyrion Lannister",
"mobile-number" : "8435739739",
"email-id" : "tyrionlannister_casterlyrock#gmail.com"
},
"product": {
"product-category" : "Credit Card",
"product-type" : "Super Value Card - Titanium"
}
}
I will get some more JSON data as below from other source.
{
"flags": {
"duplicate-flag" : "No"
"contact-flag" : "Yes"
}
}
My task is to append the new JSON in the old JSON recods as a new object as below.
{
"applicant": {
"full-name": "Tyrion Lannister",
"mobile-number" : "8435739739",
"email-id" : "tyrionlannister_casterlyrock#gmail.com"
},
"product": {
"product-category" : "Credit Card",
"product-type" : "Super Value Card - Titanium"
},
"flags": {
"duplicate-flag" : "No"
"contact-flag" : "Yes"
}
}
Can someone help to guide, how it can be achieved in NiFi ?

I recommend accumulating your components as flowfile attributes, then forming a merged object with an ExecuteScript processor using JavaScript/ECMAScript. Sometimes there's just no substitute for JavaScript. Something like the following might work:
flowFile = session.get();
if (flowFile != null) {
var OutputStreamCallback = Java.type("org.apache.nifi.processor.io.OutputStreamCallback");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
// Get attributes
var applicant = JSON.parse(flowFile.getAttribute("applicant"));
var product = JSON.parse(flowFile.getAttribute("product"));
var flags = JSON.parse(flowFile.getAttribute("flags"));
// Combine
var merged = {
"applicant": applicant,
"product": product,
"flags": flags
};
// Write output content
flowFile = session.write(flowFile, new OutputStreamCallback(function(outputStream) {
outputStream.write(JSON.stringify(merged, null, "\t").getBytes(StandardCharsets.UTF_8));
}));
session.transfer(flowFile, REL_SUCCESS);
}

Related

Remove a specific JSONObject from JSONArray in groovy

Say I have a JSON request payload like
{
"workflow": {
"approvalStore": {
"sessionInfo": {
"user": "baduser"
},
"guardType": "Transaction"
}
}
}
I get the value of user via
def user = req.get("workflow").get("approvalStore").get("sessionInfo").get("user")
Now, I get a RestResponse approvalList which I store as list and return to caller as return approvalList.json as JSON. All well so far.
Suppose the response (approvalList.json) looks like below JSONArray -
[
{
"objId": "abc2",
"maker": "baduser"
},
{
"objId": "abc1",
"maker": "baduser"
},
{
"objId": "abc4",
"maker": "gooduser"
}
]
Question : How may I filter the approvalList.json so that it doesn't contain entries (objects) that have "maker": "baduser" ? The value passed to maker should essentially be the user variable I got earlier.
Ideal required output -
It's not entirely clear if you always want a single object returned or a list of objects but using collect is going to be the key here:
// given this list
List approvalList = [
[objId: "abc2", maker: "baduser"],
[objId: "abc1", maker: "baduser"],
[objId: "abc4", maker: "gooduser"]
]
// you mentioned you wanted to match a specific user
String user = "baduser"
List filteredList = approvalList.findAll{ it.maker != user}​​​​​​
// wasn't sure if you wanted a single object or a list...
if (filteredList.size() == 1) {
return filteredList[0] as JSON
} else {
return filteredList as JSON
}​
Pretty simple. First parse the JSON into an object, then walk through and test.
JSONObject json = JSON.parse(text)
json.each(){ it ->
it.each(){ k,v ->
if(v=='baduser'){
// throw exception or something
}
}
}

How to convert a MongoDB document to JSON Object

I am trying to make a post request with the MongoDB document returned from find query, as the request body in NodeJS.But on the server I'm getting the Error : Invalid JSON. Below is the document that I'm trying to POST
{
"_id" : ObjectId("5739a6bf3f1b41477570dc89"),
"taskCount" : 2,
"study" : "cod",
"phase" : "mansa2",
"rhimeTaskId" : "5739a6bec4567f6e737fd3db",
"recordId" : "5726f3cfc4567f6e737fc3ab",
"recordStudy" : "codstudy",
"recordPhase" : "mansa2",
"recordLanguage" : "Punjabi",
"recordScript" : "Latin",
"_state" : "CodingComplete",
"tasks" : [
{
"physician" : ObjectId("5739a6bd3f1b41477570dc78"),
"stage" : "Coding",
"result" : {
"cod" : "C15",
"feedback" : {
"narrativeLength" : "Adequate",
"positiveSymptomsIncluded" : "Only Positive",
"certainty" : "High"
},
"keywords" : [
"52 yr male, died of food pipe cancer, suffered pain upper abdomen, investigated,FNAC confirmed Cancer, Put on Chemotherapy, multiple cycles, died at home, had fever with chills occasionally"
]
}
},
{
"physician" : ObjectId("5739a6bd3f1b41477570dc79"),
"stage" : "Coding",
"result" : {
"cod" : "C15",
"feedback" : {
"narrativeLength" : "Inadequate",
"positiveSymptomsIncluded" : "Only Positive",
"certainty" : "High"
},
"keywords" : [
"severe pain abdomen, ultrasonography revealed food pipe cancer, chemotherapy given, died"
]
}
}
],
"__v" : 2
}
and here is the code that I wrote to make the POST request
var MongoClient = require('mongodb').MongoClient;
var request = require('request');
var assert = require('assert');
var cmeprovisioning= 'mongodb://localhost:27017/cmeprovisioning';
MongoClient.connect(cmeprovisioning, function(err, db) {
assert.equal(null, err);
var count=0;
console.log("Connected to cmeprovisioning");
var cursor =db.collection('rhimeReport').find(
{"study":"cod","phase":"mansa2","recordStudy":"codstudy",
"recordPhase":"mansa2","_state":"CodingComplete"
});
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
console.dir(doc);
count=count+1;
request({url: "http://cme.host.net:8081/cme-provisioning/update",
method: "POST",json: true,
headers: {"content-type": "application/json"},
json: doc
},function(e,r,b){
console.log("POST Error "+count+" "+e)
console.log("POST Response "+count+" "+r)
console.log("POST BODY "+count+" "+b)
});
} else {
console.log("Some Error : "+err)
}
});
});
I also tried using JSON.stringify(doc), but still got the Invalid JSON error. Is there a way I can use mongo document returned by the find query and convert it to JSON to make the POST request.
I think those ObjectID is what making it an invalid JSON document.
Here's the actual answer:
If you want to convert a mongo object to JSON object.
There's a utility method in every mongo object toJSON
So you can simply do mongoResponseObject.toJSON() on the response object.
e.g.
Products.findById(id).then(res => {
const jsonRes = res.toJSON();
// Here jsonRes is JSON
})
Alternatively you can directly get the JSON object by using the .lean() like this.
Products.findById(id).lean().then(res => {
// Here res is JSON
})
you need to convert object id to string ie.
var result = {
"_id": ObjectId("5739a6bf3f1b41477570dc89"),
"taskCount": 2,
"study": "cod"
};
//now convert to string
result=result._id.toString();
//now you can use the result
Try this,
var cursor =db.collection('rhimeReport').find(
{"study":"cod","phase":"mansa2","recordStudy":"codstudy",
"recordPhase":"mansa2","_state":"CodingComplete"});
cursor.toString();
......
Hope this help.
Try this in robomongo
var cursor = db.getCollection('X').find({},{})
while(cursor.hasNext()) {
print(JSON.stringify(cursor.next()))
}

How to remove extra column value from jqgrid json data

Free Jqgrid has actions column. colmodel:
{"hidden":false,"label":"","name":"_actions","width":72
,"align":"left","template":"actions","fixed":false,"resizable":true,
"formatoptions":{"editbutton":true,"delbutton":true,"delOptions":{"url":"Delete" }}},
{"label":"Nimetus","name":"Nimi","index":"Nimi","editoptions":{"maxlength":80,"size":80 }
It is populated from remote json data like
{"total":1,
"page":1,
"rows":[{"id":"2ARVELDUSARV", "cell":[null,"2ARVELDUSARV"]},
{"id":"ACME","cell":[null,"ACME"]},
{"id":"KAKSKOERA","cell":[null,"KAKSKOERA"]}
]
}
In cell array first column is not used.
If this column is removed, jqgrid does not render data correctly since this column presence is required as placeholder for actions column.
How to fix this so that jqgrid will accept data without first column:
{"total":1,
"page":1,
"rows":[{"id":"2ARVELDUSARV", "cell":[null,"2ARVELDUSARV"]},
{"id":"ACME","cell":["ACME"]},
{"id":"KAKSKOERA","cell":["KAKSKOERA"]}
]
}
Update
I looked for data format change as recommended in answer.
jqgrid data is created from sql select statement in ASP.NET MVC4 using code below. Web API serializes this to format for json for jqgrid automatically.
How to create result which can serialized to propertyname: value format recommended in answer ?
object GetDataForJqGrid() {
IDbConnection conn;
using (var dataReader = DataAccessBase.ExecuteReader(sql.ToString(), out conn,
CommandBehavior.CloseConnection | CommandBehavior.SingleResult,
sql.GetParameters.ToArray()))
{
var rowList = new List<GridRow>();
var pkeys = DatabasePrimaryKey();
while (dataReader.Read())
{
var pkv = new List<object>();
int offset = 1; // required for actions column
var row = new GridRow
{
id = IdHelper.EncodeId(pkv),
cell = new object[dataReader.FieldCount + offset + imageCount]
};
for (int j = 0; j < dataReader.FieldCount; j++)
row.cell[offset + j] = dataReader.GetValue(j);
rowList.Add(row);
}
return new
{
total = rowList.Count() < rows ? page : page + 1, page,
rows = rowList
};
}
public class GridRow
{
public string id;
public object[] cell;
}
The most easy way would be to chanege the format of data returned from the server to use repeatitems: false style of the data. I mean the usage of
{
"total": 1,
"page": 1,
"rows": [
{ "id": "2ARVELDUSARV", "Nimi": "2ARVELDUSARV" },
{ "id": "ACME", "Nimi": "ACME" },
{ "id": "KAKSKOERA", "Nimi": "KAKSKOERA"}
]
}
or, after adding key: true to the definition of the column Nimi
{
"total": 1,
"page": 1,
"rows": [
{ "Nimi": "2ARVELDUSARV" },
{ "Nimi": "ACME" },
{ "Nimi": "KAKSKOERA"}
]
}
instead of
{
"total": 1,
"page": 1,
"rows": [{
"id": "2ARVELDUSARV",
"cell": ["2ARVELDUSARV"]
}, {
"id": "ACME",
"cell": ["ACME"]
}, {
"id": "KAKSKOERA",
"cell": ["KAKSKOERA"]
}]
}
Alternatively one can use jsonReader: { repeatitems: false } event with your current format of data and add jsonmap: "cell.0" property to, which means getting the first element (index 0) from the array cell:
$("#list").jqGrid({
datatype: "json",
url: "andrus.json",
colModel: [
{ label: "", name: "_actions", template: "actions" },
{ label: "Nimetus", name: "Nimi", jsonmap: "cell.0" }
],
iconSet: "fontAwesome",
jsonReader: { repeatitems: false }
});
see the demo.
I personally would recommend you don't use your original format (cell with array of values) and use just the named property with additional id property (if id value is not included in the item already). If you would do use the solution with jsonmap you should be carefully with changing the order of the columns (using remapColumns) and later reloading of data. You could required to update jsonmap values after the changing the column order. Thus I repeat that I recommend you to change format of data returned from the server.
UPDATED: The Updated part of your question formulate absolutely new question which have no relation with jqGrid. It's pure C# problem. Nevertheless I try to answer, because I use C# too.
What you can do with minimal changes of your code is the following: You should add using System.Dynamic; and using System.Linq; first of all. Then you should replace the code inside of using (...) {...} to about the following
var rowList = new List<dynamic>();
while (dataReader.Read()) {
var row = new ExpandoObject() as IDictionary<string, object>;
for (int j = 0; j < dataReader.FieldCount; j++) {
if (!dataReader.IsDBNull(j)) {
row.Add(dataReader.GetName(j), dataReader.GetValue(j));
}
}
rowList.Add(row);
}
Serializing of rowList will produce the names properties. If you know the primary key of the data, then you can add id property with the corresponding value in the same way (using row.Add("id", IdHelper.EncodeId(pkv))). I don't included the part because the code which you posted is not full and pkv is currently always new List<object>(), which is wrong. If the data have composed key (multiple value set is unique) then you can make string concatenation of the keys using '_' (underscore) as the separator.

JSON.net parsing of dynamic JSON

I have JSON that looks like this:
{
"status": {
"code": 0,
"message": "OK"
},
"data": {
"_idtype": "cusip",
"_id": "00768Y883",
"api": {
"_name": "PortfolioBreakdownsRaw",
"PortfolioDate": "2015-10-12",
"GlobalBondSuperSectorLongSalePositionBreakdown": [
{
"Name": "Municipal",
"Value": "0.57842"
},
{
"Name": "Corporate",
"Value": "1.79649"
},
{
"Name": "Securitized",
"Value": "5.29493"
},
{
"Name": "Cash & Equivalents",
"Value": "166.20776"
}
],
"GlobalBondSuperSectorShortSalePositionBreakdown": [
{
"Name": "Government",
"Value": "0.90557"
}
]
}
}
}
I am able to get the api portion of the response easily:
var jObject = JObject.Parse(json);
var api = jObject["data"]["api"];
From here, I don't what if any arrays will be included in the response. The ultimate goal will be to create a parser that will be able to get the array names (GlobalBondSuperSectorShortSalePositionBreakdown) and as many rows of key-value pairs that it may contain, without first knowing the names such as (GlobalBondSuperSectorShortSalePositionBreakdown) beforehand.
I can't seem to find a good way to loop through the object, determine there are arrays at the api level and then iterate through those to get the values.
Any help would be appreciated.
Here's an example. In this code, the api variable holds a JObject, so we can iterate over its properties. From there, we look at the Type of each property value to see if it is an array or not. If it is, then we can iterate over that array to get the JObjects within it, and extract the Name and Value values that we expect to find there. Does this help?
var jObject = JObject.Parse(json);
var api = jObject["data"]["api"];
foreach (JProperty prop in api.Children<JProperty>())
{
JToken value = prop.Value;
if (value.Type == JTokenType.Array)
{
Console.WriteLine(prop.Name + ": ");
foreach (JObject jo in value.Children<JObject>())
{
Console.WriteLine(" " + jo["Name"] + ": " + jo["Value"]);
}
}
else
{
Console.WriteLine(prop.Name + ": " + value);
}
}
Output:
_name: PortfolioBreakdownsRaw
PortfolioDate: 2015-10-12
GlobalBondSuperSectorLongSalePositionBreakdown:
Municipal: 0.57842
Corporate: 1.79649
Securitized: 5.29493
Cash & Equivalents: 166.20776
GlobalBondSuperSectorShortSalePositionBreakdown:
Government: 0.90557
Fiddle: https://dotnetfiddle.net/XyoXQy
With Linq you can play pretty nice with Json.net:
Here is an easily readable version of the chunk of code that will create two dictionaries out of the JArray properties under the api element:
var api = jObject["data"]["api"];
var arrays = api.Cast<JProperty>().Where(o => o.Value.Type == JTokenType.Array).Select(token => token.Value).ToArray();
var dictionaries = new List<Dictionary<string, string>>();
foreach (var array in arrays)
{
var dictionary = array.ToDictionary(token => token["Name"].Value<string>(), token => token["Value"].Value<string>());
dictionaries.Add(dictionary);
}
alternative:
The same thing, but a shorter, more compact version :
var api = jObject["data"]["api"];
var dictionaries = api
.Cast<JProperty>()
.Where(o => o.Value.Type == JTokenType.Array)
.Select(token => token.Value)
.Select(array => array.ToDictionary(token => token["Name"].Value<string>(), token => token["Value"].Value<string>()));

Actionscript3 parsing json with an object

I have a flash app where in a function I have to parse a json passed like an object by some external API that I can't change.
my json look like this:
{
"prodotti": [
{
"titolo": "test",
"marca": "",
"modello": "",
"cilindrata": "",
"potenza": "",
"alimentazione": "",
"images": {
"img": [
{
"thumb": "admin/uploads/img_usato/small/qekabw95L5WH1ALf6.jpg",
"big": "admin/uploads/img_usato/big/qekabw95L5WH1ALf6.jpg"
},
{
"thumb": "admin/uploads/img_usato/small/default.jpg",
"big": "admin/uploads/img_usato/big/default.jpg"
}
]
}
},
{
"titolo": "Motore Volvo TAMD 74 C",
"marca": "VOLVO PENTA",
"modello": "TAMD 74 C",
"cilindrata": "7.283 cm3",
"potenza": "331 kW a 2600 rpm",
"alimentazione": "Gasolio",
"images": {
"img": [
{
"thumb": "admin/uploads/img_usato/small/PmQwN4t4yp7P1YCWa.jpg",
"big": "admin/uploads/img_usato/big/PmQwN4t4yp7P1YCWa.jpg"
},
{
"thumb": "admin/uploads/img_usato/small/BWkjTGcy3pDM2LKRs.jpg",
"big": "admin/uploads/img_usato/big/BWkjTGcy3pDM2LKRs.jpg"
}
]
}
}
]
}
I want to parse the images inside the object.
The API send me an object not astring or json and I have this function now:
function changeData (prodotto:Object) {
img_container.graphics.clear ();
//here I want to enter and take thumb and big of images!!!
for (var index in prodotto.images.img) {
//trace('index: ' + index);
//trace("thumb: " + index.thumb + ' big: ' + index.big);
}
descrizione.htmlText = prodotto.testo_html;
titolo.text = prodotto.titolo;
alimentazione.text = prodotto.alimentazione;
potenza.text = prodotto.potenza;
cilindrata.text = prodotto.cilindrata;
modello.text = prodotto.modello;
marca.text = prodotto.marca;
}
The function works fine but not for the for loop where I try to take the bug and thumb of my json how can I retrieve this information in this object?
Thanks
I think there is something wrong with how you are setting up the call back but since you didn't show code for the api we can't fix that, plus you stated you have no control over it.
No matter what the issue is it just does not seem correct.
I put together a function that will get all the thumbs and bigs.
You did not state otherwise.
function changeData (prodotto:Object) {
for each(var item in prodotto.prodotti){
trace('')
//trace(prodotto.testo_html);
trace(item.titolo);
trace(item.alimentazione);
trace(item.potenza);
trace(item.cilindrata);
trace(item.modello);
trace(item.marca);
for each( var imgs in item.images.img) {
trace('thumb',imgs.thumb)
trace('big',imgs.big)
}
}
}
I think you need to use a JSON parser. Use the one from this link: https://github.com/mikechambers/as3corelib
1: Add the com folder to your project directory or add it to your default class path.
2: Adapt this code to your liking. I am not sure how you're getting a literal object from the API. It really should just be a string unless you're using some sort of AMF. Regardless...
import com.adobe.serialization.json.*;
var data:String = '{"prodotti":[{"titolo":"test","marca":"","modello":"","cilindrata":"","potenza":"","alimentazione":"","images":{"img":[{"thumb":"admin/uploads/img_usato/small/qekabw95L5WH1ALf6.jpg","big":"admin/uploads/img_usato/big/qekabw95L5WH1ALf6.jpg"},{"thumb":"admin/uploads/img_usato/small/default.jpg","big":"admin/uploads/img_usato/big/default.jpg"}]}},{"titolo":"Motore Volvo TAMD 74 C","marca":"VOLVO PENTA","modello":"TAMD 74 C","cilindrata":"7.283 cm3","potenza":"331 kW a 2600 rpm","alimentazione":"Gasolio","images":{"img":[{"thumb":"admin/uploads/img_usato/small/PmQwN4t4yp7P1YCWa.jpg","big":"admin/uploads/img_usato/big/PmQwN4t4yp7P1YCWa.jpg"},{"thumb":"admin/uploads/img_usato/small/BWkjTGcy3pDM2LKRs.jpg","big":"admin/uploads/img_usato/big/BWkjTGcy3pDM2LKRs.jpg"}]}}]}';
function changeData(data)
{
img_container.graphics.clear();
var obj = JSON.decode(data);
for (var i:int = 0; i < obj.prodotti.length; i++)
{
for (var k in obj.prodotti[i].images.img)
{
trace("Thumb:",obj.prodotti[i].images.img[k].thumb);
trace("Big:",obj.prodotti[i].images.img[k].big);
}
descrizione.htmlText = obj.prodotti[i].testo_html;
titolo.text = obj.prodotti[i].titolo;
alimentazione.text = obj.prodotti[i].alimentazione;
potenza.text = obj.prodotti[i].potenza;
cilindrata.text = obj.prodotti[i].cilindrata;
modello.text = obj.prodotti[i].modello;
marca.text = obj.prodotti[i].marca;
}
}
changeData(data);