MongoDB reduce nested - json

I have a collection of a class that looks something like that:
[
{
"_id": 1,
"title": "dummy title",
"assignments": [
{
"_id": 1,
"name": "a1",
"members": [
{
"_id": 11,
"full_name": "john doe",
"aga": 18
},
{
"_id": 12,
"full_name": "john doe2",
"aga": 18
}
]
}
],
"settings": [
{
"type": "light",
"status": "enabled"
},
{
"type": "flare",
"status": "disabled"
},
{
"type": "toolbar",
"status": "enabled"
}
]
}
]
I have 2 nested documents here "assignments" which have a nested "members"
and "settings". the result i want should look something like that:
{
"_id": 1,
"title": "dummy title",
"assignments": [
{
"_id": 1,
"name": "a1",
"member_ids": [11, 18]
}
],
"active_settings": ["light", "toolbar"]
}
Meaning in each "assignment" I should only return the ids of the members and not the whole member data. and in settings I should only return the settings that are set to "active"
is it possible?
Playground here:
https://mongoplayground.net/p/Le4BdTm_gOv

You can try with $map to go one by one. $mergeObjects helps to merge the output value with same object
[
{
$project: {
title: 1,
assignments: {
$map: {
input: "$assignments",
in: {
$mergeObjects: [
"$$this",
{
members: {
$map: {
input: "$$this.members",
in: "$$this._id"
}
}
}
]
}
}
},
active_settings: {
$reduce: {
input: "$settings",
initialValue: [],
in: {
$cond: [
{
$eq: [
"$$this.status",
"enabled"
]
},
{
$setUnion: [
"$$value",
[
"$$this.type"
]
]
},
"$$value"
]
}
}
}
}
}
]
Working Mongo playground

You can try,
get assignments member ids using $map and $reduce
get active_settings using $reduce
db.collection.aggregate([
{
$project: {
_id: 1,
title: 1,
assignments: {
$map: {
input: "$assignments",
in: {
_id: "$$this._id",
name: "$$this.name",
memberIds: {
$reduce: {
input: "$$this.members",
initialValue: [],
in: { $concatArrays: ["$$value", ["$$this._id"]] }
}
}
}
}
},
active_settings: {
$reduce: {
input: "$settings",
initialValue: [],
in: {
$cond: [
{ $eq: ["$$this.status", "enabled"] },
{ $concatArrays: ["$$value", ["$$this.type"]] },
"$$value"
]
}
}
}
}
}
])
Playground

Related

Sequelize - Nested Query Condition to Top Level

I want to query a database of promotions that don't have phone number in the blacklist of promotion.
my project use sequelize.org for manage database.
Mock Database http://sqlfiddle.com/#!9/438587/1
Association
Promotion Model
this.belongsToMany(models.CustomerList, {
through: models.BlacklistPromotion,
as: 'Blacklist',
});
this.belongsToMany(models.CustomerList, {
through: models.WhitelistPromotion,
as: 'Whitelist',
});
CustomerList Model
this.belongsToMany(models.CustomerPhoneNumber, {through: models.CustomerListPhoneNumber,});
CustomerPhoneNumber Model
this.belongsToMany(models.CustomerList, { through: models.CustomerListPhoneNumber});
Code sequelize findAndCountAll Promotion
await Promotion.findAndCountAll({
where : {}, <-------- i want query here or somewhere.
include :[
{
model: CustomerList,
as: 'Blacklist',
include :[{
model: CustomerPhoneNumber,
as: 'CustomerPhoneNumbers',
attributes: ['phone_number'],
through: {
attributes: [],
}
}],
through: {
attributes: [],
},
},
{
model: CustomerList,
as: 'Whitelist',
include :[{
model: CustomerPhoneNumber,
as: 'CustomerPhoneNumbers',
attributes: ['phone_number'],
through: {
attributes: [],
}
}],
through: {
attributes: [],
},
}
]
})
when run query
{
"Promotion": [
{
"id": 1,
"title": "Promotion1"
},
{
"id": 2,
"title": "Promotion2"
},
{
"id": 3,
"title": "Promotion3",
"Blacklist": [
{
"id": 1,
"name": "Group5",
"CustomerPhoneNumbers": [
{
"id": 5,
"phone_number": "0877777777"
},
{
"id": 6,
"phone_number": "0888888888"
},
{
"id": 7,
"phone_number": "0899999999"
}
]
}
]
}
]
}
try query
where : {
'$Blacklist.CustomerPhoneNumbers.phone_number$': {
[Op.ne]: '0877777777',
}
}
// output
{
"Promotion": [
{
"id": 1,
"title": "Promotion1"
},
{
"id": 2,
"title": "Promotion2"
},
{
"id": 3,
"title": "Promotion3",
"Blacklist": [
{
"id": 1,
"name": "Group5",
"CustomerPhoneNumbers": [
{
"id": 6,
"phone_number": "0888888888"
},
{
"id": 7,
"phone_number": "0899999999"
}
]
}
]
}
]
}
but i want to remove promotion3
What do you expect to happen?
Find solution query a database of promotions that don't have phone number in the blacklist of promotion.
I want to get:
{
"Promotion": [
{
"id": 1,
"title": "Promotion1"
},
{
"id": 2,
"title": "Promotion2"
},
]
}

Make node tree with recursive table with Express and Mongo

I'm working in a REST api with ExpressJS and Mongo and I have a collection with N quantity of levels.
So to solve this problem I'm using an recursive table (or collection) in mongo where a field is the id and every register has a parent_id which is at the same level as it's childs.
To explain better this, here is an E-R representation
So as you se, mongo will save the data like this json (accounts level 0 has null parent)
[
{ "id": "45TYYU", "parent_id": null, "name":"account 1", "type": 1, "category": 1 },
{ "id": "45TYYXT", "parent_id": "45TYYU", "name":"account 2", "type": 1, "category": 1 },
{ "id": "45TYYPZ", "parent_id": "45TYYU", "name":"account 3", "type": 1, "category": 1 },
{ "id": "45TYYPZRE", "parent_id": "45TYYPZ", "name":"account 4", "type": 1, "category": 1 },
{ "id": "45TYYPZSX", "parent_id": "45TYYPZ", "name":"account 5", "type": 1, "category": 1 },
{ "id": "45TYYPZGP", "parent_id": "45TYYXT", "name":"account 6", "type": 1, "category": 1 }
]
account 2 and account 3 are children of account 1, while account 4 and account 5 are children of account tree and account 6 is child of account 2 ... but every register is at the same logical level only identifying through parent_id.
so I need to transform this data into a GET method to restructure it like this:
[
{
"id": "45TYYU",
"parent_id": null,
"name":"account 1",
"type": 1,
"category": 1,
"children": [
{
"id": "45TYYXT",
"parent_id": "45TYYU",
"name":"account 2",
"type": 1,
"category": 1,
"children": [
{ "id": "45TYYPZGP", "parent_id": "45TYYXT", "name":"account 6", "type": 1, "category": 1 }
]
},
{
"id": "45TYYPZ",
"parent_id": "45TYYU",
"name":"account 3",
"type": 1,
"category": 1,
"children": [
{ "id": "45TYYPZRE", "parent_id": "45TYYPZ", "name":"account 4", "type": 1, "category": 1 },
{ "id": "45TYYPZSX", "parent_id": "45TYYPZ", "name":"account 5", "type": 1, "category": 1 }
]
}
]
},
{
"id": "45TFJK",
"parent_id": null,
"name":"account 7",
"type": 1,
"category": 1,
"children": [
{
"id": "47HJJT",
"parent_id": "45TFJK",
"name":"account 8",
"type": 1,
"category": 1
},
{
"id": "47YHJU",
"parent_id": "45TFJK",
"name":"account 8",
"type": 1,
"category": 1
}
]
}
]
Yes... the parents level 0 has null parent_id and I want to put it's children inside an array called "children" and then send like this in the GET response to my UI
What is the best way to do this in expressJS?
Is there a library or component out there that allows me to do this?
Thank you
You can use $graphLookup and other useful array operators,
$match filter that records only have parent_id is null
$graphLookup to get child records and depth number in depthField level
$unwind deconstruct children array and allow to not remove empty children
$sort by depth level field level in descending order
$group by id field and reconstruct children array
db.collection.aggregate([
{ $match: { parent_id: null } },
{
$graphLookup: {
from: "collection",
startWith: "$id",
connectFromField: "id",
connectToField: "parent_id",
depthField: "level",
as: "children"
}
},
{
$unwind: {
path: "$children",
preserveNullAndEmptyArrays: true
}
},
{ $sort: { "children.level": -1 } },
{
$group: {
_id: "$id",
parent_id: { $first: "$parent_id" },
name: { $first: "$name" },
type: { $first: "$type" },
category: { $first: 1 },
children: { $push: "$children" }
}
},
$addFields now find the nested level children and allocate to its level,
$reduce to iterate loop of children array.
initialize default field level default value is -1, presentChild is [], prevChild is [] for the conditions purpose
$let to initialize fields:
prev as per condition if both level are equal then return prevChild otherwise return presentChild
current as per condition if both level are equal then return presentChild otherwise []
in to return level field and prevChild field from initialized fields
presentChild $filter children from prev array and return, merge current objects with children array using $mergeObjects and concat with current array of let using $concatArrays
$addFields to return only presentChild array because we only required that processed array
{
$addFields: {
children: {
$reduce: {
input: "$children",
initialValue: { level: -1, presentChild: [], prevChild: [] },
in: {
$let: {
vars: {
prev: {
$cond: [
{ $eq: ["$$value.level", "$$this.level"] },
"$$value.prevChild",
"$$value.presentChild"
]
},
current: {
$cond: [{ $eq: ["$$value.level", "$$this.level"] }, "$$value.presentChild", []]
}
},
in: {
level: "$$this.level",
prevChild: "$$prev",
presentChild: {
$concatArrays: [
"$$current",
[
{
$mergeObjects: [
"$$this",
{
children: {
$filter: {
input: "$$prev",
as: "e",
cond: { $eq: ["$$e.parent_id", "$$this.id"] }
}
}
}
]
}
]
]
}
}
}
}
}
}
}
},
{
$addFields: {
id: "$_id",
children: "$children.presentChild"
}
}
])
Playground
#turivishal Im using same schema in backend nodejs im getting only show the
null object not for a parent child relation using same aggregration
this.tickets.aggregate([
{
$match: {
parent_id: null
}
},
{
$graphLookup: {
from: "collection",
startWith: "$id",
connectFromField: "id",
connectToField: "parent_id",
depthField: "level",
as: "children"
}
},
{
$unwind: {
path: "$children",
preserveNullAndEmptyArrays: true
}
},
{
$sort: {
"children.level": -1
}
},
{
$group: {
_id: "$id",
parent_id: {
$first: "$parent_id"
},
name: {
$first: "$name"
},
type: {
$first: "$type"
},
category: {
$first: 1
},
children: {
$push: "$children"
}
}
},
{
$addFields: {
children: {
$reduce: {
input: "$children",
initialValue: {
level: -1,
presentChild: [],
prevChild: []
},
in: {
$let: {
vars: {
prev: {
$cond: [
{
$eq: [
"$$value.level",
"$$this.level"
]
},
"$$value.prevChild",
"$$value.presentChild"
]
},
current: {
$cond: [
{
$eq: [
"$$value.level",
"$$this.level"
]
},
"$$value.presentChild",
[]
]
}
},
in: {
level: "$$this.level",
prevChild: "$$prev",
presentChild: {
$concatArrays: [
"$$current",
[
{
$mergeObjects: [
"$$this",
{
children: {
$filter: {
input: "$$prev",
as: "e",
cond: {
$eq: [
"$$e.parent_id",
"$$this.id"
]
}
}
}
}
]
}
]
]
}
}
}
}
}
}
}
},
{
$addFields: {
children: "$children.presentChild"
}
}
]).then((result) => {
console.log('test',result);
// callback(result);
}).catch((error) => {
callback(error);
});
output:
[
{
_id: '45TYYU',
parent_id: null,
name: 'account 1',
type: 1,
category: 1,
children: []
},
{
_id: '45TYYUA',
parent_id: null,
name: 'account 1',
type: 1,
category: 1,
children: []
}
]

Recurse for object if exists in JQ

I have the following structure:
{
"hits":
[
{
"_index": "main"
},
{
"_index": "main",
"accordions": [
{
"id": "1",
"accordionBody": "body1",
"accordionInnerButtonTexts": [
"button11",
"button12"
]
},
{
"id": "2",
"accordionBody": "body2",
"accordionInnerButtonTexts": [
"button21",
"button22"
]
}
]
}
]
}
I want to get to this structure:
{
"index": "main"
}
{
"index": "main",
"accordions":
[
{
"id": "1",
"accordionBody": "body1",
"accordionInnerButtonTexts": [
"button11",
"button12"
]
},
{
"id": "2",
"accordionBody": "body2",
"accordionInnerButtonTexts": [
"button21",
"button22"
]
}
]
}
Which means that I always want to include the _index-field as index, and I want to include the whole accordions-list IF IT EXISTS in the object. Here is my attempt:
.hits[] | {index: ._index, accordions: recurse(.accordions[]?)}
It does not produce what I want:
{
"index": "main",
"accordions": {
"_index": "main"
}
}
{
"index": "main",
"accordions": {
"_index": "main",
"accordions": [
{
"id": "1",
"accordionBody": "body1",
"accordionInnerButtonTexts": [
"button11",
"button12"
]
},
{
"id": "2",
"accordionBody": "body2",
"accordionInnerButtonTexts": [
"button21",
"button22"
]
}
]
}
}
{
"index": "main",
"accordions": {
"id": "1",
"accordionBody": "body1",
"accordionInnerButtonTexts": [
"button11",
"button12"
]
}
}
{
"index": "main",
"accordions": {
"id": "2",
"accordionBody": "body2",
"accordionInnerButtonTexts": [
"button21",
"button22"
]
}
}
It seems to create a list of all different permutations given by mixing the objects. This is not what I want. What is the correct jq command, and what is my mistake?
The problem as stated does not require any recursion. Using your attempt as a model, one could in fact simply write:
.hits[]
| {index: ._index}
+ (if has("accordions") then {accordions} else {} end)
Or, with quite different semantics:
.hits[] | {index: ._index} + . | del(._index)

Making complex Array output using JOLT

I am trying to transform json to complex array. The inner array of INVLOC is not generating the way I want.
My input json is :
{
"valid": "true",
"message": "",
"data": {
"ARINVT01": [
{
"firstName": "andrew",
"lastname": "Gilly",
"INVLOC": {
"data": [
{
"mmm": "MAIN",
"nnn": "0.000"
},
{
"mmm": "A1",
"nnn": "0.000"
}
]
}
},
{
"firstName": "Jack",
"lastname": "watson",
"INVLOC": {
"data": [
{
"mmm": "MAIN",
"nnn": "0.000"
}
]
}
}
]
}
}
Expected output :
[
{
"FNAME": "andrew",
"LNAME": "Gilly",
"INVBALANCES": [
{
"user": "MAIN",
"CURBAL": "0.000"
},
{
"user": "A1",
"CURBAL": "0.000"
}
]
},
{
"FNAME": "Jack",
"LNAME": "watson",
"INVBALANCES": [
{
"user": "A1",
"CURBAL": "0.000"
}
]
}
]
My spec :
[
{
"operation": "shift",
"spec": {
"data": {
"ARINVT01": {
"*": {
"firstName": "[&1].FNAME",
"lastname": "[&1].LNAME",
"INVLOC": {
"data": {
"*": {
"mmm": "[&1].INVBALANCES.[#1].user",
"nnn": "[&1].INVBALANCES.[#1].CURBAL"
}
}
}
}
}
}
}
}
]
Getting output :
[ {
"FNAME" : "andrew",
"LNAME" : "Gilly",
"INVBALANCES" : [ {
"user" : "MAIN"
}, {
"CURBAL" : "0.000"
} ]
}, {
"INVBALANCES" : [ {
"user" : "A1"
}, {
"CURBAL" : "0.000"
} ],
"FNAME" : "Jack",
"LNAME" : "watson"
} ]
Can someone Help me with my spec?
And also if I can get some sort of tutorial in understanding the array transformation of JOLT.
Spec
[
{
"operation": "shift",
"spec": {
"data": {
"ARINVT01": {
"*": { // array index of ARINVT01
//
// &0 is "firstName"
// &1 is the current array index of ARINVT01
"firstName": "[&1].FNAME",
"lastname": "[&1].LNAME",
"INVLOC": {
"data": {
"*": { // array index of data array
//
// &0 is "mmm"
// &1 is the current array index of the data array
// &2 is "data"
// &3 is "INVLOC"
// &4 is the current array index of ARINVT01
// basically the "&" "number" logic is
// how many levels to walk back up the document
// to lookup the currently matched value
"mmm": "[&4].INVBALANCES.[&1].user",
"nnn": "[&4].INVBALANCES.[&1].CURBAL"
}
}
}
}
}
}
}
}
]

Re-arrange JSON file (using adjacency matrix)

I have a json file that looks like this:
[
{
"id": 1,
"country": "Greece",
"names": [
"Alex",
"Betty",
"Catherine",
"Dave",
"Edward",
"Frank",
"George",
"Helen",
"Irene"
]
},
{
"id": 2,
"country": "US",
"names": [
"John",
"Alex",
"Edward",
"Kate",
"Robert",
"Irene",
"Tim",
"Sam"
]
},
{
"id": 3,
"country": "France",
"names": [
"Helen",
"Kate",
"Louise",
"Tim",
"Catherine",
"Arthur",
"Frank",
"Natalie",
"Dave"
]
},
{
"id": 4,
"country": "India",
"names": [
"Ritesh",
"Alex",
"Betty",
"Robert"
]
},
{
"id": 5,
"country": "India",
"names": [
"Nafeez",
"Tom",
"Natalie",
"Gunar",
"Louise",
"Arthur"
]
}
]
I want it to be "name centered" and look like this:
{
"groups": [
{
"gr_id":1
"name":"Alex",
"country":"Greece"
},
.........
{
"gr_id":1
"name":"Irene",
"country":"Greece"
},
{
"gr_id":2
"name":"John",
"country":"US"
..........
{
"gr_id":2
"name":"Sam",
"country":"US"
},
{
"gr_id":3
"name":"Helen",
"country":"France"
},
.........
{
"gr_id":3
"name":"Dave",
"country":"France"
},
{
"gr_id":4
"name":"Ritesh",
"country":"India"
},
........
{
"gr_id":4
"name":"Robert",
"country":"India"
},
{
"gr_id":5
"name":"Nafeez",
"country":"India"
},
...........
{
"gr_id":5
"name":"Arthur",
"country":"India"
}
],
"links": [
{
"source":"Alex"
"target":"Irene",
"count":1
"country":"Greece"
},
...
{
"source":"Alex"
"target":"Arthur",
"count":0
"country":"India"
},
...
]
}
For count in Links I have an adjacency matrix for each country/name (csv format) like this :screenshot of csv file (ad. matrix for India)
This json is just an example. I have much bigger file (I need it for D3 graph visualization)
Reduce() and map() work perfectly for this. This basically takes each item and then maps over the names, appending the results of map() to an array:
let obj = {}
obj.groups = json.reduce(
(acc, curr) => acc.concat(curr.names.map(
item => ({gr_id: curr.id, country: curr.country, name: item})
)), [])
console.log(obj)
// { groups:
// [ { gr_id: 1, country: 'Greece', name: 'Alex' },
// { gr_id: 1, country: 'Greece', name: 'Betty' },
// ...etc
// ]
// }