how to convert datetime from epoc to utc in mongodb - json

Hi all I have a collection name test_data and in this collect I have data like in given formate
Data = [{
"_id" : "53fe7ae0ef038fee879263d5",
"chanal_id" : "can1",
"instrumentnam" : "New Demo Device",
"siteid" : "53fe741d66455060e003c2d8",
"locationid" : "53fe74a866455060e003c2db",
"timestamp" : 1406916000,
"status" : "online",
"msgtype" : "demographics",
"fps" : "5",
"people" : [
{
"id" : 1,
"age" : 23,
"gender" : "male",
"ethnicity" : null,
"mood" : "happy",
"ethnicity_mod" : "White",
"mood_mod" : "Happy",
"gender_mod" : "Female",
"age_mod" : 22
}
],
"dateUTC" : 1409190000
},
{
"_id" : "53fe7ae0ef038fee879263d6",
"chanel_id" : "can1",
"instrumentnam" : "New Demo Device",
"siteid" : "53fe741d66455060e003c2d8",
"locationid" : "53fe74a866455060e003c2db",
"timestamp" : 1406917800,
"status" : "online",
"msgtype" : "demographics",
"fps" : "5",
"people" : [
{
"id" : 2,
"age" : 16,
"gender" : "female",
"ethnicity" : null,
"mood" : "happy",
"ethnicity_mod" : "White",
"mood_mod" : "Happy"
}
],
"dateUTC" : 1409186528
},
{
"_id" : "53fe7ae0ef038fee879263d7",
"chanal_id" : "can1",
"instrumentnam" : "New Demo Device",
"siteid" : "53fe741d66455060e003c2d8",
"locationid" : "53fe74a866455060e003c2db",
"timestamp" : 1406918800,
"status" : "online",
"msgtype" : "demographics",
"fps" : "5",
"people" : [
{
"id" : 3,
"age" : 33,
"gender" : "male",
"ethnicity" : null,
"mood" : "happy",
"ethnicity_mod" : "White"
}
],
"dateUTC" : 1409186528
}
,
{
"_id" : "53fe7ae0ef038fee879263d8",
"chanal_id" : "can1",
"instrumentnam" : "New Demo Device",
"siteid" : "53fe741d66455060e003c2d8",
"locationid" : "53fe74a866455060e003c2db",
"timestamp" : 1406919800,
"status" : "online",
"msgtype" : "demographics",
"fps" : "5",
"people" : [
{
"id" : 4,
"age" : 45,
"gender" : "female",
"ethnicity" : null,
"mood" : "happy",
"age_mod" : 5
}
],
"dateUTC" : 1409186528
},
{
"_id" : "53fe7ae0ef038fee879263d9",
"chanal_id" : "can1",
"instrumentnam" : "New Demo Device",
"siteid" : "53fe741d66455060e003c2d8",
"locationid" : "53fe74a866455060e003c2db",
"timestamp" : 1406920800,
"status" : "online",
"msgtype" : "demographics",
"fps" : "5",
"people" : [
{
"id" : 5,
"age" : 14,
"gender" : "female",
"ethnicity" : null,
"mood" : "angry"
}
],
"dateUTC" : 1409186528
},
{
"_id" :"53ff927266455060e003c2e0",
"chanal_id" : "rtsp://192.168.1.83:554/live.sdp",
"instrumentnam" : "Test Device",
"siteid" : "53f62abf66455068373665ff",
"locationid" : "53f62bb86645506837366603",
"timestamp" : 1409258116,
"status" : "online",
"msgtype" : "demographics",
"fps" : "32",
"people" : [],
"dateUTC" : 1409258098
}]
which is in json format,
What I am trying to do
1. I want to convert timestamp from epoc time to utc time in linux system. and
2. I want to convert this json data to csv format using mongodb code in linux system.
So can anyone please help me out to do this.

Related

One jq command to get expected output

I want to format below json code and print only specific values.
I tried using below command and got the below json output. But when I further try to parse the code for end result it didn't worked as expected.
So, a little help can make things work for me.
Command that results in below JSON code output:
mgmt_cli -r true -d 192.168.86.201 show access-rulebase name Network --format json | jq --raw-output ' .rulebase[].rulebase[] | {name: .name, rule_number: ."rule-number"}'
JSON Code:
{
"name": "MgmtRule",
"rule_number": 1
}
{
"name": null,
"rule_number": 2
}
{
"name": null,
"rule_number": 3
}
{
"name": "Inside Rules",
"rule_number": 4
}
{
"name": null,
"rule_number": 5
}
{
"name": null,
"rule_number": 6
}
{
"name": null,
"rule_number": 7
}
{
"name": null,
"rule_number": 8
}
{
"name": null,
"rule_number": 9
}
{
"name": "Cleanup rule",
"rule_number": 10
}
Further formatting jq command that I added to the above command to get the end result:
jq --slurp '"\(.[0])-\(.[length -1])"'
Unexpected Output:
"{\"name\":\"MgmtRule\",\"rule_number\":1}-{\"name\":\"Cleanup rule\",\"rule_number\":10}"
Expected Output:
MgmtRule (1-3)
Inside Rules (4-9)
Cleanup rule (10)
Raw output:
Short form of original raw output:
{
"uid" : "38271c2f-ab44-4e25-9aa4-e219cb6e12cf",
"name" : "Network",
"rulebase" : [ {
"uid" : "38acf747-85ee-4962-a3e7-d3cd678c388f",
"name" : "Internal Rules",
"type" : "access-section",
"from" : 1,
"to" : 3,
"rulebase" : [ {
"uid" : "bdd9d868-51b2-4210-8c37-f0237cd560fa",
"name" : "Test",
"type" : "access-rule",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"rule-number" : 1,
"track" : {
"type" : "598ead32-aa42-4615-90ed-f51a5928d41d",
"per-session" : false,
"per-connection" : true,
"accounting" : false,
"enable-firewall-session" : false,
"alert" : "none"
},
"source" : [ "6ddb84da-ef72-45c8-b2c4-bb3a849a6aa9" ],
"source-negate" : false,
"destination" : [ "375f4a15-1a7d-0b48-8ffc-2e9eb4e512ae" ],
"destination-negate" : false,
"service" : [ "97aeb443-9aea-11d5-bd16-0090272ccb30", "18ec9eaa-1657-4240-ab97-5f234623336b" ],
"service-negate" : false,
"service-resource" : "",
"vpn" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"action" : "6c488338-8eec-4103-ad21-cd461ac2c472",
"action-settings" : {
"enable-identity-captive-portal" : false
},
"content" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"content-negate" : false,
"content-direction" : "any",
"time" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"custom-fields" : {
"field-1" : "",
"field-2" : "",
"field-3" : ""
},
"meta-info" : {
"lock" : "unlocked",
"validation-state" : "ok",
"last-modify-time" : {
"posix" : 1674486777583,
"iso-8601" : "2023-01-23T20:42+0530"
},
"last-modifier" : "admin",
"creation-time" : {
"posix" : 1674486722309,
"iso-8601" : "2023-01-23T20:42+0530"
},
"creator" : "admin"
},
"comments" : "",
"enabled" : true,
"install-on" : [ "6c488338-8eec-4103-ad21-cd461ac2c476" ],
"available-actions" : {
"edit" : "true",
"delete" : "true",
"clone" : "not_supported"
}
}, {
"uid" : "bb93b38a-c672-49ec-afb0-8a90e8518437",
"type" : "access-rule",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"rule-number" : 2,
"track" : {
"type" : "598ead32-aa42-4615-90ed-f51a5928d41d",
"per-session" : false,
"per-connection" : true,
"accounting" : false,
"enable-firewall-session" : false,
"alert" : "none"
},
"source" : [ "6ddb84da-ef72-45c8-b2c4-bb3a849a6aa9" ],
"source-negate" : false,
"destination" : [ "8dbb91f3-786b-4d11-a029-45ee5b7d59cf" ],
"destination-negate" : false,
"service" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"service-negate" : false,
"service-resource" : "",
"vpn" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"action" : "6c488338-8eec-4103-ad21-cd461ac2c472",
"action-settings" : {
"enable-identity-captive-portal" : false
},
"content" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"content-negate" : false,
"content-direction" : "any",
"time" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"custom-fields" : {
"field-1" : "",
"field-2" : "",
"field-3" : ""
},
"meta-info" : {
"lock" : "unlocked",
"validation-state" : "ok",
"last-modify-time" : {
"posix" : 1674488952100,
"iso-8601" : "2023-01-23T21:19+0530"
},
"last-modifier" : "admin",
"creation-time" : {
"posix" : 1674488917458,
"iso-8601" : "2023-01-23T21:18+0530"
},
"creator" : "admin"
},
"comments" : "",
"enabled" : true,
"install-on" : [ "6c488338-8eec-4103-ad21-cd461ac2c476" ],
"available-actions" : {
"edit" : "true",
"delete" : "true",
"clone" : "not_supported"
}
}, {
"uid" : "925f210e-ce4f-4139-870e-3cd53f46d832",
"type" : "access-rule",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"rule-number" : 3,
"track" : {
"type" : "598ead32-aa42-4615-90ed-f51a5928d41d",
"per-session" : false,
"per-connection" : true,
"accounting" : false,
"enable-firewall-session" : false,
"alert" : "none"
},
"source" : [ "6ddb84da-ef72-45c8-b2c4-bb3a849a6aa9" ],
"source-negate" : false,
"destination" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"destination-negate" : false,
"service" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"service-negate" : false,
"service-resource" : "",
"vpn" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"action" : "6c488338-8eec-4103-ad21-cd461ac2c473",
"action-settings" : { },
"content" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"content-negate" : false,
"content-direction" : "any",
"time" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"custom-fields" : {
"field-1" : "",
"field-2" : "",
"field-3" : ""
},
"meta-info" : {
"lock" : "unlocked",
"validation-state" : "ok",
"last-modify-time" : {
"posix" : 1674490197006,
"iso-8601" : "2023-01-23T21:39+0530"
},
"last-modifier" : "admin",
"creation-time" : {
"posix" : 1674490169763,
"iso-8601" : "2023-01-23T21:39+0530"
},
"creator" : "admin"
},
"comments" : "",
"enabled" : true,
"install-on" : [ "6c488338-8eec-4103-ad21-cd461ac2c476" ],
"available-actions" : {
"edit" : "true",
"delete" : "true",
"clone" : "not_supported"
}
} ]
}, {
"uid" : "da0fdbf6-360b-44a4-94a8-b01894d3b5a0",
"name" : "cleanup rule",
"type" : "access-section",
"from" : 4,
"to" : 4,
"rulebase" : [ {
"uid" : "2b922948-da96-4c9d-a654-063e0183f9ae",
"name" : "Cleanup rule",
"type" : "access-rule",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"rule-number" : 4,
"track" : {
"type" : "598ead32-aa42-4615-90ed-f51a5928d41d",
"per-session" : false,
"per-connection" : true,
"accounting" : false,
"enable-firewall-session" : false,
"alert" : "none"
},
"source" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"source-negate" : false,
"destination" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"destination-negate" : false,
"service" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"service-negate" : false,
"service-resource" : "",
"vpn" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"action" : "6c488338-8eec-4103-ad21-cd461ac2c473",
"action-settings" : { },
"content" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"content-negate" : false,
"content-direction" : "any",
"time" : [ "97aeb369-9aea-11d5-bd16-0090272ccb30" ],
"custom-fields" : {
"field-1" : "",
"field-2" : "",
"field-3" : ""
},
"meta-info" : {
"lock" : "unlocked",
"validation-state" : "ok",
"last-modify-time" : {
"posix" : 1674486751969,
"iso-8601" : "2023-01-23T20:42+0530"
},
"last-modifier" : "admin",
"creation-time" : {
"posix" : 1668629634218,
"iso-8601" : "2022-11-17T01:43+0530"
},
"creator" : "System"
},
"comments" : "",
"enabled" : true,
"install-on" : [ "6c488338-8eec-4103-ad21-cd461ac2c476" ],
"available-actions" : {
"edit" : "true",
"delete" : "true",
"clone" : "not_supported"
}
} ]
} ],
"objects-dictionary" : [ {
"uid" : "6c488338-8eec-4103-ad21-cd461ac2c472",
"name" : "Accept",
"type" : "RulebaseAction",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"icon" : "Actions/actionsAccept",
"color" : "none"
}, {
"uid" : "8dbb91f3-786b-4d11-a029-45ee5b7d59cf",
"name" : "All_Internet",
"type" : "address-range",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"ipv4-address-first" : "0.0.0.0",
"ipv4-address-last" : "255.255.255.255",
"icon" : "Objects/ip",
"color" : "sienna"
}, {
"uid" : "97aeb369-9aea-11d5-bd16-0090272ccb30",
"name" : "Any",
"type" : "CpmiAnyObject",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"icon" : "General/globalsAny",
"color" : "black"
}, {
"uid" : "6c488338-8eec-4103-ad21-cd461ac2c473",
"name" : "Drop",
"type" : "RulebaseAction",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"icon" : "Actions/actionsDrop",
"color" : "none"
}, {
"uid" : "375f4a15-1a7d-0b48-8ffc-2e9eb4e512ae",
"name" : "dummy",
"type" : "simple-gateway",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"icon" : "NetworkObjects/management",
"color" : "black"
}, {
"uid" : "6ddb84da-ef72-45c8-b2c4-bb3a849a6aa9",
"name" : "H_192.168.86.111",
"type" : "host",
"domain" : {
"uid" : "41e821a0-3720-11e3-aa6e-0800200c9fde",
"name" : "SMC User",
"domain-type" : "domain"
},
"ipv4-address" : "192.168.86.111",
"icon" : "Objects/host",
"color" : "black"
}, {
"uid" : "97aeb443-9aea-11d5-bd16-0090272ccb30",
"name" : "https",
"type" : "service-tcp",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"port" : "443",
"icon" : "Protocols/HTTP",
"color" : "red"
}, {
"uid" : "598ead32-aa42-4615-90ed-f51a5928d41d",
"name" : "Log",
"type" : "Track",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"icon" : "Track/tracksLog",
"color" : "none"
}, {
"uid" : "6c488338-8eec-4103-ad21-cd461ac2c476",
"name" : "Policy Targets",
"type" : "Global",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"icon" : "General/globalsAny",
"color" : "none"
}, {
"uid" : "18ec9eaa-1657-4240-ab97-5f234623336b",
"name" : "ssh",
"type" : "service-tcp",
"domain" : {
"uid" : "a0bbbc99-adef-4ef8-bb6d-defdefdefdef",
"name" : "Check Point Data",
"domain-type" : "data domain"
},
"port" : "22",
"icon" : "Services/TCPService",
"color" : "red"
} ],
"from" : 1,
"to" : 4,
"total" : 4
}
Elements that I'm trying to fetch and get the end result from above raw output:
"name" & "rule-number"
Expected Output from above raw data:
Internal Rules (1-3)
cleanup rule (4)

By transforming the unprocessed, raw output generated by mgmt_cli, the solution becomes almost trivial since the JSON already contains the "to" and "from" properties.
Input (shortened to the relevant bits):
{
"uid": "38271c2f-ab44-4e25-9aa4-e219cb6e12cf",
"name": "Network",
"rulebase": [
{
"uid": "38acf747-85ee-4962-a3e7-d3cd678c388f",
"name": "Internal Rules",
"type": "access-section",
"from": 1,
"to": 3
},
{
"uid": "da0fdbf6-360b-44a4-94a8-b01894d3b5a0",
"name": "cleanup rule",
"type": "access-section",
"from": 4,
"to": 4
}
],
"from": 1,
"to": 4,
"total": 4
}
Jq transformation:
... | jq -r '.rulebase[]
| if .from == .to then "\(.name) (\(.from))"
else "\(.name) (\(.from)-\(.to))"
end'
Output:
Internal Rules (1-3)
cleanup rule (4)
Or perhaps (but I find the if/then more straightforward):
.rulebase[]
| (select(.from == .to) | "\(.name) (\(.from)-\(.to))")
// "\(.name) (\(.from)"

SwiftyJSON parsing data in array of objects

I am trying to get the value for "name" in the tracks section of JSON. The following JSON is from Spotify. I am using SwiftyJSON for parsing the data.
I am able to pull values from the base json. But once I try to pull anything in the "track" section I get errors.
Here is a sample of the code I got working
let swiftJSON = try? JSON(data: data)
let addedAT = swiftJSON!["items"].arrayValue.map {$0["added_at"].stringValue}
print(addedAT)
{
"items" : [
{
"video_thumbnail" : {
"url" : null
},
"track" : {
"is_playable" : true,
"type" : "track",
"id" : "3F5CgOj3wFlRv51JsHbxhe",
"track" : true,
"duration_ms" : 218364,
"explicit" : true,
"uri" : "spotify:track:3F5CgOj3wFlRv51JsHbxhe",
"is_local" : false,
"artists" : [
{
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/artist\/3TVXtAsR1Inumwj472S9r4"
},
"uri" : "spotify:artist:3TVXtAsR1Inumwj472S9r4",
"type" : "artist",
"href" : "https:\/\/api.spotify.com\/v1\/artists\/3TVXtAsR1Inumwj472S9r4",
"name" : "Drake",
"id" : "3TVXtAsR1Inumwj472S9r4"
},
{
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/artist\/1URnnhqYAYcrqrcwql10ft"
},
"uri" : "spotify:artist:1URnnhqYAYcrqrcwql10ft",
"type" : "artist",
"href" : "https:\/\/api.spotify.com\/v1\/artists\/1URnnhqYAYcrqrcwql10ft",
"name" : "21 Savage",
"id" : "1URnnhqYAYcrqrcwql10ft"
}
],
"album" : {
"name" : "Honestly, Nevermind",
"total_tracks" : 14,
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/album\/3cf4iSSKd8ffTncbtKljXw"
},
"images" : [
{
"height" : 640,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d0000b2738dc0d801766a5aa6a33cbe37",
"width" : 640
},
{
"height" : 300,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d00001e028dc0d801766a5aa6a33cbe37",
"width" : 300
},
{
"height" : 64,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d000048518dc0d801766a5aa6a33cbe37",
"width" : 64
}
],
"artists" : [
{
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/artist\/3TVXtAsR1Inumwj472S9r4"
},
"uri" : "spotify:artist:3TVXtAsR1Inumwj472S9r4",
"type" : "artist",
"href" : "https:\/\/api.spotify.com\/v1\/artists\/3TVXtAsR1Inumwj472S9r4",
"name" : "Drake",
"id" : "3TVXtAsR1Inumwj472S9r4"
}
],
"href" : "https:\/\/api.spotify.com\/v1\/albums\/3cf4iSSKd8ffTncbtKljXw",
"release_date_precision" : "day",
"type" : "album",
"id" : "3cf4iSSKd8ffTncbtKljXw",
"album_type" : "album",
"release_date" : "2022-06-17",
"uri" : "spotify:album:3cf4iSSKd8ffTncbtKljXw"
},
"track_number" : 14,
"external_ids" : {
"isrc" : "USUG12204897"
},
"episode" : false,
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/track\/3F5CgOj3wFlRv51JsHbxhe"
},
"popularity" : 90,
"disc_number" : 1,
"name" : "Jimmy Cooks (feat. 21 Savage)",
"preview_url" : "https:\/\/p.scdn.co\/mp3-preview\/5feee7c2fed489c933a31ef125aa4ffb5205a457?cid=263c670f8c17484d89d8bd2e6e0964f7",
"href" : "https:\/\/api.spotify.com\/v1\/tracks\/3F5CgOj3wFlRv51JsHbxhe"
},
"added_by" : {
"uri" : "spotify:user:12493212",
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/user\/12493212"
},
"type" : "user",
"href" : "https:\/\/api.spotify.com\/v1\/users\/12493212",
"id" : "12493212"
},
"primary_color" : null,
"added_at" : "2022-06-28T16:06:35Z",
"is_local" : false
},
{
"video_thumbnail" : {
"url" : null
},
"track" : {
"is_playable" : true,
"type" : "track",
"id" : "1qMMYpVatbRITKCfq1gasi",
"track" : true,
"duration_ms" : 172933,
"explicit" : true,
"uri" : "spotify:track:1qMMYpVatbRITKCfq1gasi",
"is_local" : false,
"artists" : [
{
"href" : "https:\/\/api.spotify.com\/v1\/artists\/1RyvyyTE3xzB2ZywiAwp0i",
"uri" : "spotify:artist:1RyvyyTE3xzB2ZywiAwp0i",
"name" : "Future",
"id" : "1RyvyyTE3xzB2ZywiAwp0i",
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/artist\/1RyvyyTE3xzB2ZywiAwp0i"
},
"type" : "artist"
}
],
"album" : {
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/album\/6tE9Dnp2zInFij4jKssysL"
},
"href" : "https:\/\/api.spotify.com\/v1\/albums\/6tE9Dnp2zInFij4jKssysL",
"release_date" : "2022-04-29",
"release_date_precision" : "day",
"artists" : [
{
"href" : "https:\/\/api.spotify.com\/v1\/artists\/1RyvyyTE3xzB2ZywiAwp0i",
"uri" : "spotify:artist:1RyvyyTE3xzB2ZywiAwp0i",
"name" : "Future",
"id" : "1RyvyyTE3xzB2ZywiAwp0i",
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/artist\/1RyvyyTE3xzB2ZywiAwp0i"
},
"type" : "artist"
}
],
"name" : "I NEVER LIKED YOU",
"type" : "album",
"album_type" : "album",
"id" : "6tE9Dnp2zInFij4jKssysL",
"uri" : "spotify:album:6tE9Dnp2zInFij4jKssysL",
"images" : [
{
"height" : 640,
"width" : 640,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d0000b27386badd635b69aea887862214"
},
{
"height" : 300,
"width" : 300,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d00001e0286badd635b69aea887862214"
},
{
"height" : 64,
"width" : 64,
"url" : "https:\/\/i.scdn.co\/image\/ab67616d0000485186badd635b69aea887862214"
}
],
"total_tracks" : 16
},
"track_number" : 5,
"external_ids" : {
"isrc" : "USSM12203787"
},
"episode" : false,
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/track\/1qMMYpVatbRITKCfq1gasi"
},
"popularity" : 83,
"disc_number" : 1,
"name" : "PUFFIN ON ZOOTIEZ",
"preview_url" : "https:\/\/p.scdn.co\/mp3-preview\/ff3bd626fd7f0bd6af4a429b3119b06ff0aee089?cid=263c670f8c17484d89d8bd2e6e0964f7",
"href" : "https:\/\/api.spotify.com\/v1\/tracks\/1qMMYpVatbRITKCfq1gasi"
},
"added_by" : {
"external_urls" : {
"spotify" : "https:\/\/open.spotify.com\/user\/12493212"
},
"uri" : "spotify:user:12493212",
"type" : "user",
"id" : "12493212",
"href" : "https:\/\/api.spotify.com\/v1\/users\/12493212"
},
"primary_color" : null,
"added_at" : "2022-06-28T16:53:13Z",
"is_local" : false
}
],
"href" : "https:\/\/api.spotify.com\/v1\/playlists\/1nJflwAwbyhpQzKksqwC7H\/tracks?offset=0&limit=100&market=ES&locale=en-US;q=1.0",
"offset" : 0,
"previous" : null,
"limit" : 100,
"next" : null,
"total" : 2
}

Elasticsearch aggregation - groupby & filter max of those document and group by again

I am trying to put a query to elasticsearch(ES) to do calculation for me with the condition following the example data,
{user_id: 1, score: 10, status: passed, date: 2020-02-26},
{user_id: 1, score: 10, status: passed, date: 2020-02-25},
{user_id: 1, score: 8, status: passed, date: 2020-02-25},
{user_id: 1, score: 4, status: failed, date: 2020-02-27},
{user_id: 2, score: 5, status: passed, date: 2020-02-26},
{user_id: 2, score: 6, status: passed, date: 2020-02-25},
{user_id: 3, score: 1, status: failed, date: 2020-02-25},
{user_id: 3, score: 1, status: failed, date: 2020-02-25},
{user_id: 4, score: 7, status: passed, date: 2020-02-25}
{user_id: 4, score: 2, status: failed, date: 2020-02-26}
{user_id: 5, score: 0, status: failed, date: 2020-02-25}
I'd like to get max score(not score of es) of each user_id for counting number of status in each duration (group by status again)
or step: group by user_id get top score record, and group by those status
sample output:
{
"passed":3
"failed":2
"date": 2020-02-25
},
{
"passed":2
"failed":1
"date": 2020-02-26
},
{
"passed":0
"failed":1
"date": 2020-02-27
}
even, I play with SQL it still not easy to solve coz, I cannot get a row_number of record making the result can be wrong (please correct me If I am wrong)
with SQL It like
// this nested query give a wrong answer, just to be an example
// I still playing with it
select user_id, status from table t1 inner join (
select user_id, max(score)
from table
where date = '2020-02-25'
group by user_id, max(score)
) table t2 on t1.user_id = t2.user_id and t1.score = t2.score
group by t1.status
a reason, I want to do this coz, my work have data stored in es and I think it may be good if es could do this as well to reduce performance load of backend side.
Thank you in advance and so open to any suggestion

I have used term aggregation to get count of docs under date and status. Let me know if it works for you.
Sample mapping
PUT testindex13
{
"mappings": {
"properties": {
"userid": {
"type": "keyword"
},
"score": {
"type": "integer"
},
"status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"date":{
"type": "date",
"format": "yyyy-MM-dd"
}
}
}
}
Data:
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "9XtAhXABqQAB2FLojE2i",
"_score" : 1.0,
"_source" : {
"user_id" : 1,
"score" : 10,
"status" : "passed",
"date" : "2020-02-26"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "9ntAhXABqQAB2FLojU0l",
"_score" : 1.0,
"_source" : {
"user_id" : 1,
"score" : 10,
"status" : "passed",
"date" : "2020-02-25"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "93tAhXABqQAB2FLojk2P",
"_score" : 1.0,
"_source" : {
"user_id" : 1,
"score" : 8,
"status" : "passed",
"date" : "2020-02-25"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-HtAhXABqQAB2FLoj02H",
"_score" : 1.0,
"_source" : {
"user_id" : 1,
"score" : 4,
"status" : "failed",
"date" : "2020-02-27"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-XtAhXABqQAB2FLokE0m",
"_score" : 1.0,
"_source" : {
"user_id" : 2,
"score" : 5,
"status" : "passed",
"date" : "2020-02-26"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-ntAhXABqQAB2FLokU2x",
"_score" : 1.0,
"_source" : {
"user_id" : 2,
"score" : 6,
"status" : "passed",
"date" : "2020-02-25"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-3tEhXABqQAB2FLoS00O",
"_score" : 1.0,
"_source" : {
"user_id" : 1,
"score" : 10,
"status" : "passed",
"date" : "2020-02-26"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_HtEhXABqQAB2FLoTE0W",
"_score" : 1.0,
"_source" : {
"user_id" : 3,
"score" : 1,
"status" : "failed",
"date" : "2020-02-25"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_XtEhXABqQAB2FLoTE2L",
"_score" : 1.0,
"_source" : {
"user_id" : 3,
"score" : 1,
"status" : "failed",
"date" : "2020-02-25"
}
},
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_ntEhXABqQAB2FLoTU2P",
"_score" : 1.0,
"_source" : {
"user_id" : 4,
"score" : 7,
"status" : "passed",
"date" : "2020-02-25"
}
}
]
Query:
GET testindex13/_search
{
"size": 0,
"aggs": {
"Date": {
"terms": {
"field": "date"
},
"aggs": {
"Status": {
"terms": {
"field": "status.keyword",
"size": 10
},
"aggs": {
"Count": {
"value_count": {
"field": "user_id"
}
}
}
}
}
}
}
}
Result:
"hits" : {
"total" : {
"value" : 12,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"Date" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1582588800000,
"key_as_string" : "2020-02-25",
"doc_count" : 7,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "passed",
"doc_count" : 4,
"Count" : {
"value" : 4
}
},
{
"key" : "failed",
"doc_count" : 3,
"Count" : {
"value" : 3
}
}
]
}
},
{
"key" : 1582675200000,
"key_as_string" : "2020-02-26",
"doc_count" : 4,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "passed",
"doc_count" : 3,
"Count" : {
"value" : 3
}
},
{
"key" : "failed",
"doc_count" : 1,
"Count" : {
"value" : 1
}
}
]
}
},
{
"key" : 1582761600000,
"key_as_string" : "2020-02-27",
"doc_count" : 1,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "failed",
"doc_count" : 1,
"Count" : {
"value" : 1
}
}
]
}
}
]
}
}
}
EDIT 2. using sort to return top most user by score
GET testindex13/_search
{
"size": 0,
"aggs": {
"Date": {
"terms": {
"field": "date"
},
"aggs": {
"Status": {
"terms": {
"field": "status.keyword",
"size": 10
},
"aggs": {
"Users":{
"terms": {
"field": "user_id",
"size": 10
},
"aggs": {
"TopDocuments": {
"top_hits": {
"size": 1,
"sort": {"score":{"order" : "desc"}}
}
}
}
},
"Count": {
"value_count": {
"field": "user_id"
}
}
}
}
}
}
}
}
Result:
"aggregations" : {
"Date" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1582588800000,
"key_as_string" : "2020-02-25",
"doc_count" : 7,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "passed",
"doc_count" : 4,
"Count" : {
"value" : 4
},
"Users" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1,
"doc_count" : 2,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "9ntAhXABqQAB2FLojU0l",
"_score" : null,
"_source" : {
"user_id" : 1,
"score" : 10,
"status" : "passed",
"date" : "2020-02-25"
},
"sort" : [
10
]
}
]
}
}
},
{
"key" : 2,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-ntAhXABqQAB2FLokU2x",
"_score" : null,
"_source" : {
"user_id" : 2,
"score" : 6,
"status" : "passed",
"date" : "2020-02-25"
},
"sort" : [
6
]
}
]
}
}
},
{
"key" : 4,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_ntEhXABqQAB2FLoTU2P",
"_score" : null,
"_source" : {
"user_id" : 4,
"score" : 7,
"status" : "passed",
"date" : "2020-02-25"
},
"sort" : [
7
]
}
]
}
}
}
]
}
},
{
"key" : "failed",
"doc_count" : 3,
"Count" : {
"value" : 3
},
"Users" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 3,
"doc_count" : 2,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_HtEhXABqQAB2FLoTE0W",
"_score" : null,
"_source" : {
"user_id" : 3,
"score" : 1,
"status" : "failed",
"date" : "2020-02-25"
},
"sort" : [
1
]
}
]
}
}
},
{
"key" : 5,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "AHtEhXABqQAB2FLoT04D",
"_score" : null,
"_source" : {
"user_id" : 5,
"score" : 0,
"status" : "failed",
"date" : "2020-02-25"
},
"sort" : [
0
]
}
]
}
}
}
]
}
}
]
}
},
{
"key" : 1582675200000,
"key_as_string" : "2020-02-26",
"doc_count" : 4,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "passed",
"doc_count" : 3,
"Count" : {
"value" : 3
},
"Users" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1,
"doc_count" : 2,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "9XtAhXABqQAB2FLojE2i",
"_score" : null,
"_source" : {
"user_id" : 1,
"score" : 10,
"status" : "passed",
"date" : "2020-02-26"
},
"sort" : [
10
]
}
]
}
}
},
{
"key" : 2,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-XtAhXABqQAB2FLokE0m",
"_score" : null,
"_source" : {
"user_id" : 2,
"score" : 5,
"status" : "passed",
"date" : "2020-02-26"
},
"sort" : [
5
]
}
]
}
}
}
]
}
},
{
"key" : "failed",
"doc_count" : 1,
"Count" : {
"value" : 1
},
"Users" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 4,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "_3tEhXABqQAB2FLoTU3v",
"_score" : null,
"_source" : {
"user_id" : 4,
"score" : 2,
"status" : "failed",
"date" : "2020-02-26"
},
"sort" : [
2
]
}
]
}
}
}
]
}
}
]
}
},
{
"key" : 1582761600000,
"key_as_string" : "2020-02-27",
"doc_count" : 1,
"Status" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "failed",
"doc_count" : 1,
"Count" : {
"value" : 1
},
"Users" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1,
"doc_count" : 1,
"TopDocuments" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "testindex13",
"_type" : "_doc",
"_id" : "-HtAhXABqQAB2FLoj02H",
"_score" : null,
"_source" : {
"user_id" : 1,
"score" : 4,
"status" : "failed",
"date" : "2020-02-27"
},
"sort" : [
4
]
}
]
}
}
}
]
}
}
]
}
}
]
}
}

Using mongo queries to return new json structure

Currently playing around with aggregation filters to group results by sessionID. Is it possible to change this in the database:
{
"_id" : ObjectId("xxx"),
"quizId" : "xxx",
"sessionId" : "xxx",
"questionId" : "xxx",
"categoryId" : "xxx",
"question" : "text here",
"answer" : "male",
"created" : ISODate("2015-12-04T12:19:43.050Z"),
"__v" : 0
},
{
"_id" : ObjectId("xxx"),
"quizId" : "xxx",
"sessionId" : "xxx",
"questionId" : "xxx",
"categoryId" : "xxx",
"question" : "another text here",
"answer" : "25",
"created" : ISODate("2015-12-04T12:19:43.050Z"),
"__v" : 0
},
{
"_id" : ObjectId("xxx"),
"quizId" : "xxx",
"sessionId" : "xxx",
"questionId" : "xxx",
"categoryId" : "xxx",
"question" : "first question text here",
"answer" : "answer A",
"created" : ISODate("2015-12-04T12:19:43.050Z"),
"__v" : 0
} // several questions more with same session ID//
to return this after a MongoDb query:
{
"_id" : ObjectId("xxx"),
"quizId" : "12345",
"sessionId" : "xxx",
"Gender" : "male",
"Age" : "25",
"Q1" : "answer A",
// These aggregated from other objects with same sessionId
"Q2" : "answer D",
"Q3" : "answer C"
}

Thanks to the help of user3632894, i grouped answers by sessionID using
db.questionanswers.aggregate(
[
{ $group : { _id : {sessionId: "$sessionId" }, answer: {$addToSet: "$answer"} } }
]
)
Which returns
{
"sessionId" : "xxx",
"answer" : [
"0" : "answer A",
"1" : "answer D",
"2" : "answer C"
]
}

How to convert JSON to POJO

Anyway to create a Java POJO template from a JSON string? I'm using Jersey
{
"reviewData" : [ {
"projectKey" : "CR-FOO",
"name" : "Example review.",
"description" : "Description or statement of objectives for this example review.",
"author" : {
"userName" : "joe",
"displayName" : "Joe Krustofski",
"avatarUrl" : "http://foo.com/avatar"
},
"moderator" : {
"userName" : "scott",
"displayName" : "Scott the Moderator",
"avatarUrl" : "http://foo.com/avatar"
},
"creator" : {
"userName" : "joe",
"displayName" : "Joe Krustofski",
"avatarUrl" : "http://foo.com/avatar"
},
"permaId" : {
"id" : "CR-FOO-21"
},
"permaIdHistory" : [ "CR-FOO-21" ],
"type" : "REVIEW",
"allowReviewersToJoin" : true,
"metricsVersion" : 4,
"createDate" : "2013-10-08T15:46:11.022+0200",
"dueDate" : "2013-10-09T15:46:11.022+0200",
"jiraIssueKey" : "FOO-6754"
}, {
"projectKey" : "CR-FOO",
"name" : "Example review.",
"description" : "Description or statement of objectives for this example review.",
"author" : {
"userName" : "joe",
"displayName" : "Joe Krustofski",
"avatarUrl" : "http://foo.com/avatar"
},
"moderator" : {
"userName" : "scott",
"displayName" : "Scott the Moderator",
"avatarUrl" : "http://foo.com/avatar"
},
"creator" : {
"userName" : "joe",
"displayName" : "Joe Krustofski",
"avatarUrl" : "http://foo.com/avatar"
},
"permaId" : {
"id" : "CR-FOO-21"
},
"permaIdHistory" : [ "CR-FOO-21" ],
"type" : "REVIEW",
"allowReviewersToJoin" : true,
"metricsVersion" : 4,
"createDate" : "2013-10-08T15:46:11.022+0200",
"dueDate" : "2013-10-09T15:46:11.022+0200",
"jiraIssueKey" : "FOO-6754"
} ]
}

The best solution I found was http://www.jsonschema2pojo.org/
It has support for Jackson and Maven and an online tool as well.