Related
I need help with jq syntax on how to return the Gitlab job ID if it contains an artifact. The JSON output looks like this (removed a lot of unrelated info from it and added [...]):
[{
"id": 3219589880,
"status": "success",
"stage": "test",
"name": "job_with_no_artifact",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.119Z",
"started_at": "2022-10-24T18:21:25.986Z",
"finished_at": "2022-10-24T18:21:38.464Z",
"duration": 12.478682,
"queued_duration": 0.499786,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
"pipeline": {
"id": 123456789,
[...]
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts": [],
"runner": {
"id": 12270859,
[...]
},
"artifacts_expire_at": null,
"tag_list": []
}, {
"id": 3219589878,
"status": "success",
"stage": "test",
"name": "create_artifact_job_2",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.111Z",
"started_at": "2022-10-24T18:21:25.922Z",
"finished_at": "2022-10-24T18:21:39.090Z",
"duration": 13.168405,
"queued_duration": 0.464364,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
},
"pipeline": {
"id": 675641982,
[...],
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts_file": {
"filename": "artifacts.zip",
"size": 223
},
"artifacts": [{
"file_type": "archive",
"size": 223,
"filename": "artifacts.zip",
"file_format": "zip"
}, {
"file_type": "metadata",
"size": 153,
"filename": "metadata.gz",
"file_format": "gzip"
}],
"runner": {
"id": 12270845,
[...]
},
"artifacts_expire_at": "2022-10-25T18:21:35.859Z",
"tag_list": []
}, {
"id": 3219589876,
"status": "success",
"stage": "test",
"name": "create_artifact_job_1",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.103Z",
"started_at": "2022-10-24T18:21:25.503Z",
"finished_at": "2022-10-24T18:21:41.407Z",
"duration": 15.904028,
"queued_duration": 0.098837,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
},
"pipeline": {
"id": 123456789,
[...]
},
"web_url": "WEB_URL",
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts_file": {
"filename": "artifacts.zip",
"size": 217
},
"artifacts": [{
"file_type": "archive",
"size": 217,
"filename": "artifacts.zip",
"file_format": "zip"
}, {
"file_type": "metadata",
"size": 152,
"filename": "metadata.gz",
"file_format": "gzip"
}],
"runner": {
"id": 12270857,
},
"artifacts_expire_at": "2022-10-25T18:21:37.808Z",
"tag_list": []
}]
I've been trying to do either of the following using jQ:
Either:
Check if artifacts_file key exists in each iteration and if it does return the (job) id (so .[].id)
Check if artifacts array is empty in each iteration and if it is empty return the (job) id.
In both cases I'm able to do the first part but I am not sure how to return the .id key.
Related stackoverflow questions that I've been trying to utilize and adapt to my case:
jq - return array value if its length is not null
How to check for presence of 'key' in jq before iterating over the values
What I have so far: jq '[.[].artifacts[]|select(length > 0)] | .[]' which returns all the artifacts found (but it doesn't contain the .id of the job).
Checking the existence of a field using has:
.[] | select(has("artifacts_file")).id
3219589878
3219589876
Demo
Checking if a field is an empty array by comparing it to []:
.[] | select(.artifacts == []).id
3219589880
Demo
I have a json file as shown below
{
"body": {
"results": [
[
{
"field": "#timestamp",
"value": "2020-04-26 19:28:40.136"
},
{
"field": "#message",
"value": "Hi"
},
{
"field": "#time",
"value": "19:28:40,023"
},
{
"field": "#name",
"value": "Nitish"
},
{
"field": "#hobby",
"value": "Pool"
},
{
"field": "#duration",
"value": "19 mins"
}
],
[
{
"field": "#timestamp",
"value": "2020-04-26 19:28:40.136"
},
{
"field": "#message",
"value": "Hello"
},
{
"field": "#time",
"value": "19:28:40,023"
},
{
"field": "#name",
"value": "Amuri"
},
{
"field": "#Totalruns",
"value": "2"
},
{
"field": "#wickets",
"value": "10"
},
{
"field": "#hobby",
"value": "cricket"
},
{
"field": "#commentry",
"value": "ubbjb"
}
],
[
{
"field": "#timestamp",
"value": "2020-04-26 19:28:40.136"
},
{
"field": "#message",
"value": "how are you"
},
{
"field": "#time",
"value": "19:28:40,023"
},
{
"field": "#name",
"value": "Kit"
},
{
"field": "#Totalruns",
"value": "90"
},
{
"field": "#wickets",
"value": "1"
},
{
"field": "#hobby",
"value": "cricket"
},
{
"field": "#commentry",
"value": "jbunib"
}
]
]
}
}
I'm trying to parse #Totalruns and #wickets only if #hobby = cricket
I'm able to reach till accessing cricket unable to figure out to get data of #Totalruns and #wickets
code i've tried import json
f = open('/Users/amurin/Documents/test.json','r')
data = json.load(f)
for result in data['body']['results']:
for res in result:
if res['value']=='cricket':
print("Hello")
f.close()
I need output as
wickets = 1 , Totalruns = 90
wickets = 10 , Totalruns = 2
Solution 1, keeping the structure as it is:
def sublist_is_valid(sublist):
return any(d["field"] == "#hobby" and d["value"] == "cricket" for d in sublist)
def filter_fields(sublist):
return [d for d in sublist if d["field"] in ["#wickets", "#Totalruns"]]
filtered = [
filter_fields(sublist)
for sublist in data["body"]["results"]
if sublist_is_valid(sublist)
]
print(filtered)
Output:
[[{'field': '#Totalruns', 'value': '2'}, {'field': '#wickets', 'value': '10'}],
[{'field': '#Totalruns', 'value': '90'}, {'field': '#wickets', 'value': '1'}]]
Solution 2 (better IMO) is to use dictionaries right from the beginning:
def sublist_to_dict(sublist):
return {d["field"]: d["value"] for d in sublist}
filtered = [
{key: d[key] for key in ["#Totalruns", "#wickets"]}
for d in [sublist_to_dict(sublist) for sublist in data["body"]["results"]]
if d.get("#hobby") == "cricket"
]
print(filtered)
Output:
[{'#Totalruns': '2', '#wickets': '10'}, {'#Totalruns': '90', '#wickets': '1'}]
I am new to scala and JSON parsing and need some help. I need to parse the complex JSON (below) to get the values of "name" in "dimension" key i.e I need PLATFORM and OS_VERSION.
I tried multiple options, but it is not working. Any help is appreciated
This is a snippet of the code I tried, but I am not able to proceed further in parsing the list. I believe the 'ANY' keyword is causing some mismatch / issues.
import org.json4s._
import org.json4s.jackson.JsonMethods._
implicit val formats = org.json4s.DefaultFormats
val mapJSON = parse(tmp).extract[Map[String, Any]]
println(mapJSON)
//for ((k,v) <- mapJSON) printf("key: %s, value: %s\n", k, v)
val list_map = mapJSON("dimensions")
{
"uuid": "uuidddd",
"last_modified": 1559080222953,
"version": "2.6.1.0",
"name": "FULL_DAY_2_mand_date",
"is_draft": false,
"model_name": "FULL_DAY_1_may05",
"description": "",
"null_string": null,
"dimensions": [
{
"name": "PLATFORM",
"table": "tbl1",
"column": "PLATFORM",
"derived": null
},
{
"name": "OS_VERSION",
"table": "tbl1",
"column": "OS_VERSION",
"derived": null
},
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1"
},
"returntype": "bigint"
}
},
{
"name": "UU",
"function": {
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
"value": "tbl1.USER_ID"
},
"returntype": "hllc(12)"
}
},
{
"name": "CONT_SIZE",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "tbl1.SIZE"
},
"returntype": "bigint"
}
},
{
"name": "CONT_COUNT",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "tbl1.COUNT"
},
"returntype": "bigint"
}
}
],
"dictionaries": [],
"rowkey": {
"rowkey_columns": [
{
"column": "tbl1.OS_VERSION",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "tbl1.PLATFORM",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "tbl1.DEVICE_FAMILY",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
}
]
},
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_",
"CONT_SIZE",
"CONT_COUNT"
]
}
]
},
{
"name": "F2",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"UU"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"tbl1.PLATFORM",
"tbl1.OS_VERSION"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"tbl1.DATE_HR"
],
"joint_dims": []
}
}
],
"signature": "ttrrs==",
"notify_list": [],
"status_need_notify": [
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start": 0,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"volatile_range": 0,
"retention_range": 0,
"engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {
"job.queuename": "root.production.P0",
"is-mandatory-only-valid": "true"
},
"cuboid_black_list": [],
"parent_forward": 3,
"mandatory_dimension_set_list": [],
"snapshot_table_desc_list": []
}
You need to make more specific classes for parsing the data, something like this:
case class Dimension(name: String, table: String, column: String)
case class AllData(uuid: String, dimensions: List[Dimension])
val data = parse(tmp).extract[AllData]
val names = data.dimensions.map(_.name)
My Json result from the API is as below
Json result:
"Issues": [{
"Id": null,
"Key": null,
"Values": [{
"Key": "Display Name",
"Value": "Rya"
},
{
"Key": "UserName",
"Value": "RH"
},
{
"Key": "Count",
"Value": "350"
}
]
},
{
"Id": null,
"Key": null,
"Values": [{
"Key": "Display Name",
"Value": "Mike"
},
{
"Key": "UserName",
"Value": "ML"
},
{
"Key": "Count",
"Value": "90"
}
]
}
]
I did a mapping by doing as below-
.Issues.map(o =>
o.Values.reduce((acc, {
Key,
Value
}) =>
(acc[Key] = Value, acc), {}));
The result of the mapping is as below-
{ "Display Name": 'Rya', "UserName" : "RH", value: 350 },
{ "Display Name": 'Mike', "UserName" : "ML", value: 90 }
Desired Result:
{ "Display Name": 'Rya', "UserName" : "RH" },
{ "Display Name": 'Mike', "UserName" : "ML"}
In my requirement, I want to ignore the last element as shown in the desired result.
The easiest solution is a combination of map, slice and reduce:
json.Issues.map(b =>
b.Values.slice(0, -1).reduce((c,d) => {
c[d.Key] = d.Value;
return c;
}, {}));
Demo:
let j = {
"Issues": [{
"Id": null,
"Key": null,
"Values": [{
"Key": "Display Name",
"Value": "Rya"
},
{
"Key": "UserName",
"Value": "RH"
},
{
"Key": "Count",
"Value": "350"
}
]
},
{
"Id": null,
"Key": null,
"Values": [{
"Key": "Display Name",
"Value": "Mike"
},
{
"Key": "UserName",
"Value": "ML"
},
{
"Key": "Count",
"Value": "90"
}
]
}
]
};
let r = j.Issues.map(b =>
b.Values.slice(0, -1).reduce((c, d) => {
c[d.Key] = d.Value;
return c;
}, {}));
console.log(r);
One solution is to add a filter before the reduce to filter out objects with the unwanted Count property.
.Issues.map(o =>
o.Values
.filter(({ Key }) => Key !== 'Count')
.reduce((acc, {
Key,
Value
}) =>
(acc[Key] = Value, acc), {}));
You could also do this filtering inline during the reduction by not adding objects when Key === 'Count'.
Note: There is no such thing as the last property in a JS object. It is a collection of properties whose actual order is implementation dependent and unreliable. For example, printing your object in different browsers and platforms could give any order whatsoever, nothing guarantees that consistency.
To remove the last element regardless of how many items in the array I would favor something like this:
let result = data.Issues.map(issue => {
let temp = issue.Values;
temp.splice(-1);
return temp.reduce((acc, {Key, Value}) => (acc[Key] = Value, acc), {});
});
Here's a fiddle
.Issues.map(o => o.Values.reduce((acc, {Key, Value}) => (Key !== 'Count') ? (acc[Key] = Value, acc) : acc, {}));
full code:
const j = {"Issues": [
{
"Id": null,
"Key": null,
"Values": [
{
"Key": "Display Name",
"Value": "Rya"
},
{
"Key": "UserName",
"Value": "RH"
},
{
"Key": "Count",
"Value": "350"
}
]
},
{
"Id": null,
"Key": null,
"Values": [
{
"Key": "Display Name",
"Value": "Mike"
},
{
"Key": "UserName",
"Value": "ML"
},
{
"Key": "Count",
"Value": "90"
}
]
}
]
}
const r = j.Issues.map(o => o.Values.reduce((acc, {Key, Value}) => (Key !== 'Count') ? (acc[Key] = Value, acc) : acc, {}));
console.log(JSON.stringify(r, null, 2))
Note that in a reduce method, the function is also passed in the current index and the array reduce is being called on. So if you want to ignore the last element of the array, you can do something like:
Issues.map(o =>
o.Values.reduce((acc, {Key, Value}, idx, arry) => {
if(idx < arry.length -1)
acc[Key] = Value;
return acc;
}, {}
);
I am trying to convert my JSON result to an array to bind it to my Kendo controls.
The JSON result that I am getting is as follows.
"Issues": [
{
"Id": null,
"Key": null,
"Values": [
{
"Key": "Display Name",
"Value": "Rya"
},
{
"Key": "UserName",
"Value": "RH"
},
{
"Key": "Count",
"Value": "350"
}
]
},
{
"Id": null,
"Key": null,
"Values": [
{
"Key": "Display Name",
"Value": "Mike"
},
{
"Key": "UserName",
"Value": "ML"
},
{
"Key": "Count",
"Value": "90"
}
]
}
]
The array that I needed to bind it to the Kendo control is as below.
{ "Display Name": 'Rya', "UserName" : "RH", value: 350 },
{ "Display Name": 'Mike', "UserName" : "ML", value: 90 }
i)I dont want to hardcode the strings "Display Name", "User Name", "RH". I tried v.Values[0].Key: v.Values[0].Value, but it didn't work.
ii) Also I will not know how many "key, value" pairs will be present, so I need to loop through the Values and generate the array instead of fixed
category: v.Values[0].Value,
UserName: v.Values[1].Value,
value: v.Values[2].Value,
.
.
.
score: v.values[n].value
If you're using ES6, you don't really need lodash in this case:
var r = json.Issues.map(v => ({
category: v.Values[0].Value,
value: v.Values[2].Value,
}));
http://codepen.io/cjke/pen/RprJdG?editors=0010
If want to use lodash or not-ES6, then:
var r = _.map(json.Issues, function(v) {
return {
category: v.Values[0].Value,
value: v.Values[2].Value,
}
});