Related
I need help with jq syntax on how to return the Gitlab job ID if it contains an artifact. The JSON output looks like this (removed a lot of unrelated info from it and added [...]):
[{
"id": 3219589880,
"status": "success",
"stage": "test",
"name": "job_with_no_artifact",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.119Z",
"started_at": "2022-10-24T18:21:25.986Z",
"finished_at": "2022-10-24T18:21:38.464Z",
"duration": 12.478682,
"queued_duration": 0.499786,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
"pipeline": {
"id": 123456789,
[...]
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts": [],
"runner": {
"id": 12270859,
[...]
},
"artifacts_expire_at": null,
"tag_list": []
}, {
"id": 3219589878,
"status": "success",
"stage": "test",
"name": "create_artifact_job_2",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.111Z",
"started_at": "2022-10-24T18:21:25.922Z",
"finished_at": "2022-10-24T18:21:39.090Z",
"duration": 13.168405,
"queued_duration": 0.464364,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
},
"pipeline": {
"id": 675641982,
[...],
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts_file": {
"filename": "artifacts.zip",
"size": 223
},
"artifacts": [{
"file_type": "archive",
"size": 223,
"filename": "artifacts.zip",
"file_format": "zip"
}, {
"file_type": "metadata",
"size": 153,
"filename": "metadata.gz",
"file_format": "gzip"
}],
"runner": {
"id": 12270845,
[...]
},
"artifacts_expire_at": "2022-10-25T18:21:35.859Z",
"tag_list": []
}, {
"id": 3219589876,
"status": "success",
"stage": "test",
"name": "create_artifact_job_1",
"ref": "main",
"tag": false,
"coverage": null,
"allow_failure": false,
"created_at": "2022-10-24T18:21:25.103Z",
"started_at": "2022-10-24T18:21:25.503Z",
"finished_at": "2022-10-24T18:21:41.407Z",
"duration": 15.904028,
"queued_duration": 0.098837,
"user": {
"id": 123456789,
[...]
},
"commit": {
"id": "5e0e1f287d20daf2036a3ca71c656dce55999265",
[...]
},
"pipeline": {
"id": 123456789,
[...]
},
"web_url": "WEB_URL",
"project": {
"ci_job_token_scope_enabled": false
},
"artifacts_file": {
"filename": "artifacts.zip",
"size": 217
},
"artifacts": [{
"file_type": "archive",
"size": 217,
"filename": "artifacts.zip",
"file_format": "zip"
}, {
"file_type": "metadata",
"size": 152,
"filename": "metadata.gz",
"file_format": "gzip"
}],
"runner": {
"id": 12270857,
},
"artifacts_expire_at": "2022-10-25T18:21:37.808Z",
"tag_list": []
}]
I've been trying to do either of the following using jQ:
Either:
Check if artifacts_file key exists in each iteration and if it does return the (job) id (so .[].id)
Check if artifacts array is empty in each iteration and if it is empty return the (job) id.
In both cases I'm able to do the first part but I am not sure how to return the .id key.
Related stackoverflow questions that I've been trying to utilize and adapt to my case:
jq - return array value if its length is not null
How to check for presence of 'key' in jq before iterating over the values
What I have so far: jq '[.[].artifacts[]|select(length > 0)] | .[]' which returns all the artifacts found (but it doesn't contain the .id of the job).
Checking the existence of a field using has:
.[] | select(has("artifacts_file")).id
3219589878
3219589876
Demo
Checking if a field is an empty array by comparing it to []:
.[] | select(.artifacts == []).id
3219589880
Demo
I have a long list of json file . I want to extract the subdomain of harvard.edu which is in the variable host in "host": "ceonlineb2b.hms.harvard.edu using bash . I would be happy if anyone can help out .Below is only a snippet of json file.
{
"data": {
"total_items": 3,
"offset": 0,
"limit": 1,
"items": [
{
"name": "ceonlineb2b.hms.harvard.edu",
"alexa": null,
"cert_summary": null,
"dns_records": {
"A": [
"3.221.168.206",
"54.174.253.3"
],
"AAAA": null,
"CAA": null,
"CNAME": [
"hms-moodleb2b-prod.cabem.com"
],
"MX": null,
"NS": null,
"SOA": null,
"TXT": null,
"SPF": null,
"updated_at": "2021-05-14T23:12:43.332816923Z"
},
"hosts_enrichment": [
{
"ip": "3.221.168.206",
"as_num": 14618,
"as_org": "amazon-aes",
"isp": "amazon.com",
"city_name": "ashburn",
"country": "united states",
"country_iso_code": "us",
"location": {
"lat": 39.0481,
"lon": -77.4728
}
},
{
"ip": "54.174.253.3",
"as_num": 14618,
"as_org": "amazon-aes",
"isp": "amazon.com",
"city_name": "ashburn",
"country": "united states",
"country_iso_code": "us",
"location": {
"lat": 39.0481,
"lon": -77.4728
}
}
],
"http_extract": {
"cookies": [
{
"domain": "",
"expire": "0001-01-01T00:00:00Z",
"http_only": true,
"key": "MoodleSession",
"max_age": 0,
"path": "/",
"security": true,
"value": "tqhmqc4muk513sad1bmnl3kocj"
}
],
"description": "",
"emails": null,
"final_redirect_url": {
"full_uri": "https://ceonlineb2b.hms.harvard.edu/login/index.php",
"host": "ceonlineb2b.hms.harvard.edu",
"path": "/login/index.php"
},
"extracted_at": "2020-10-04T20:55:26.043777194Z",
"favicon_sha256": "",
"http_headers": [
{
"name": "date",
"value": "Sun, 04 Oct 2020 20:55:25 GMT"
},
{
"name": "content-type",
"value": "text/html; charset=utf-8"
},
{
"name": "server",
"value": "Apache/2.4.46 () OpenSSL/1.0.2k-fips"
},
{
"name": "x-powered-by",
"value": "PHP/7.2.24"
},
{
"name": "content-language",
"value": "en"
},
{
"name": "content-script-type",
"value": "text/javascript"
},
{
"name": "content-style-type",
"value": "text/css"
},
{
"name": "x-ua-compatible",
"value": "IE=edge"
},
{
"name": "cache-control",
"value": "private, pre-check=0, post-check=0, max-age=0, no-transform"
},
{
"name": "pragma",
"value": "no-cache"
},
{
"name": "expires",
"value": ""
},
{
"name": "accept-ranges",
"value": "none"
},
{
"name": "set-cookie",
"value": "MoodleSession=tqhmqc4muk513sad1bmnl3kocj; path=/; secure;HttpOnly;Secure;SameSite=None"
}
],
"http_status_code": 200,
"links": [
{
"anchor": "Forgotten your username or password?",
"url": "https://ceonlineb2b.hms.harvard.edu/login/forgot_password.php",
"url_host": "ceonlineb2b.hms.harvard.edu"
},
{
"anchor": "Privacy Statement",
"url": "/local/staticpage/view.php?page=privacy-statement",
"url_host": ""
},
{
"anchor": "Terms of Service",
"url": "/local/staticpage/view.php?page=terms-of-service",
"url_host": ""
},
{
"anchor": "Copyright Information",
"url": "/local/staticpage/view.php?page=copyright-information",
"url_host": ""
}
],
"meta_tags": [
{
"name": "keywords",
"value": "moodle, HMS Postgraduate Courses: Log in to the site"
},
{
"name": "format-detection",
"value": "telephone=no"
},
{
"name": "robots",
"value": "noindex"
},
{
"name": "viewport",
"value": "width=device-width, initial-scale=1.0"
}
],
"robots_txt": "",
"scripts": [
"https://ceonlineb2b.hms.harvard.edu/theme/yui_combo.php?rollup/3.17.2/yui-moodlesimple-min.js",
"https://ceonlineb2b.hms.harvard.edu/lib/javascript.php/1589465014/lib/javascript-static.js",
"https://ceonlineb2b.hms.harvard.edu/lib/javascript.php/1589465014/lib/requirejs/require.min.js",
"https://ceonlineb2b.hms.harvard.edu/theme/javascript.php/hms/1589465013/footer"
],
"styles": [
"https://ceonlineb2b.hms.harvard.edu/theme/yui_combo.php?rollup/3.17.2/yui-moodlesimple-min.css",
"https://ceonlineb2b.hms.harvard.edu/theme/styles.php/hms/1589465013_1/all"
],
"title": "HMS Postgraduate Courses: Log in to the site"
},
"is_CNAME": null,
"is_MX": null,
"is_NS": null,
"is_PTR": null,
"is_subdomain": true,
"name_without_suffix": "ceonlineb2b.hms.harvard",
"updated_at": "2021-05-16T10:25:01.59086376Z",
"user_scan_at": null,
"whois_parsed": null,
"security_score": {
"score": 100
},
"cve_list": null,
"technologies": [
{
"name": "Moodle",
"version": ""
},
{
"name": "RequireJS",
"version": ""
}
],
"trackers": null,
"organizations": null
}
]
}
}
For json parsing on bash, I recommend checking out jq. It's lightweight and versatile.
We can use the -r flag to output only values.
Output the fields of each object with the keys in sorted order.
--raw-output / -r:
The structure of the JSON you provided has the subdomain at .data.items[].http_extract.final_redirect_url.host
{
"data": {
"items": [
{
"http_extract": {
"final_redirect_url": {
"full_uri": "https://ceonlineb2b.hms.harvard.edu/login/index.php",
"host": "ceonlineb2b.hms.harvard.edu",
"path": "/login/index.php"
},
...
I've saved your json to a file, se.json
Example extracting full domain with jq
jq -r '.data.items[].http_extract.final_redirect_url.host' se.json
Output
ceonlineb2b.hms.harvard.edu
To extract the subdomain, just perform a search/replace using sub().
sub(regex; tostring) sub(regex; string; flags)
Emit the string obtained by replacing the first match of regex in the input string with tostring, after interpolation. tostring should be a jq string, and may contain references to named captures. The named captures are, in effect, presented as a JSON object (as constructed by capture) to tostring, so a reference to a captured variable named "x" would take the form: "(.x)".
Extracting subdomain using jq
jq -r '.data.items[].http_extract.final_redirect_url.host | sub(".hms.harvard.edu";"")' se.json
Output
ceonlineb2b
I have a big file that has lines as below.
{
"total": 320,
"assets": [
{
"audit": {
"created": {
"date": "2019-09-30T12:38:01.421Z"
},
"updated": {}
},
"organizationId": "12345678",
"id": 211123898,
"name": "groupId:760c47ad-c9f2958be:assetId:8o-api",
"exchangeAssetName": "8O API",
"groupId": "760c47ad-c9f2958be",
"assetId": "8o-api",
"apis": [
{
"audit": {
"created": {
"date": "2019-09-30T12:38:03.139Z"
},
"updated": {
"date": "2020-03-09T21:37:55.745Z"
}
},
"organizationId": "12345678",
"id": 15822364,
"groupId": "760c47ad-c9f2958be",
"assetId": "8o-api",
"assetVersion": "1.0.0",
"productVersion": "v1",
"description": null,
"tags": [],
"order": 1,
"providerId": null,
"deprecated": false,
"lastActiveDate": "2021-01-15T22:43:33.881Z",
"isPublic": false,
"stage": "release",
"lastActiveDelta": 7,
"pinned": false,
"activeContractsCount": 6,
"autodiscoveryInstanceName": "v1:15822364"
}
],
"totalApis": 1,
"autodiscoveryApiName": "groupId:760c47ad-c9f2958be:assetId:8o-api"
},
{
"audit": {
"created": {
"date": "2018-06-22T19:41:35.760Z"
},
"updated": {
"date": "2018-09-13T06:20:51.151Z"
}
},
"organizationId": "760c47ad-c9f2958be",
"id": 210914379,
"name": "hips-ts",
"exchangeAssetName": "hips-ts",
"groupId": "760c47ad-c9f2958be",
"assetId": "hips-ts",
"apis": [
{
"audit": {
"created": {
"date": "2018-06-22T19:41:35.759Z"
},
"updated": {
"date": "2020-03-09T21:37:55.745Z"
}
},
"organizationId": "760c47ad-c9f2958be",
"id": 15470738,
"groupId": "760c47ad-c9f2958be",
"assetId": "hips-ts",
"assetVersion": "1.0.0",
"productVersion": "v1",
"description": null,
"tags": [],
"order": 1,
"providerId": null,
"deprecated": false,
"lastActiveDate": "2021-01-15T22:43:30.004Z",
"endpointUri": null,
"isPublic": false,
"stage": "release",
"lastActiveDelta": 11,
"pinned": false,
"activeContractsCount": 1,
"autodiscoveryInstanceName": "1-test"
}
],
"totalApis": 1,
"autodiscoveryApiName": "hips-ts"
}
]
}
I am trying to use jq to just get the assetId and the activeContractsCount from this in a comma separated way.
So my output for this text should be
8o-api, 6
hips-ts, 1
I tried the following wiht jq jq -r '[.assets[].assetId, .assets[].apis[].activeContractsCount]|#csv' and I tried mapping too but nothing seems to stick.
Can you help me here? Any help is appreciated.
The following will produce exactly two columns, one for the "assetID" and one for the count:
.assets[].apis[]
| [.assetId, .activeContractsCount]
| #csv
I figured it out as soon as I posted the question. But please let me know if there's a better way to do this.
jq -r '.assets[]|[.apis[].assetId, .apis[].activeContractsCount]|#csv'
I'm trying to push JSON from a service in to Firebase, and it's not accepting it.
I cant figure out why. JSONLint validates it as valid.
The error I get is:
error: "Invalid data; couldn't parse JSON object, array, or value."
Here's the JSON payload:
{
"app_id": "e4805b8d5a9f4032b0bb8b6d9c6726b8",
"archived": false,
"attachments": {
"form.xml": {
"content_type": "text/xml",
"length": 4500,
"url": "https://someurl.com/form.xml"
}
},
"build_id": "3c5703e20346462ebcb07a3f36d5fe9b",
"domain": "my-test",
"edited_by_user_id": null,
"edited_on": null,
"form": {
"#type": "data",
"#name": "Beneficiary Registration",
"#uiVersion": "1",
"#version": "1",
"#xmlns": "http://openrosa.org/formdesigner/123456",
"beneficiary_age": {
"beneficiary_age_num": "8",
"beneficiary_exact_age_ind": "N"
},
"beneficiary_gender_cd": "M",
"beneficiary_id": "Hhr-O6I3C5L-J3G1L",
"case": {
"#case_id": "a07972bc-1a34-4c84-90ea-91250639f2a4",
"#date_modified": "2020-03-17T20:57:26.986000Z",
"#user_id": "2275c340c48ac83b6852035b0a15b5d3",
"#xmlns": "http://someurl.org/case/transaction/v2"
},
"case_name": "Hhr-O6I3C5L-J3G1L|joel galager|Katsekera|Katsekera|Mpando|Katsekera|KE",
"existing_beneficiaries": "Jane Doe\n\nJoan Doe",
"existing_beneficiaries_label": "",
"household_information": {
"beneficiary_household_information_display": "",
"beneficiary_location_hierarchy_1": "KE",
"beneficiary_location_hierarchy_1_text": "Kenya",
"beneficiary_location_hierarchy_2": "HF0001",
"beneficiary_location_hierarchy_2_text": "Katsekera",
"beneficiary_location_hierarchy_3": "TA0001",
"beneficiary_location_hierarchy_3_text": "Mpando",
"beneficiary_location_hierarchy_4": "GHV0001",
"beneficiary_location_hierarchy_4_text": "Katsekera",
"beneficiary_location_hierarchy_5": "V0001",
"beneficiary_location_hierarchy_5_text": "Katsekera",
"correct_information": "",
"hh_first_name": "John",
"hh_full_name": "John Doe",
"hh_id_fk": "Hhr-O6I3C5L",
"hh_last_name": "Doe",
"household_information": ""
},
"meta": {
"#xmlns": "http://openrosa.org/jr/xforms",
"appVersion": "Formplayer Version: 2.47",
"app_build_version": 1,
"commcare_version": null,
"deviceID": "Formplayer",
"drift": "0",
"geo_point": null,
"instanceID": "123-321-232",
"timeEnd": "2020-03-17T20:57:26.986000Z",
"timeStart": "2020-03-17T20:57:13.958000Z",
"userID": "2275c340c48ac83b6852035b0a15b5d3",
"username": "some_username"
},
"name_group": {
"beneficiary_first_name": "joel",
"beneficiary_full_name": "joel galager",
"beneficiary_last_name": "galager"
},
"subcase_0": {
"case": {
"#case_id": "2a4bfe27-a5c3-4f3a-8540-5f8ded86db85",
"#date_modified": "2020-03-17T20:57:26.986000Z",
"#user_id": "abc123",
"#xmlns": "http://commcarehq.org/case/transaction/v2",
"create": {
"case_name": "Hhr-O6I3C5L-J3G1L|joel galager|Katsekera|Katsekera|Mpando|Katsekera|KE",
"case_type": "beneficiary_case",
"owner_id": "abc123"
},
"index": {
"parent": {
"#text": "a07972bc-1a34-4c84-90ea-91250639f2a4",
"#case_type": "household_case"
}
},
"update": {
"beneficiary_age_num": "8",
"beneficiary_exact_age_ind": "N",
"beneficiary_first_name": "joel",
"beneficiary_full_name": "joel galager",
"beneficiary_gender_cd": "M",
"beneficiary_id": "Hhr-O6I3C5L-J3G1L",
"beneficiary_last_name": "galager",
"beneficiary_location_hierarchy_1": "KE",
"beneficiary_location_hierarchy_1_text": "Kenya",
"beneficiary_location_hierarchy_2": "HF0001",
"beneficiary_location_hierarchy_2_text": "Katsekera",
"beneficiary_location_hierarchy_3": "TA0001",
"beneficiary_location_hierarchy_3_text": "Mpando",
"beneficiary_location_hierarchy_4": "GHV0001",
"beneficiary_location_hierarchy_4_text": "Katsekera",
"beneficiary_location_hierarchy_5": "V0001",
"beneficiary_location_hierarchy_5_text": "Katsekera",
"hh_id_fk": "Hhr-O6I3C5L"
}
}
}
},
"id": "d547ccea-503c-4c50-b974-38ed564ae78a",
"indexed_on": "2020-03-17T21:01:04.695654",
"initial_processing_complete": true,
"is_phone_submission": true,
"metadata": {
"appVersion": "Formplayer Version: 2.47",
"app_build_version": 1,
"commcare_version": null,
"deviceID": "Formplayer",
"drift": "0",
"geo_point": null,
"instanceID": "01010101",
"location": null,
"timeEnd": "2020-03-17T20:57:26.986000Z",
"timeStart": "2020-03-17T20:57:13.958000Z",
"userID": "2275c340c48ac83b6852035b0a15b5d3",
"username": "myusername"
},
"problem": null,
"received_on": "2020-03-17T20:57:27.179986Z",
"resource_uri": "",
"server_modified_on": "2020-03-17T20:57:27.382548Z",
"type": "data",
"uiversion": "1",
"version": "1"
}
This is probably because of #type inside form.
The # character can't be used in RTDB paths.
See How data is structured from the docs.
Keys cannot contain
.
$
#
[
]
/
ASCII control characters 0-31 or 127
I have the following JSON and I need to get id values for instances which do not have type = Jenkins
{
"data": [
{
"id": "35002399-6fd7-40b7-b0d0-8be64e4ec09c",
"name": "94Jenkins",
"url": "http://127.0.0.1:8084",
"authProvider": false,
"siteId": "cce1b6e2-4b5d-4455-ac96-6b5d4c0d901d",
"status": {
"status": "ONLINE"
},
"instanceStateReady": true,
"instanceState": {
"#type": "InstanceStateDto",
"version": "2.60.3"
},
"adminUser": "admin1",
"hasDRConfig": false,
"managed": true,
"type": "JENKINS",
"siteName": "City",
"lastRefreshTime": "2018-04-24T09:43:01.694Z"
},
{
"id": "5cd3caf6-bac1-4f07-8793-5f124b90eaf5",
"name": "RJO",
"url": "http://test.com",
"authProvider": false,
"status": {
"status": "UNAUTHORIZED"
},
"instanceStateReady": true,
"instanceState": {
"#type": "numberOfArtifacts",
"version": "5.5.2-m002",
"licenses": {
"RJO : artrjo-m": {
"type": "ENTERPRISE",
"validThrough": "Jun 12, 2021",
"licensedTo": "Test",
"licenseHash": "asdadsdb612bda1aae745bd2a3",
"expired": false
},
"RJO : artrjo-s1": {
"type": "ENTERPRISE",
"validThrough": "Jun 12, 2021",
"licensedTo": "JFrog",
"licenseHash": "asaswca236350205a3798c0fa3",
"expired": false
}
}
},
"adminUser": "jfmc",
"hasDRConfig": false,
"managed": false,
"warnings": [
"Site is missing",
"Failed to connect to the service. Please verify that the service information provided is correct."
],
"type": "ARTIFACTORY"
},
{
"id": "0727a49a-6c95-433e-9fc5-7e5c760cc76f",
"name": "NinetyTwo",
"url": "http:127.0.0.1:8081",
"authProvider": true,
"siteId": "cce1b6e2-4b5d-4455-ac96-6b5d4c0d901d",
"status": {
"status": "ONLINE"
},
"instanceStateReady": true,
"instanceState": {
"#type": "numberOfArtifacts",
"version": "5.9.0",
"licenses": {
"NinetyTwo": {
"type": "ENTERPRISE",
"validThrough": "Dec 30, 2018",
"licensedTo": "Test",
"licenseHash": "qweqwed95f712dbabee98184da52443",
"expired": false
}
}
},
"adminUser": "admin",
"hasDRConfig": false,
"managed": true,
"type": "ARTIFACTORY",
"serviceId": "jfrt#01c7g4c7hq0dpd0qa71r8c09sj",
"siteName": "Test",
"lastRefreshTime": "2018-04-24T09:43:01.698Z"
}
]
}
And I use $..[?(#.type!='JENKINS')].id path to receive id-s which relate to the instances with type NOT JENKINS, however JSON Extractor returns me the Jenkins's id too. The question is how can I receive ids for non-jenkins instances only?
Replace your JSON path expression with the below and it should work:
$.data.[*][?(#.type != "JENKINS")].id