I have an IoT Device sending data to Azure IoT. I am trying to get a select statement query in Stream Analytics to parse the data to PowerBI but the out is giving me a lot of null and it does not look like the structure is correct.
A sample of the raw input stream is below but sit is not all of it.
[
{
"v": [
{
"ts": 133034720942536260,
"CLIENTDCOutput:Date": "28/ 7/2022"
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Good_Bags": 225
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Notes": "OEE Data testing"
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Operator": "Andy"
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Palletised_Tonnage": 4.9
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Planned_Run_Time": 240
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Rejected_Bags": 10
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Rejected_Tonnage": 0.2
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Run_Time": 235
},
{
"ts": 133034720942536260,
"CLIENTDCOutput:Total_Tonnage": 5.1
}
],
"EventProcessedUtcTime": "2022-07-28T08:56:38.8542591Z",
"PartitionId": 2,
"EventEnqueuedUtcTime": "2022-07-28T08:55:10.4330000Z",
"IoTHub": {
"MessageId": "23",
"CorrelationId": null,
"ConnectionDeviceId": "Sc01af47gm04.secomea.com",
"ConnectionDeviceGenerationId": "637866488901561354",
"EnqueuedTime": "2022-07-28T08:55:10.3710000Z"
}
},
{
"v": [
{
"ts": 133034721179030620,
"CLIENTDCStopages:Area_Category": ""
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stopages_Operator": "Andy"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Area_code": "000"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stoppage_Date": "28 7 2022"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stoppages_Notes": ""
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stoppage_Code": "None"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stoppage_Type": "None"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Stoppage_category": "No Stoppage"
},
{
"ts": 133034721179030620,
"CLIENTDCStopages:Mins_Stopped": 0
}
],
"EventProcessedUtcTime": "2022-07-28T08:56:38.8542591Z",
"PartitionId": 2,
"EventEnqueuedUtcTime": "2022-07-28T08:56:10.6350000Z",
"IoTHub": {
"MessageId": "24",
"CorrelationId": null,
"ConnectionDeviceId": "Sc01af47gm04.secomea.com",
"ConnectionDeviceGenerationId": "637866488901561354",
"EnqueuedTime": "2022-07-28T08:56:10.5570000Z"
}
}
]
I am trying to get the data stored in a row that pivots (for want of a better word) on the "ts" value.
My query is set out below.
SELECT
Stamp.ArrayValue.[CLIENTDCOutput:Date] AS OutputDate,
Stamp.ArrayValue.ts AS TSStamp,
Stamp.ArrayValue.[CLIENTDCOutput:Rejected_Tonnage] AS Rejected_Tonnage,
Stamp.ArrayValue.[CLIENTDCOutput:Total_Tonnage] AS Total_Tonnage,
Stamp.ArrayValue.[CLIENTDCOutput:Run_Time] as Run_Time,
Stamp.ArrayValue. [CLIENTDCOutput:Good_Bags] AS Good_Bags,
Stamp.ArrayValue. [CLIENTDCOutput:Notes] AS Notes,
Stamp.ArrayValue. [CLIENTDCOutput:Operator] AS Operator,
Stamp.ArrayValue. [CLIENTDCOutput:Palletised_Tonnage] AS Palletised_Tonnage,
Stamp.ArrayValue. [CLIENTDCOutput:Planned_Run_Time] AS Planned_Run_Time,
Stamp.ArrayValue. [CLIENTDCOutput:Rejected_Bags] AS Rejected_Bags
INTO ClientOEEOutputPowerBI
FROM
ClientIoTInput
CROSS APPLY GetArrayElements(v) as Stamp
Any help would be greatly appreciated.
Related
I have this json object and I am trying to find a way to use regex in jq to search value in issuer_dn that is more than 8 characters. A regex like [A-Z]{8} should work but I am unable to get results. I am still learning jq and if there is any other tool that can be used then please share.
{
"ip": "127.0.0.1",
"data": {
"tls": {
"status": "success",
"protocol": "tls",
"result": {
"handshake_log": {
"server_hello": {
"version": {
"name": "TLSv1.2",
"value": 771
},
"random": "hhdshfhhdhfhshdh",
"session_id": "hjdsfyyueujhfjaskdfjjl",
"cipher_suite": {
"hex": "0xC014",
"name": "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
"value": 49172
},
"compression_method": 0,
"ocsp_stapling": false,
"ticket": false,
"secure_renegotiation": true,
"heartbeat": false,
"extended_master_secret": false
},
"server_certificates": {
"certificate": {
"raw": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC==",
"parsed": {
"version": 3,
"serial_number": "1234567890",
"signature_algorithm": {
"name": "SHA256-RSA",
"oid": "1.2.840.113549.1.1.11"
},
"issuer": {
"common_name": [
"ABC"
],
"country": [
"ABC"
],
"locality": [
"ABC"
],
"province": [
"ABC"
],
"organization": [
"ABC"
],
"organizational_unit": [
"ABC"
]
},
"issuer_dn": "C=ABCD, ST=ABCD, L=ABCD, O=ABCD, OU=ABCD, CN=ABCD",
"validity": {
"start": "2020-02-01T01:09:22Z",
"end": "2021-02-01T03:09:22Z",
"length": 7883663
},
"subject": {
"common_name": [
"ABC"
],
"country": [
"ABC"
],
"locality": [
"ABC"
],
"province": [
"ABC"
],
"organization": [
"ABC"
],
"organizational_unit": [
"ABC"
]
},
"subject_dn": "C=ABCD, ST=ABCD, L=ABCD, O=ABCD, OU=ABCD, CN=ABCD",
"subject_key_info": {
"key_algorithm": {
"name": "RSA"
},
"rsa_public_key": {
"exponent": 65537,
"modulus": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC",
"length": 2048
},
"fingerprint_sha256": "73hh3g39920jfjj38723bb3993hh3774994002"
},
"extensions": {
"basic_constraints": {
"is_ca": true
},
"authority_key_id": "73hh3g39920jfjj38723bb3993hh3774994002",
"subject_key_id": "73hh3g39920jfjj38723bb3993hh3774994002"
},
"signature": {
"signature_algorithm": {
"name": "SHA256-RSA",
"oid": "1.2.840.113549.1.1.11"
},
"value": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC",
"valid": true,
"self_signed": true
},
"fingerprint_md5": "73hh3g39920jfjj38723bb3993hh3774994002",
"fingerprint_sha1": "73hh3g39920jfjj38723bb3993hh3774994002",
"fingerprint_sha256": "73hh3g39920jfjj38723bb3993hh3774994002",
"tbs_noct_fingerprint": "73hh3g39920jfjj38723bb3993hh3774994002",
"spki_subject_fingerprint": "73hh3g39920jfjj38723bb3993hh3774994002",
"tbs_fingerprint": "73hh3g39920jfjj38723bb3993hh3774994002",
"validation_level": "73hh3g39920jfjj38723bb3993hh3774994002",
"redacted": false
}
},
"validation": {
"browser_trusted": false,
"browser_error": "x509: failed to load system roots and no roots provided"
}
},
"server_key_exchange": {
"ecdh_params": {
"curve_id": {
"name": "secp256r1",
"id": 23
},
"server_public": {
"x": {
"value": "73hh3g39920jfjj38723bb3993hh3774994002",
"length": 256
},
"y": {
"value": "73hh3g39920jfjj38723bb3993hh3774994002",
"length": 256
}
}
},
"digest": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC",
"signature": {
"raw": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC",
"type": "rsa",
"valid": true,
"signature_and_hash_type": {
"signature_algorithm": "rsa",
"hash_algorithm": "sha256"
},
"tls_version": {
"name": "TLSv1.2",
"value": 771
}
}
},
"client_key_exchange": {
"ecdh_params": {
"curve_id": {
"name": "secp256r1",
"id": 23
},
"client_public": {
"x": {
"value": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC=",
"length": 256
},
"y": {
"value": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC=",
"length": 256
}
},
"client_private": {
"value": "dGVzdCBkYXRhIGFuZCBnYXJiYWdldGVzdC=",
"length": 32
}
}
},
"client_finished": {
"verify_data": "dGVzdCBkY"
},
"server_finished": {
"verify_data": "dGVzdCBkY"
},
"key_material": {
"master_secret": {
"value": "dGVzdCBkY",
"length": 48
},
"pre_master_secret": {
"value": "dGVzdCBkY",
"length": 32
}
}
}
},
"timestamp": "2020-02-9T07:14:47Z"
}
}
}
I would use the following :
.data.tls.result.handshake_log.server_certificates.certificate.parsed.issuer_dn \
| select(length > 8)
You can try it here.
You could use:
test("[A-Z]{8}")
I am new to python, using python3. I have json data like:
{
"message": {
"count": 46,
"limit": 1000,
"schools": [
{
"class": "1",
"class_id": "1c8***",
"charges": [
{
"cost": 10,
"breakdown": [
{
"books": "1",
"unitQuantity": "10"
}
]
}
],
"area": "maccau"
},
{
"class": "2",
"class_id": "1c3***",
"charges": [
{
"cost": 100,
"breakdown": [
{
"books": "1",
"unitQuantity": "100"
}
]
}
],
"area": "maccau"
},
{
"class": "1",
"class_id": "1c3***",
"charges": [
{
"cost": 10,
"breakdown": [
{
"books": "1",
"unitQuantity": "10"
}
]
}
],
"area": "maccau"
},
{
"class": "2",
"class_id": "1c8***",
"charges": [
{
"cost": 50,
"breakdown": [
{
"books": "1",
"unitQuantity": "50"
}
]
}
],
"area": "maccau"
}
],
"url": {
"link": "/"
}
}
}
I was able to use json.loads to load data and I am trying to get results like:
class Cost
1 20
2 150
I tried converting json to a dictionary:
item_dict = json.load(json_data)
Tried to get data out using for loop and checking if class = 1 and then summing up the cost. But I feel like that is not the best approach. Can someone please tell me what would be the best way of doing this?
My json is like below. I am trying to group my objects based on the role id for target id equal to 1083.
My json:
[ {
"id" :1,
"role": {
"id": "25",
},
"target": {
"id": "1083",
}
},
{
"id" :2,
"role": {
"id": "25",
},
"target": {
"id": "1083",
}
},
{
"id" :3,
"role": {
"id": "25",
},
"target": {
"id": "1084",
}
},
{
"id" :4,
"role": {
"id": "3",
},
"target": {
"id": "1083",
}
}
]
Expected result:
{
"25" : [
{
"id": 1
"role": {
"id": "25",
},
"target": {
"id": "1083",
}
},
{ "id": 2
"role": {
"id": "25",
},
"target": {
"id": "1083",
}
}
],
"3": [
{
"id" :4,
"role": {
"id": "3",
},
"target": {
"id": "1083",
}
}]
}
So for my result json, it's grouped by 25 and 3. The third record doesn't exist in my expected result as it's target id is 1084
I tried filter to get only target id 1083, but when I tried to use grouby, I am getting errors
data.filter(o => {
return o.target.id === "1083"
})
With lodash via filter and groupBy:
var data = [{ "id": 1, "role": { "id": "25", }, "target": { "id": "1083", } }, { "id": 2, "role": { "id": "25", }, "target": { "id": "1083", } }, { "id": 3, "role": { "id": "25", }, "target": { "id": "1084", } }, { "id": 4, "role": { "id": "3", }, "target": { "id": "1083", } } ]
const groupIt = (targetId) => _.chain(data)
.filter(x => x.target.id ===targetId)
.groupBy('role.id')
.value()
console.log(groupIt('1083'))
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js"></script>
You could achieve the same without lodash via filter and then reduce:
var data = [{ "id": 1, "role": { "id": "25", }, "target": { "id": "1083", } }, { "id": 2, "role": { "id": "25", }, "target": { "id": "1083", } }, { "id": 3, "role": { "id": "25", }, "target": { "id": "1084", } }, { "id": 4, "role": { "id": "3", }, "target": { "id": "1083", } } ]
const groupIt = (targetId) => data.filter(x => x.target.id === targetId)
.reduce((r, c) => (r[c.role.id] = [...r[c.role.id] || [], c], r), {})
console.log(groupIt('1083'))
My mule code is hitting two tables and getting some details.
First one is order details, which I am storing in a flow variable i.e order and another database is returning order item details which I am storing in orderitem variable.
I want to aggregate both the payload based on one condition. Every orderId has order items (which is stored in flowVars.orderitem) and map these order items to respective orderID.
flowVars.order value is as below
[{partnerId=e83185e9f33e4234ba9eaa81dba515ad, orderId=12345, orderDate=2017-02-28 16:41:41.0, id=22}, {partnerId=e83185e9f33e4234ba9eaa81dba515ad, orderId=123456, orderDate=2017-02-28 16:41:41.0, id=23}, {partnerId=e83185e9f33e4234ba9eaa81dba515ad, orderId=11111, orderDate=2017-02-28 16:41:41.0, id=24}, {partnerId=e83185e9f33e4234ba9eaa81dba515ad, orderId=321123, orderDate=2017-05-19 15:25:41.0, id=26}]
and flowVars.orderitem value is as below
[{productCode=ELT-LP-ICND1-020067, orderId=12345, quantity=10, id=14}, {productCode=ELT-IP-ICND1-1.0, orderId=12345, quantity=11, id=15}, {productCode=ELT-LP-ICND1-020067, orderId=123456, quantity=12, id=16}, {productCode=ELT-IP-ICND1-1.0, orderId=123456, quantity=13, id=17}, {productCode=ELT-LP-ICND1-020067, orderId=11111, quantity=14, id=18}, {productCode=ELT-IP-ICND1-1.0, orderId=11111, quantity=15, id=19}, {productCode=ELT-LP-ICND2-020067, orderId=321123, quantity=5, id=20}]
Expected Output
[
{
"orderId": "12345",
"orderDate": "2017-02-28T16:41:41",
"partnerId": "e83185e9f33e4234ba9eaa81dba515ad",
"orderItems": [
{
"productCode": "ELT-LP-ICND1-020067",
"quantity": "10"
},
{
"productCode": "ELT-IP-ICND1-1.0",
"quantity": "11"
}
]
},
{
"orderId": "123456",
"orderDate": "2017-02-28T16:41:41",
"partnerId": "e83185e9f33e4234ba9eaa81dba515ad",
"orderItems": [
{
"productCode": "ELT-LP-ICND1-020067",
"quantity": "12"
},
{
"productCode": "ELT-IP-ICND1-1.0",
"quantity": "13"
}
]
},
{
"orderId": "11111",
"orderDate": "2017-02-28T16:41:41",
"partnerId": "e83185e9f33e4234ba9eaa81dba515ad",
"orderItems": [
{
"productCode": "ELT-LP-ICND1-020067",
"quantity": "14"
},
{
"productCode": "ELT-IP-ICND1-1.0",
"quantity": "15"
}
]
},
{
"orderId": "321123",
"orderDate": "2017-05-19T15:25:41",
"partnerId": "e83185e9f33e4234ba9eaa81dba515ad",
"orderItems": [
{
"productCode": "ELT-LP-ICND1-020067",
"quantity": "5"
}
]
}
]
Here I need to show respective order item details of an order. So basically I need to combine both the payloads.
I tried using dataweave but not luck.
Dataweave code:
%dw 1.0
%output application/json
%var mergeddata = flowVars.orderitem groupBy $.orderId
---
flowVars.order map ((data,index) ->
{
orderid: data.orderId,
partnerid: data.partnerId,
orderdate: data.orderDate,
order: flowVars.orderitem default [] map ((data1 ,indexOf) ->
{
(productcode: data1.productCode) when (data1.orderId == data.orderId),
(quantity: data1.quantity) when (data1.orderId == data.orderId) ,
(id: data1.id) when (data1.orderId == data.orderId)
}
)})
And output after transformation:
{
"orderid": "12345",
"partnerid": "e83185e9f33e4234ba9eaa81dba515ad",
"orderdate": "2017-02-28T16:41:41",
"order": [
{
"productcode": "ELT-LP-ICND1-020067",
"quantity": 10,
"id": 14
},
{
"productcode": "ELT-IP-ICND1-1.0",
"quantity": 11,
"id": 15
},
{
},
{
},
{
},
{
},
{
}
]
},
{
"orderid": "123456",
"partnerid": "e83185e9f33e4234ba9eaa81dba515ad",
"orderdate": "2017-02-28T16:41:41",
"order": [
{
},
{
},
{
"productcode": "ELT-LP-ICND1-020067",
"quantity": 12,
"id": 16
},
{
"productcode": "ELT-IP-ICND1-1.0",
"quantity": 13,
"id": 17
},
{
},
{
},
{
}
]
},
{
"orderid": "11111",
"partnerid": "e83185e9f33e4234ba9eaa81dba515ad",
"orderdate": "2017-02-28T16:41:41",
"order": [
{
},
{
},
{
},
{
},
{
"productcode": "ELT-LP-ICND1-020067",
"quantity": 14,
"id": 18
},
{
"productcode": "ELT-IP-ICND1-1.0",
"quantity": 15,
"id": 19
},
{
}
]
},
{
"orderid": "321123",
"partnerid": "e83185e9f33e4234ba9eaa81dba515ad",
"orderdate": "2017-05-19T15:25:41",
"order": [
{
},
{
},
{
},
{
},
{
},
{
},
{
"productcode": "ELT-LP-ICND2-020067",
"quantity": 5,
"id": 20
}
]
}
]
As you can see that I am almost there and able to map order item details with respective orderId but still I am getting some blank values after transformation.
Can anyone help me to achieve expected output. Thanks in advance!!!
You need to filter the flowVars.orderitem map, rather than iterate it in full and only print values when the orderId matches.
order: ((flowVars.orderitem default []) filter (data.orderId == $.orderId)) map ((data1 ,indexOf) -> {
productcode: data1.productCode,
quantity: data1.quantity
id: data1.id
})
You can then remove all of those 'when' statements too.
I'm trying to read element's values.
For test purposes I was trying to read account_name element value.
I've tried this
jsonloop query = "data/ads/data/insights/data[*]"
account_name = "$.data.ads.data.insights.data.[*].account_name"
But not getting result. Apart from this I've googled alot and my logic seems to be okay..but I dont know where am doing wrong.
{
"data": [{
"id": "act_78425484545145418"
}, {
"id": "act_87814545415645416"
}, {
"ads": {
"data": [{
"insights": {
"data": [{
"account_name": "Emirates WW",
"campaign_name": "FR_Ozone_aug",
"adset_name": "Android",
"ad_name": "C_mail_9sept",
"spend": 12.2,
"impressions": "1146",
"clicks": "16",
"_store_clicks": "0",
"inline_click_pointss": "16",
"actions": [{
"action_type": "custom_event._activate_",
"value": 2
}, {
"action_type": "custom_event.other",
"value": 1
}, {
"action_type": "click_points",
"value": 16
}, {
"action_type": "__install",
"value": 1
}, {
"action_type": "offsite_points.view_content_points",
"value": 25
}, {
"action_type": "post_points",
"value": 16
}, {
"action_type": "post_points",
"value": 16
}, {
"action_type": "offsite_points",
"value": 25
}, {
"action_type": "custom_event",
"value": 3
}],
"date_start": "2016-09-09",
"date_stop": "2016-09-19"
}],
"paging": {
"cursors": {
"before": "ADZZZZ",
"after": "ADZZZZ"
}
}
},
"id": "6054027758549"
}, {
"insights": {
"data": [{
"account_name": "Emirates WW",
"campaign_name": "FR_Ozone_aug",
"adset_name": "Reta_Phy_iOS",
"ad_name": "Council_email_9sept",
"spend": 10.13,
"impressions": "1004",
"clicks": "10",
"_store_clicks": "8",
"inline_click_pointss": "10",
"actions": [{
"action_type": "click_points",
"value": 10
}, {
"action_type": "offsite_points.view_content_points",
"value": 38
}, {
"action_type": "post_points",
"value": 10
}, {
"action_type": "post_points",
"value": 10
}, {
"action_type": "offsite_points",
"value": 38
}],
"date_start": "2016-09-09",
"date_stop": "2016-09-19"
}],
"paging": {
"cursors": {
"before": "ADZZZZ",
"after": "ADZZZZ"
}
}
},
"id": "6054030075149"
}],
"paging": {
"cursors": {
"before": "Juysubgysbyy",
"after": "Nhwunhseuubeyegb"
}
}
}
}]
}
Can anyone help ?
Thanks in Advance !!
First and second data properties have array values, so you should add .[*] after them:
account_name = "$.data.[*].ads.data.[*].insights.data.[*].account_name"
The result:
[
"Emirates WW",
"Emirates WW"
]