I have the code, which reads perfectly fine, when the referred sections are present in JSON and it erros out when referred section is not present. Trying to workaround to still read that section and return NULL values when not found.
Input data:
{
"ProjectId": "P.2001850",
"OperationId": "O.2001850.01",
"ActivityId": "A.2001850.01.04",
"Description": "",
"Combos": [
{
"ComboId": "28632",
"Demands": {
"DownHoleTools": {
"PrimaryTools": [
],
"BackupTools": [
]
},
"SurfaceTools": {
"PrimaryTools": [
],
"BackupTools": [
]
},
"Techniques": {
"PrimaryTools": [
],
"BackupTools": [
]
},
"Services": [
],
"Tools": ""
},
"ComboType": 0,
"HashCode": "",
"SequenceNumber": "",
"ConveyanceInfo": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "0f987389-f724-4954-94fb-fd9b8f981951"
},
{
"ComboId": "28631",
"Demands": {
"DownHoleTools": "",
"SurfaceTools": "",
"Techniques": {
"PrimaryTools": [
],
"BackupTools": [
]
},
"Services": [
],
"Tools": {
"PrimaryTools": [
{
"ToolCode": "ToolName",
"ToolDescription": "WDIS9",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-04-02T05:00:00",
"EndDate": "2020-04-05T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "WDP"
},
"BrandInfo": {
"Code": "DM-WDP",
"Description": "WDP",
"WkId": "6:DM-WDP"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "d5c2d518-e6b0-4b1f-b490-5d22f14efd9e"
},
{
"ToolCode": "ToolName",
"ToolDescription": "TeleScope 825",
"ToolSize": "825",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-25T05:00:00",
"EndDate": "2020-03-31T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Mud Pulse"
},
"BrandInfo": {
"Code": "DM-TEL",
"Description": "[DM-TEL] TELESCOPE",
"WkId": "6:DM-TEL"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "b6fd7582-6850-4ea4-a1ab-2caa69a956c2"
},
{
"ToolCode": "ToolName",
"ToolDescription": "SDNSC 475 - TOOL ASSY",
"ToolSize": "475",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-25T05:00:00",
"EndDate": "2020-03-31T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Logging While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Nuclear"
},
"BrandInfo": {
"Code": "DM-IDNSC",
"Description": "[DM-IDNSC] IDNSC",
"WkId": "6:DM-IDNSC"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "4aeb2a19-6dfd-43ab-8dbb-ee79f607d326"
},
{
"ToolCode": "ToolName",
"ToolDescription": "IDNSC 675 - TOOL ASSY",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-25T05:00:00",
"EndDate": "2020-03-31T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Logging While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Nuclear"
},
"BrandInfo": {
"Code": "DM-IDNSC",
"Description": "[DM-IDNSC] IDNSC",
"WkId": "6:DM-IDNSC"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "1ab693cd-61ff-490b-9dff-6b8a77a8727c"
},
{
"ToolCode": "ToolName",
"ToolDescription": "CTD-MWD",
"ToolSize": "",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-18T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Directional Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Coil Tubing Drilling"
},
"BrandInfo": {
"Code": "DM-FD",
"Description": "COIL TUBING DRILLING",
"WkId": "6:DM-FD"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "c06a357b-6c8c-44e3-83cf-8c4d40cad1e4"
},
{
"ToolCode": "ToolName",
"ToolDescription": "FUSION Cycle Disconnect",
"ToolSize": "",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-18T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Directional Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Coil Tubing Drilling"
},
"BrandInfo": {
"Code": "DM-FD",
"Description": "COIL TUBING DRILLING",
"WkId": "6:DM-FD"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "7c434710-63c4-4a84-8221-63a53682019f"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM675",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-18T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "test",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "2045d05f-ac3c-40a5-aca4-714da606f844"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM900",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-18T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "890dae1e-8204-47d7-9335-9fb5604e7ca3"
},
{
"ToolCode": "ToolName",
"ToolDescription": "FUSION Orienting Tool",
"ToolSize": "",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-18T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Directional Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Coil Tubing Drilling"
},
"BrandInfo": {
"Code": "DM-FD",
"Description": "[DM-FD] COIL TUBING DRILLING",
"WkId": "6:DM-FD"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "b89c973a-d9fc-4241-8e77-f1651b80a4e4"
},
{
"ToolCode": "ToolName",
"ToolDescription": "GRPI8",
"ToolSize": "825",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-19T05:00:00",
"EndDate": "2020-03-28T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "WDP"
},
"BrandInfo": {
"Code": "DM-WDP",
"Description": "[DM-WDP] WDP",
"WkId": "6:DM-WDP"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "881abae1-3e3c-4df7-9663-afeff5b5eb11"
}
],
"BackupTools": [
{
"ToolCode": "ToolName",
"ToolDescription": "ShortPulse 475",
"ToolSize": "475",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-04-01T05:00:00",
"EndDate": "2020-04-05T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Mud Pulse"
},
"BrandInfo": {
"Code": "DM-BMP",
"Description": "SHORTPULSE",
"WkId": "6:DM-BMP"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "637a4465-0045-49f5-963d-ee65b93478b1"
},
{
"ToolCode": "ToolName",
"ToolDescription": "TeleScope 900",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-16T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Mud Pulse"
},
"BrandInfo": {
"Code": "DM-TEL",
"Description": "[DM-TEL] TELESCOPE",
"WkId": "6:DM-TEL"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "dcf0fbef-05af-44af-bde1-2acdd09d2b85"
},
{
"ToolCode": "ToolName",
"ToolDescription": "IDNSC 675 - TOOL ASSY",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-16T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Logging While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Nuclear"
},
"BrandInfo": {
"Code": "DM-IDNSC",
"Description": "[DM-IDNSC] IDNSC",
"WkId": "6:DM-IDNSC"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "401f8e09-6591-4f6b-92ca-e207b1679eed"
},
{
"ToolCode": "ToolName",
"ToolDescription": "SONICPACER 675 - TOOL ASSY",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-16T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Logging While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Sonic"
},
"BrandInfo": {
"Code": "DM-SONP",
"Description": "SONICPACER",
"WkId": "6:DM-SONP"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "91feada7-807e-46dd-9fe8-17e255dbf87f"
},
{
"ToolCode": "ToolName",
"ToolDescription": "GDIS 900",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-16T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Gyro"
},
"BrandInfo": {
"Code": "DM-GDIS",
"Description": "GDIS",
"WkId": "6:DM-GDIS"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "446a1535-848d-49f6-a6a3-53c2102aee46"
},
{
"ToolCode": "ToolName",
"ToolDescription": "WDIS9",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-04-01T05:00:00",
"EndDate": "2020-04-05T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "WDP"
},
"BrandInfo": {
"Code": "DM-WDP",
"Description": "WDP",
"WkId": "6:DM-WDP"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "70b21610-ff2f-4c87-9132-1b2d6dd59fa0"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM675",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-04-01T05:00:00",
"EndDate": "2020-04-05T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "aa0aa23c-d3f6-4920-897d-ecd3a10cde59"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM900",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-12T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "f9f5fb49-7ec5-40fe-a737-9b145ae800a6"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM675",
"ToolSize": "675",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-12T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "b68c5c40-8cca-4ed0-a82c-a11a04a216a3"
},
{
"ToolCode": "ToolName",
"ToolDescription": "DMM900",
"ToolSize": "900",
"Source": "A",
"DemandDurationInfo": {
"StartDate": "2020-03-12T05:00:00",
"EndDate": "2020-03-20T05:00:00"
},
"ToolGroupInfo": {
"Code": "ToolGroup",
"Description": "Measurements While Drilling"
},
"ToolTypeInfo": {
"Code": "ToolType",
"Description": "Drilling Mechanics"
},
"BrandInfo": {
"Code": "DM-DMM",
"Description": "OPTIDRILL",
"WkId": "6:DM-DMM"
},
"Comments": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "41eb924d-fe79-4743-a981-b9af73e272cb"
}
]
}
},
"ComboType": 1,
"HashCode": "",
"SequenceNumber": "",
"ConveyanceInfo": "",
"CreatedDate": "0001-01-01T00:00:00",
"CreatedBy": "",
"LastModifiedDate": "0001-01-01T00:00:00",
"LastModifiedBy": "",
"Id": "20b7a14b-b038-40a0-9da3-d13708838332"
}
],
"FmpEdpId": "5e7ae62757b51d0001cbf6b5",
"CreatedDate": "2020-03-13T20:28:27.539",
"CreatedBy": "FDPSystemUser",
"LastModifiedDate": "2020-03-26T11:38:28.621",
"LastModifiedBy": "RNighot",
"Id": "5e6beceb3d6a2f7154aaed9f"
}
Code:
CREATE TEMPORARY FUNCTION CUSTOM_JSON_EXTRACT(json STRING, json_path STRING)
RETURNS ARRAY<STRING> LANGUAGE js AS '''
var result = jsonPath(JSON.parse(json), json_path);
if(result){return result;}
else {return [];}
'''
OPTIONS (library='gs://slb-it-op-dev-temp-dev-workspace/json_temp/jsonpath-0.8.0.js');
CREATE TEMP FUNCTION jsonparse(input STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
var result= JSON.parse(input).map(x=>JSON.stringify(x));
if(result){return result;}
else {return [];}
"""
OPTIONS (
library="gs://slb-it-op-dev-temp-dev-workspace/json_temp/jsonpath-0.8.0.js"
);
SELECT
job_id,
combo_id,
JSON_EXTRACT_SCALAR(combo, '$.ComboType') Combo_Type,
JSON_EXTRACT_SCALAR(combo, '$.HashCode') Combo_Hash_code,
JSON_EXTRACT_SCALAR(combo, '$.Id') Combos_Id,
JSON_EXTRACT_SCALAR(primary_tools, '$.ToolDescription') tool_name,
JSON_EXTRACT_SCALAR(primary_tools, '$.ToolSize') tool_size,
JSON_EXTRACT_SCALAR(primary_tools, '$.Source') Source,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.Code') Brand_Info_Code,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.Description') Brand_Info_Description,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.WkId') Brand_Info_WKID,
FROM temp_db.eqp_temp,
UNNEST(jsonparse(JSON_EXTRACT(conv_column, '$.Combos'))) combo
LEFT JOIN UNNEST(CUSTOM_JSON_EXTRACT(combo, '$.ComboId')) combo_id
LEFT JOIN UNNEST(jsonparse(JSON_EXTRACT(combo, '$.Demands.Tools.PrimaryTools'))) primary_tools
Output:
we need to read the comboid "28632" for '$.Demands.Tools.PrimaryTools', catch here is "Tools" section doesnt have "PrimaryTools" section for this combo.
The only minor fix was required in TEMP FUNCTION jsonparse(input STRING) as in below code
#standardSQL
CREATE TEMPORARY FUNCTION CUSTOM_JSON_EXTRACT(json STRING, json_path STRING)
RETURNS ARRAY<STRING> LANGUAGE js AS '''
var result = jsonPath(JSON.parse(json), json_path);
if(result){return result;}
else {return [];}
'''
OPTIONS (library='gs://slb-it-op-dev-temp-dev-workspace/json_temp/jsonpath-0.8.0.js');
CREATE TEMP FUNCTION jsonparse(input STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS '''
if(input){return JSON.parse(input).map(x => JSON.stringify(x));}
else {return [];}
'''
OPTIONS (library='gs://slb-it-op-dev-temp-dev-workspace/json_temp/jsonpath-0.8.0.js');
SELECT
job_id,
combo_id,
JSON_EXTRACT_SCALAR(combo, '$.ComboType') Combo_Type,
JSON_EXTRACT_SCALAR(combo, '$.HashCode') Combo_Hash_code,
JSON_EXTRACT_SCALAR(combo, '$.Id') Combos_Id,
JSON_EXTRACT_SCALAR(primary_tools, '$.ToolDescription') tool_name,
JSON_EXTRACT_SCALAR(primary_tools, '$.ToolSize') tool_size,
JSON_EXTRACT_SCALAR(primary_tools, '$.Source') Source,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.Code') Brand_Info_Code,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.Description') Brand_Info_Description,
JSON_EXTRACT_SCALAR(primary_tools, '$.BrandInfo.WkId') Brand_Info_WKID,
FROM `temp_db.eqp_temp`,
UNNEST(jsonparse(JSON_EXTRACT(conv_column, '$.Combos'))) combo
LEFT JOIN UNNEST(CUSTOM_JSON_EXTRACT(combo, '$.ComboId')) combo_id
LEFT JOIN UNNEST(jsonparse(JSON_EXTRACT(combo, '$.Demands.Tools.PrimaryTools'))) primary_tools
with output
Row job_id combo_id Combo_Type Combo_Hash_code Combos_Id tool_name tool_size Source Brand_Info_Code Brand_Info_Description Brand_Info_WKID
1 1 28632 0 0f987389-f724-4954-94fb-fd9b8f981951 null null null null null null
2 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 WDIS9 900 A DM-WDP WDP 6:DM-WDP
3 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 TeleScope 825 825 A DM-TEL [DM-TEL] TELESCOPE 6:DM-TEL
4 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 SDNSC 475 - TOOL ASSY 475 A DM-IDNSC [DM-IDNSC] IDNSC 6:DM-IDNSC
5 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 IDNSC 675 - TOOL ASSY 675 A DM-IDNSC [DM-IDNSC] IDNSC 6:DM-IDNSC
6 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 CTD-MWD A DM-FD COIL TUBING DRILLING 6:DM-FD
7 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 FUSION Cycle Disconnect A DM-FD COIL TUBING DRILLING 6:DM-FD
8 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 DMM675 675 A DM-DMM OPTIDRILL 6:DM-DMM
9 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 DMM900 900 A DM-DMM OPTIDRILL 6:DM-DMM
10 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 FUSION Orienting Tool A DM-FD [DM-FD] COIL TUBING DRILLING 6:DM-FD
11 1 28631 1 20b7a14b-b038-40a0-9da3-d13708838332 GRPI8 825 A DM-WDP [DM-WDP] WDP 6:DM-WDP
My question is: What would be the correct syntax to add 2 or more tables to an existing database using xmla file and SSMS? I have a model deployed to Azure Analysis Services server. So the database is already created. I want to create or replace tables by running one single xmla script in SSMS.
When I use the script below, to create one table, it works perfectly. But what I need is to create several tables (not only one) using one single xmla script.
Script that works for one table
The script below works perfectly and correctly "creates or replaces" one table on the database.
{
"createOrReplace": {
"object": {
"database": "MyDatabase",
"table": "MyTable"
},
"table": {
"name": "MyTable",
"columns": [
{
"name": "MyTableId",
"dataType": "int64",
"sourceColumn": "MyTableId"
},
{
"name": "MyTable",
"dataType": "string",
"sourceColumn": "MyTable"
}
],
"partitions": [
{
"name": "Partition",
"dataView": "full",
"source": {
"type": "m",
"expression": [
"let",
" Source=GetFileList(),",
" #\"MyTable txt\" = Source{[Name=\"MyTable.txt\"]}[Content],",
" #\"Imported CSV\" = Csv.Document(#\"MyTable txt\",[Delimiter=\",\", Columns=9, Encoding=1252, QuoteStyle=QuoteStyle.None]),",
" #\"Promoted Headers\" = Table.PromoteHeaders(#\"Imported CSV\", [PromoteAllScalars=true]),",
" #\"Changed Type\" = Table.TransformColumnTypes(#\"Promoted Headers\",{{\"MyTableId\", Int64.Type}, {\"MyTable\", type text}, {\"Description\", type text}}),",
" #\"Removed Columns\" = Table.RemoveColumns(#\"Changed Type\",{\"Description\"})",
"in",
" #\"Removed Columns\""
]
}
}
]
}
}
}
I tried the code below to add 2 tables but got an error The JSON DDL request failed with the following error: Unrecognized JSON property: tables. Check path 'tables', line 6, position 16..
{
"createOrReplace": {
"database": {
"name": "MyDatabase",
"tables": [
{"name": "TableA",
"columns": [
{
"name": "TableAId",
"dataType": "int64",
"sourceColumn": "TableAId"
},
{
"name": "TableA",
"dataType": "string",
"sourceColumn": "TableA"
}
],
"partitions": [
{
"name": "Partition",
"dataView": "full",
"source": {
"type": "m",
"expression": [
"let",
" Source=GetFileList(),",
" #\"TableA txt\" = Source{[Name=\"TableA.txt\"]}[Content],",
" #\"Imported CSV\" = Csv.Document(#\"TableA txt\",[Delimiter=\",\", Columns=9, Encoding=1252, QuoteStyle=QuoteStyle.None]),",
" #\"Promoted Headers\" = Table.PromoteHeaders(#\"Imported CSV\", [PromoteAllScalars=true]),",
" #\"Changed Type\" = Table.TransformColumnTypes(#\"Promoted Headers\",{{\"TableAId\", Int64.Type}, {\"TableA\", type text}, {\"Description\", type text}}),",
" #\"Removed Columns\" = Table.RemoveColumns(#\"Changed Type\",{\"Description\"})",
"in",
" #\"Removed Columns\""
]
}
}
] },
{"name": "TableB",
"columns": [
{
"name": "TableBId",
"dataType": "int64",
"sourceColumn": "TableBId"
},
{
"name": "TableB",
"dataType": "string",
"sourceColumn": "TableB"
}
],
"partitions": [
{
"name": "Partition",
"dataView": "full",
"source": {
"type": "m",
"expression": [
"let",
" Source=GetFileList(),",
" #\"TableB txt\" = Source{[Name=\"TableB.txt\"]}[Content],",
" #\"Imported CSV\" = Csv.Document(#\"TableB txt\",[Delimiter=\",\", Columns=11, Encoding=1252, QuoteStyle=QuoteStyle.None]),",
" #\"Promoted Headers\" = Table.PromoteHeaders(#\"Imported CSV\", [PromoteAllScalars=true]),",
" #\"Changed Type\" = Table.TransformColumnTypes(#\"Promoted Headers\",{{\"TableBId\", Int64.Type}, {\"TableB\", type text}, {\"Description\", type text}}),",
" #\"Removed Columns\" = Table.RemoveColumns(#\"Changed Type\",{\"Description\"})",
"in",
" #\"Removed Columns\""
]
}
}
] }
]
}
}
}
I also tried the code below but got error Error -1055784777: The JSON DDL request failed with the following error: Unrecognized JSON property: tables. Check path 'createOrReplace.tables', line 6, position 14..
The JSON DDL request failed with the following error: Unrecognized JSON property: tables. Check path 'createOrReplace.tables', line 6, position 14..
{
"createOrReplace": {
"object": {
"database": "MyDatabase"
},
"tables": [
{
"name": "TableA",
"columns": [
{
"name": "TableAId",
"dataType": "int64",
"sourceColumn": "TableAId"
},
{
"name": "TableA",
"dataType": "string",
"sourceColumn": "TableA"
}
],
"partitions": [
{
"name": "Partition",
"dataView": "full",
"source": {
"type": "m",
"expression": [
"let",
" Source=GetFileList(),",
" #\"TableA txt\" = Source{[Name=\"TableA.txt\"]}[Content],",
" #\"Imported CSV\" = Csv.Document(#\"TableA txt\",[Delimiter=\",\", Columns=9, Encoding=1252, QuoteStyle=QuoteStyle.None]),",
" #\"Promoted Headers\" = Table.PromoteHeaders(#\"Imported CSV\", [PromoteAllScalars=true]),",
" #\"Changed Type\" = Table.TransformColumnTypes(#\"Promoted Headers\",{{\"TableAId\", Int64.Type}, {\"TableA\", type text}, {\"Description\", type text}}),",
" #\"Removed Columns\" = Table.RemoveColumns(#\"Changed Type\",{\"Description\"})",
"in",
" #\"Removed Columns\""
]
}
}
]
},
{
"name": "TableB",
"columns": [
{
"name": "TableBId",
"dataType": "int64",
"sourceColumn": "TableBId"
},
{
"name": "TableB",
"dataType": "string",
"sourceColumn": "TableB"
}
],
"partitions": [
{
"name": "Partition",
"dataView": "full",
"source": {
"type": "m",
"expression": [
"let",
" Source=GetFileList(),",
" #\"TableB txt\" = Source{[Name=\"TableB.txt\"]}[Content],",
" #\"Imported CSV\" = Csv.Document(#\"TableB txt\",[Delimiter=\",\", Columns=11, Encoding=1252, QuoteStyle=QuoteStyle.None]),",
" #\"Promoted Headers\" = Table.PromoteHeaders(#\"Imported CSV\", [PromoteAllScalars=true]),",
" #\"Changed Type\" = Table.TransformColumnTypes(#\"Promoted Headers\",{{\"TableBId\", Int64.Type}, {\"TableB\", type text}, {\"Description\", type text}}),",
" #\"Removed Columns\" = Table.RemoveColumns(#\"Changed Type\",{\"Description\"})",
"in",
" #\"Removed Columns\""
]
}
}
]
}
]
}
}
Use the sequence command:
{
"sequence": {
"operations": [
{ CREATE TABLE 1},
{ CREATE TABLE 2}
]
}
}
You can include as many operations as you want.
My output :
{
"EmployeePositions": {
"EmployeeID": "05383803",
**"EmployeeName": null,**
"Positions": [
{
"PositionID": null,
"EffectiveDate": "2017-06-17",
"BusinessUnit": null,
"Customer": {
**"JobReqNumber": null,**
"CustomerID": "243720",
"CustomerName": "TEKGS UK#BANK OF AMERICA",
"CustomerOrderNumber": null
},
"WorkSiteLocaiton": {
"Address1": "BLUE SQUARES HOUSE",
"Address2": "PRIORS WAY",
"Address3": " ",
"Address4": " ",
"Number1": " ",
"Number2": " ",
"City": "BERKS",
"State": "BRACKNELL",
"Postal": "RG42 ING",
"Country": "GBR"
}
},
{
"PositionID": null,
"EffectiveDate": "2017-06-15",
"EffectiveStatus": "I",
"BusinessUnit": null,
"Customer": {
"JobReqNumber": null,
"CustomerID": "243721",
"CustomerName": "TEKGS UK#UK Bank",
"CustomerOrderNumber": null
},
"WorkSiteLocaiton": {
"Address1": "BLUE SQUARES",
"Address2": "PRIORS WAY",
"Address3": " ",
"Address4": " ",
"Number1": " ",
"Number2": " ",
"City": "BERKSS",
"State": "BRACKNEL",
"Postal": "RG42 ING",
"Country": "GBR"
}
}
]
}
}
expected output:
{
"EmployeePositions": {
"EmployeeID": "05383803",
"EmployeeName": "John Smith",
"Positions": [
{
"PositionID": "",
"EffectiveDate": "2017-06-17",
"BusinessUnit": "",
"Customer": {
"JobReqNumber": "",
"CustomerID": "243720",
"CustomerName": "TEKGS UK#BANK OF AMERICA",
"CustomerOrderNumber": ""
},
"WorkSiteLocaiton": {
"Address1": "BLUE SQUARES HOUSE",
"Address2": "PRIORS WAY",
"Address3": " ",
"Address4": " ",
"Number1": " ",
"Number2": " ",
"City": "BERKS",
"State": "BRACKNELL",
"Postal": "RG42 ING",
"Country": "GBR"
}
},
{
"PositionID": "",
"EffectiveDate": "2017-06-15",
"EffectiveStatus": "I",
"BusinessUnit": "",
"Customer": {
"JobReqNumber": "",
"CustomerID": "243721",
"CustomerName": "TEKGS UK#UK Bank",
"CustomerOrderNumber": ""
},
"WorkSiteLocaiton": {
"Address1": "BLUE SQUARES",
"Address2": "PRIORS WAY",
"Address3": " ",
"Address4": " ",
"Number1": " ",
"Number2": " ",
"City": "BERKSS",
"State": "BRACKNEL",
"Postal": "RG42 ING",
"Country": "GBR"
}
}
]
}
}
I want exactly the same same output instead of null i want ("") as an output
i tried (%output application/json skipNullOn="everywhere") and also (when payload != null otherwise "") but i have to write this after every field. Do we have any better solution.
Any leads will be appreciated
This will work for you:
%dw 1.0
%output application/json
%function applyToValues(e, fn)
e match {
:object -> $ mapObject {($$): applyToValues($, fn)},
:array -> $ map applyToValues($, fn),
default -> fn($)
}
%function replaceNull(e, replacement="")
applyToValues(e, ((v) -> replacement when (v == null) otherwise v))
---
replaceNull(payload)
I have multiple json scripts with a similar format as:
{
"name": "xyz",
"slug": "xyz",
"supplier": "xyz.limited",
"attributes": [
{
"name": "mass",
"productConfiguration": "base",
"description": "",
"value": "0.24",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "kg",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "length",
"productConfiguration": "base",
"description": "assumed to be length",
"value": "58.0",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "mm",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "height",
"productConfiguration": "base",
"description": "assumed to be height",
"value": "25.0",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "mm",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "width",
"productConfiguration": "base",
"description": "assumed to be width",
"value": "58.0",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "mm",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "momentum-storage",
"productConfiguration": "base",
"description": "",
"value": "0.050",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "N m s",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "maximum-torque",
"productConfiguration": "base",
"description": "",
"value": "0.007",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "N m",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "voltage",
"productConfiguration": "base",
"description": "voltage is given in DC current",
"value": "12.0",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "V",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "maximum-power",
"productConfiguration": "base",
"description": "",
"value": "9.0",
"minimumValue": "",
"maximumValue": "",
"measurementUnit": "W",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
},
{
"name": "lifetime",
"productConfiguration": "base",
"description": "",
"value": "",
"minimumValue": "5.0",
"maximumValue": "",
"measurementUnit": "yr",
"uuid": "",
"attributeClassUuid": "",
"productUuid": ""
}
]
}
Each of these files have different set of attributes defined. I would like to plot the number of times each attribute is repeated across all files. I try to do this using python3.5 functionalities 'os' and 'panda.DataFrame'. However, got lost somewhere! Can use some help with this. Thanks in advance!