I have an interesting issue (well from my perspective) :)
With the JSON file in where I am using SQL Server 2019 to populate a db.
The file has many sub-sections (categories) with required data from the root. I need to import the subsections, though I am trying to figure out:
Do I create multiple rows in the same table or create sub tables?
How do I change/add to my script to do either one based on #1
Sample data:
"name": "TAPMN14GLBSM125",
"comment": null,
"displayCategoryProgrammaticName": "NETWORK_SWITCH",
"displayCategoryProgrammaticNameDisplay": "Network Switch",
"categories": [
{
"programmaticName": "ASSET",
"programmaticNameDisplay": "Asset",
"id": "00000000-0000-0000-0000-400000000102"
},
{
"programmaticName": "DEVICE",
"programmaticNameDisplay": "Device",
"id": "00000000-0000-0000-0000-400000000129"
},
{
"programmaticName": "IT_EQUIPMENT",
"programmaticNameDisplay": "IT Equipment",
"id": "00000000-0000-0000-0000-400000000145"
},
{
"programmaticName": "NETWORK",
"programmaticNameDisplay": "Network",
"id": "00000000-0000-0000-0000-400000000154"
},
{
"programmaticName": "NETWORK_SWITCH",
"programmaticNameDisplay": "Network Switch",
"id": "00000000-0000-0000-0000-400000000157"
},
{
"programmaticName": "PHYSICAL_OBJECT",
"programmaticNameDisplay": "Physical Object",
"id": "00000000-0000-0000-0000-400000000201"
}
],
"owner": null,
"ownerEmail": null,
"recordCreateDate": "2017-06-15T15:01:47.086+0000",
"location": "_My Operations Center->BS->ER->A22-07 (21.6)",
"capacities": [
{
"programmaticName": "COOLING-RATED",
"programmaticNameDisplay": "Heat Rated",
"capacityValue": null
},
{
"programmaticName": "RACKSPACE-RATED",
"programmaticNameDisplay": "Rack Space Rated",
"capacityValue": 0.3371
},
{
"programmaticName": "WEIGHT-RATED",
"programmaticNameDisplay": "Weight Rated",
"capacityValue": 1.0
},
{
"programmaticName": "FLOORSPACE-RATED",
"programmaticNameDisplay": "Floor Space Rated",
"capacityValue": 0.1805
},
{
"programmaticName": "COOLING-DESIGNED",
"programmaticNameDisplay": "Heat Designed",
"capacityValue": null
},
{
"programmaticName": "WEIGHT-DESIGNED",
"programmaticNameDisplay": "Weight Designed",
"capacityValue": 1.0
},
{
"programmaticName": "FLOORSPACE-DESIGNED",
"programmaticNameDisplay": "Floor Space Designed",
"capacityValue": 0.1805
}
],
"userDefinedProperties": [],
"alarmCounts": {
"criticalAlarmCount": 0,
"warningAlarmCount": 0,
"normalAlarmCount": 0,
"totalAlarmCount": 0,
"activeAlarmCount": null,
"clearedAlarmCount": null,
"supressedAlarmCount": null,
"aggregationTime": null,
"sourceType": null
},
Code snippet:
FROM OPENROWSET (BULK '''+ #File+''', SINGLE_CLOB) as j
CROSS APPLY OPENJSON(BulkColumn, '''+#JSONRoot+''')
WITH
(
[name] VARCHAR(50),
[comment] VARCHAR(50),
[displayCategoryProgrammaticName] VARCHAR(50),
[displayCategoryProgrammaticNameDisplay] VARCHAR(50),
[owner] VARCHAR(100),
[ownerEmail] VARCHAR(50),
[recordCreateDate] VARCHAR(50),
[location] VARCHAR(50),
[userDefinedProperties] VARCHAR(100),
[ownerTenantId] UNIQUEIDENTIFIER,
[tenantIds] UNIQUEIDENTIFIER,
[sharedTenancyOption] VARCHAR(50),
[updatesInProgress] bit,
[symbolId] UNIQUEIDENTIFIER,
[symbolVersion] VARCHAR(50),
[typeIdentifierTag] VARCHAR(100),
[manufacturer] VARCHAR(50),
[model] VARCHAR(50),
[modelQualifier] VARCHAR(50),
[depth] decimal(18,2),
[height] decimal(18,2),
[weight] decimal(18,2),
[width] decimal(18,2),
[rfid] VARCHAR(50),
[rfidPosition] Int,
[rackUPosition] NVARCHAR(10),
[lockedForGraphicalMoveInThePhysicalAspect] bit,
[isPlaced] bit,
[description] VARCHAR(50),
[maintenanceMode] VARCHAR(50),
[productLine] VARCHAR(50),
[installDate] VARCHAR(50),
[licenseTierTag] VARCHAR(50),
[licenseTierTagDisplayName] VARCHAR(50),
[url] VARCHAR(100),
[deviceState] VARCHAR(50),
[monitoringState] VARCHAR(100),
[alarmStatus] VARCHAR(50),
[communicationStatus] VARCHAR(50),
[openings] VARCHAR(50),
[slots] VARCHAR(50),
[zoneId] VARCHAR(50),
[monitoringConfiguration] VARCHAR(50),
[permissions] VARCHAR(50),
[alarmAssignmentRules] VARCHAR(100),
[assetNumber] VARCHAR(50),
[barcodeNumber] VARCHAR(50),
[serialNumber] VARCHAR(50),
[id] UNIQUEIDENTIFIER
)'
Related
This is some part of my input json :-
"orderLine": {
"agreementHeaderId": [
null
],
"agreementNumber": [
null
],
"customerProductId": null,
"displayLineNumber": "2",
"estimateFulfillmentCost": 9.97,
"extendedAmount": null,
"fulfilledQuantity": null,
"fulfillToleranceAbove": null,
"inventoryOrganizationCode": "107",
"inventoryOrganizationId": "107",
"jobName": null,
"orderedQuantity": 1.0,
"orderedUom": null,
"origSystemDocumentLineReference": "2",
"paymentTerms": "CASH ON DEMAND",
"paymentTermsCode": "COD",
"productId": "4065412*107",
"productNumber": "4065412",
"productDescription": "20 DFU AAV W/ 1-1/2-2 PVC ADPT",
"purchasingUom": null,
"quantityPerModel": 0.0,
"requestedShipDate": "2022-12-19T05:00:00.000Z",
"requestedSupplierNumber": null,
"requisitionInventoryOrganizationId": "107",
"rmaDeliveredQuantity": null,
"scheduleShipDate": null,
"shippedQuantity": 1.0,
"shippingCarrier": "OT",
"shippingInstructions": null,
"shippingMode": "OUR TRUCK",
"shippingModeCode": "OT",
"subInventory": null,
"transactionLineTypeCode": null,
"transactionCategoryCode": null,
"transactionLineType": "MP",
"unitListPrice": 33.23,
"unitQuantity": 1.0,
"unitSellingPrice": 44.0,
Few values are double type not string
So, while giving this JSONObject oldQty = (JSONObject) orderLine.get("orderedQuantity")
I'm getting Cannot cast object '0.0' with class 'java.lang.Double' to class 'org.json.JSONObject' this error
I need to extract values from a column which has a JSON format.
Here is an example of one of rows in table for this column, here is two operatorKeys but in other rows there might be more:
{
"officeId": "office8685",
"TypeOfOffice": null,
"Telefonnummer": "",
"Operatorer": [
{
"OperatorKey": "1",
"OperatorUserID": {
"Agency": "Other",
"AgencySpecified": true,
"Value": "TM-Oper-1"
},
"ContactInformation": {
"FirstName": "mike",
"LastName": null,
"Address": {
"Street": null,
"City": null,
"Province": null,
"Country": null
},
"Phone": null,
"Fax": null,
"Email": null,
}
},
{
"OperatorKey": "4",
"OperatorUserID": {
"Agency": "Other",
"AgencySpecified": true,
"Value": "TM-Oper-4"
},
"ContactInformation": {
"FirstName": "xxx",
"LastName": null,
"Address": {
"Street": null,
"City": null,
"Province": null,
"Country": null
},
"Phone": null,
"Fax": null,
"Email": null,
}
}
]
}
And here is SQL view I'm trying to read JSON, I'm using openjson and cross apply to read values:
Alter view dbo.vOffice as
select
column1,column2
--,column3 --this column contains values with json format
,OfficId,TypeOfOffice,Telefonnummer,Operatorer,OperatorKey,OperatorUserID
,ContactInformation,Agency,AgencySpecified,[Value],FirstName,LastName
,[Address],Phone, Fax, Email
FROM [produktion_dim].[VirkesverdeStandardSCD1]
CROSS APPLY OPENJSON (column3,'$.OfficId') with (
OfficId varchar(4096) '$.OfficId',TypeOfOffice varchar(4096) '$.TypeOfOffice',
Telefonnummer varchar(4096) '$.Telefonnummer',Operatorer varchar(4096) '$.Operatorer')
cross apply openjson(Operatorer, '$') with(OperatorKey varchar(4096) '$.OperatorKey', OperatorUserID varchar(4096) '$.OperatorUserID', ContactInformation varchar(4096) '$.ContactInformation')
cross apply openjson(OperatorUserID,'$') with ( Agency varchar(4096) '$.Agency',
AgencySpecified varchar(4096) '$.AgencySpecified',[Value] varchar(4096) '$.Value' )
cross apply openjson( ContactInformation, '$') with (FirstName varchar(4096) '$.FirstName',LastName varchar(4096) '$.LastName',[Address] varchar(4096) '$.Address',Phone varchar(4096) '$.Phone',Fax varchar(4096) '$.Fax',Email varchar(4096) '$.Email')
cross apply openjson([Address],'$') with ( Street varchar(4096) '$.Street',
City varchar(4096) '$.City',Province varchar(4096) '$.Province',Country varchar(4096) '$.Country')
But I'm getting error. Can you please tell me what is missing or what should be changed in SQL view?
Once you can organize the sender to send correctly formatted JSON data here are some tips to help you parse the JSON data:
The '$' in openjson(something, '$') with (...) is redundant, just use openjson(something) with (...)
JSON path queries are case sensitive: $.officeId is different than $. OfficeId and neither would match '$.OfficId'.
inside with (...) when declaring a column to be consumed by another openjson() call it needs to use the type nvarchar(max) and include as json in its definition, e.g.: OperatorUserID nvarchar(max) '$.OperatorUserID' as json.
I have edited your query to extract the JSON data I think you're looking for...
create table dbo.VirkesverdeStandardSCD1 (
column3 nvarchar(max)
);
insert dbo.VirkesverdeStandardSCD1 (column3) values (N'{
"officeId": "office8685",
"TypeOfOffice": null,
"Telefonnummer": "",
"Operatorer": [
{
"OperatorKey": "1",
"OperatorUserID": {
"Agency": "Other",
"AgencySpecified": true,
"Value": "TM-Oper-1"
},
"ContactInformation": {
"FirstName": "mike",
"LastName": null,
"Address": {
"Street": null,
"City": null,
"Province": null,
"Country": null
},
"Phone": null,
"Fax": null,
"Email": null
}
},
{
"OperatorKey": "4",
"OperatorUserID": {
"Agency": "Other",
"AgencySpecified": true,
"Value": "TM-Oper-4"
},
"ContactInformation": {
"FirstName": "xxx",
"LastName": null,
"Address": {
"Street": null,
"City": null,
"Province": null,
"Country": null
},
"Phone": null,
"Fax": null,
"Email": null
}
}
]
}');
select
--column1,column2
--,column3 --this column contains values with json format
OfficId,TypeOfOffice,Telefonnummer,Operatorer,OperatorKey,OperatorUserID
,ContactInformation,Agency,AgencySpecified,[Value],FirstName,LastName
,[Address],Phone, Fax, Email
from dbo.VirkesverdeStandardSCD1
cross apply openjson(column3) with (
OfficId varchar(4096) '$.officeId',
TypeOfOffice varchar(4096) '$.TypeOfOffice',
Telefonnummer varchar(4096) '$.Telefonnummer',
Operatorer nvarchar(max) '$.Operatorer' as json
)
cross apply openjson(Operatorer) with (
OperatorKey varchar(4096) '$.OperatorKey',
OperatorUserID nvarchar(max) '$.OperatorUserID' as json,
ContactInformation nvarchar(max) '$.ContactInformation' as json
)
cross apply openjson(OperatorUserID) with (
Agency varchar(4096) '$.Agency',
AgencySpecified varchar(4096) '$.AgencySpecified',
[Value] varchar(4096) '$.Value'
)
cross apply openjson(ContactInformation) with (
FirstName varchar(4096) '$.FirstName',
LastName varchar(4096) '$.LastName',
[Address] nvarchar(max) '$.Address' as json,
Phone varchar(4096) '$.Phone',
Fax varchar(4096) '$.Fax',
Email varchar(4096) '$.Email'
)
cross apply openjson([Address]) with (
Street varchar(4096) '$.Street',
City varchar(4096) '$.City',
Province varchar(4096) '$.Province',
Country varchar(4096) '$.Country'
);
I'm trying to retrieve a given number of values from the following json object (just used 2 for brevity):
[
{
"content": null,
"deleted": false,
"metadata": null,
"name": "name/windows-2016-osDisk.170b8936-d66d-4139-a409-26f4e9d354fe.vhd",
"properties": {
"appendBlobCommittedBlockCount": null,
"blobTier": "P10",
"blobTierChangeTime": null,
"blobTierInferred": true,
"blobType": "PageBlob",
"contentLength": 136367309312,
"contentRange": null,
"contentSettings": {
"cacheControl": null,
"contentDisposition": null,
"contentEncoding": null,
"contentLanguage": null,
"contentMd5": "-0q970378r08==",
"contentType": "application/octet-stream"
},
"copy": {
"completionTime": null,
"id": null,
"progress": null,
"source": null,
"status": null,
"statusDescription": null
},
"creationTime": "2019-11-21T08:34:33+00:00",
"deletedTime": null,
"etag": "0237502375",
"lastModified": "2019-11-21T08:34:33+00:00",
"lease": {
"duration": null,
"state": "available",
"status": "unlocked"
},
"pageBlobSequenceNumber": null,
"remainingRetentionDays": null,
"sequenceNumber": 1,
"serverEncrypted": true
},
"snapshot": null
},
{
"content": null,
"deleted": false,
"metadata": null,
"name": "name/windows-2019-osDisk.f122fb3c-0edb-42a8-b98c-56657b447f15.vhd",
"properties": {
"appendBlobCommittedBlockCount": null,
"blobTier": "P10",
"blobTierChangeTime": null,
"blobTierInferred": true,
"blobType": "PageBlob",
"contentLength": 136367309312,
"contentRange": null,
"contentSettings": {
"cacheControl": null,
"contentDisposition": null,
"contentEncoding": null,
"contentLanguage": null,
"contentMd5": "0237502375/hKOg==",
"contentType": "application/octet-stream"
},
"copy": {
"completionTime": null,
"id": null,
"progress": null,
"source": null,
"status": null,
"statusDescription": null
},
"creationTime": "2019-11-21T08:35:03+00:00",
"deletedTime": null,
"etag": "20397520i3h523",
"lastModified": "2019-11-21T08:35:03+00:00",
"lease": {
"duration": null,
"state": "available",
"status": "unlocked"
},
"pageBlobSequenceNumber": null,
"remainingRetentionDays": null,
"sequenceNumber": 1,
"serverEncrypted": true
},
"snapshot": null
}
]
I sort this on creationTime in jq like so: jq 'sort_by(.properties.creationTime)' When I pipe this through | .[].name making jq 'sort_by(.properties.creationTime) | .[].name I get a sorted list of the names.
My question is: How can I pass an integer to this command and remove these many names, starting from the most recent created (bottom)?
following
https://stedolan.github.io/jq/manual/v1.6/#Builtinoperatorsandfunctions
slicing should work:
jq 'sort_by(.properties.creationTime) | .[-1:]'
The answer I've used is the following:
jq 'sort_by(.properties.creationTime) | .[-4:] | .[].name'
Where -4 is the amount of names to display starting from the bottom.
I have this items in a JSON file that I need to compare wherever I have Curr or Prev values and compare if they are equals. this algorithm is with angular 5 but I'm being unable to do this, I did the load function with httpClient
Example :
[{
"id": 1,
"comparisonDate": "2017-06-22",
"milestone": "CURR_PREV",
"udDomain": "BASIS",
"udIdentifier": "332A251000.01",
"rpt": "D70",
"approvedCurr": true,
"approvedPrev": null,
"commercialReferenceCurr": "BASIS - 01.01.2007",
"commercialReferencePrev": null,
"customizationImpactCurr": null,
"customizationImpactPrev": null,
"deliveryTypeCurr": null,
"deliveryTypePrev": null,
"aircraftId": 184,
"aircraftSapId": "580652",
"aircraftName": "BUL 011",
"aircraftSerialNumber": "02713",
"quantityCurr": "1",
"quantityPrev": null,
"statusCurr": "O",
"statusPrev": null,
"typeModification": "CREATION",
"validityCurr": "V",
"validityPrev": null
}, {
"id": 2,
"comparisonDate": "2017-06-22",
"milestone": "CURR_PREV",
"udDomain": "BASIS",
"udIdentifier": "332A270028.01",
"rpt": "E15",
"approvedCurr": true,
"approvedPrev": null,
"commercialReferenceCurr": "BASIS - 01.01.2007",
"commercialReferencePrev": null,
"customizationImpactCurr": null,
"customizationImpactPrev": null,
"deliveryTypeCurr": null,
"deliveryTypePrev": null,
"aircraftId": 184,
"aircraftSapId": "580652",
"aircraftName": "BUL 011",
"aircraftSerialNumber": "02713",
"quantityCurr": "1",
"quantityPrev": null,
"statusCurr": "O",
"statusPrev": null,
"typeModification": "CREATION",
"validityCurr": "V",
"validityPrev": null
}]
this should be with using lodash.
i assume your want something like this
for each object in your array
filter keys that end with "Curr"
for each get Prev property
compare it with curr property
let obj = [{
"id": 1,
"comparisonDate": "2017-06-22",
"milestone": "CURR_PREV",
"udDomain": "BASIS",
"udIdentifier": "332A251000.01",
"rpt": "D70",
"approvedCurr": true,
"approvedPrev": null,
"commercialReferenceCurr": "BASIS - 01.01.2007",
"commercialReferencePrev": null,
"customizationImpactCurr": null,
"customizationImpactPrev": null,
"deliveryTypeCurr": null,
"deliveryTypePrev": null,
"aircraftId": 184,
"aircraftSapId": "580652",
"aircraftName": "BUL 011",
"aircraftSerialNumber": "02713",
"quantityCurr": "1",
"quantityPrev": null,
"statusCurr": "O",
"statusPrev": null,
"typeModification": "CREATION",
"validityCurr": "V",
"validityPrev": null
}, {
"id": 2,
"comparisonDate": "2017-06-22",
"milestone": "CURR_PREV",
"udDomain": "BASIS",
"udIdentifier": "332A270028.01",
"rpt": "E15",
"approvedCurr": true,
"approvedPrev": null,
"commercialReferenceCurr": "BASIS - 01.01.2007",
"commercialReferencePrev": null,
"customizationImpactCurr": null,
"customizationImpactPrev": null,
"deliveryTypeCurr": null,
"deliveryTypePrev": null,
"aircraftId": 184,
"aircraftSapId": "580652",
"aircraftName": "BUL 011",
"aircraftSerialNumber": "02713",
"quantityCurr": "1",
"quantityPrev": null,
"statusCurr": "O",
"statusPrev": null,
"typeModification": "CREATION",
"validityCurr": "V",
"validityPrev": null
}];
obj.forEach(x=>{
Object.keys(x)
.filter(y=>y.endsWith("Curr"))
.forEach(y=>{
let prevProp = y.replace("Curr", "Prev")
console.log(y, x[y]);
console.log(prevProp ,x[prevProp ]);
if(x[y]===x[prevProp ]){
//equal
}else{
//not equal
}
})
})
you could use lodash's _.forEach, _.Keys, _.filter but i think there is not much point on this.
So I am trying to convert a pandas Dataframe with 12+ columns into corresponding JSON Record. I'm able to get it. However, I want few columns of the frame to be a subrecord of a new column. How this can be achieved.?
{
"ADRNR": 2692629,
"AlertID": "",
"AlertTimestamp": "14-12-2015 14:44:14",
"BANKL": null,
"BANKN": null,
"BANKS": "nan",
"BEGRU": "NPIV",
"BUKRS": "2646",
"C_Block": "No",
"KOINH": null,
"LAND1": "US",
"LOEVM_x": null,
"LOEVM_y": null,
"MasterDataID": "10099",
"MasterDataType": "Vendor",
"NAME1": "LEGAL",
"NODEL_x": null,
"NODEL_y": null,
"ORT01": null,
"OtherData": null,
"PSTLZ": null,
"RuleID": "Rule3",
"RuleName": "Vendor and Bank Country is Different",
"STCD1": null,
"STCD2": null,
"STCEG": null,
"STRAS": null,
"TELF1": null
}
The above JSON is what I Get. But I want the following structure. Kindly guide me.
{
"RuleID": "Rule3",
"RuleName": "Vendor and Bank Country is Different",
"AlertID": "",
"AlertTimestamp": "14-12-2015 14:44:14",
"MasterDataID": "10099",
"MasterDataType": "Vendor",
"OtherData": {
"BANKL": null,
"BANKN": null,
"BANKS": "nan",
"BEGRU": "NPIV",
"BUKRS": "2646",
"C_Block": "No",
"KOINH": null,
"LAND1": "US",
"LOEVM_x": null,
"LOEVM_y": null,
"NAME1": "LEGAL",
"NODEL_x": null,
"NODEL_y": null,
"ORT01": null,
"PSTLZ": null,
"ADRNR": 2692629,
"STCD1": null,
"STCD2": null,
"STCEG": null,
"STRAS": null,
"TELF1": null
}
}
Edit : Flg is my code
Final_Table['AlertID'] = ''
Final_Table['AlertTimestamp'] = datetime.now().strftime('%d-%m-%Y %H:%M:%S')
Final_Table['MasterDataType'] = 'Vendor'
Final_Table['RuleID'] = 'Rule3'
Final_Table['RuleName'] = 'Vendor and Bank Country is Different'
Final_Table = Final_Table.rename(columns={'LIFNR': 'MasterDataID'})
Result = Final_Table[Final_Table['BANKS'] != Final_Table['LAND1']]
Result['OtherData'] = np.NaN
final_result = {'alerts': json.loads(Result.to_json(orient = 'records',force_ascii = False).encode('utf8'))}
result = {'results': final_result}
with open('output_Rule3.json', 'w') as outfile:
json.dump(result, outfile, indent = 5, sort_keys = True)
log.info("Rule3 : Execution Successful")
Rather than within pandas, I would modify the json output.
Starting with your json data
print(json.dumps(data, indent=4, sort_keys=True))
[
{
"ADRNR": 2692629,
"AlertID": "",
"AlertTimestamp": "14-12-2015 14:44:14",
"BANKL": null,
"BANKN": null,
"BANKS": "nan",
"BEGRU": "NPIV",
"BUKRS": "2646",
"C_Block": "No",
"KOINH": null,
"LAND1": "US",
"LOEVM_x": null,
"LOEVM_y": null,
"MasterDataID": "10099",
"MasterDataType": "Vendor",
"NAME1": "LEGAL",
"NODEL_x": null,
"NODEL_y": null,
"ORT01": null,
"OtherData": null,
"PSTLZ": null,
"RuleID": "Rule3",
"RuleName": "Vendor and Bank Country is Different",
"STCD1": null,
"STCD2": null,
"STCEG": null,
"STRAS": null,
"TELF1": null
}
]
and a list of some of the columns you are interested in:
other_data = ["BANKL", "BANKN", "BANKS", "BEGRU", "BUKRS", "C_Block"]
you can modify the json directly:
for i, record in enumerate(data):
data[i]['OtherData'] = {}
for key in list(record.keys()):
if key in other_data:
data[i]['OtherData'][key] = record[key]
del data[i][key]
print(json.dumps(data, indent=4, sort_keys=True))
[
{
"ADRNR": 2692629,
"AlertID": "",
"AlertTimestamp": "14-12-2015 14:44:14",
"KOINH": null,
"LAND1": "US",
"LOEVM_x": null,
"LOEVM_y": null,
"MasterDataID": "10099",
"MasterDataType": "Vendor",
"NAME1": "LEGAL",
"NODEL_x": null,
"NODEL_y": null,
"ORT01": null,
"OtherData": {
"BANKL": null,
"BANKN": null,
"BANKS": "nan",
"BEGRU": "NPIV",
"BUKRS": "2646",
"C_Block": "No"
},
"PSTLZ": null,
"RuleID": "Rule3",
"RuleName": "Vendor and Bank Country is Different",
"STCD1": null,
"STCD2": null,
"STCEG": null,
"STRAS": null,
"TELF1": null
}
]