Read data from hierarchy document in SQL Server - json

I use SQL Server 2016 and I develop SSIS packages. I call a web service and I have below file.
{
"id": "Voyage",
"fields": {
"CommenceDateGmt": "2018-10-09T04:00:00",
"CommenceDateLocal": "2018-10-09T15:00:00",
"CompleteDateGmt": "2018-11-02T10:30:00",
"CompleteDateLocal": "2018-11-02T20:30:00",
"VoyageStatus": "Completed"
},
"dataSources": [
{
"id": "VoyageItineraries",
"joinType": "toMany",
"values": [
{
"fields": {
"VesselCode": "aaaa",
"VoyageNo": 10000,
"Seq": 258,
"PortFunc": "C",
"PortName": "aaaa (A)"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "aaaa"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": []
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "aaasss"
}
}
]
},
{
"fields": {
"VesselCode": "www",
"VoyageNo": 5454,
"Seq": 5454,
"PortFunc": "L54",
"PortName": "54545"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "fsfsdsd&S"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 54545.0,
"CargoShortName": "dfdfdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "dfewff"
}
}
]
},
{
"fields": {
"VesselCode": "sdfsdf",
"VoyageNo": 23423,
"Seq": 234,
"PortFunc": "Dd",
"PortName": "fewffe"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "erwerwer"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 23423.0,
"CargoShortName": "sdfsdf"
}
},
{
"fields": {
"NominatedLiftQty": 23423.0,
"CargoShortName": "sdefsdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "dfsdf"
}
}
]
},
{
"fields": {
"VesselCode": "dfsdf",
"VoyageNo": 234,
"Seq": 32433,
"PortFunc": "L",
"PortName": "sdfsdf"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "sdfsdf"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 234234.0,
"CargoShortName": "sdfsdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "sdfsdf"
}
}
]
}
]
},
{
"id": "Cargos",
"joinType": "toMany",
"values": [
{
"fields": {
"CoaNo": "sdfsdf"
},
"dataSources": [
{
"id": "Counterparty",
"joinType": "toOne",
"fields": {
"FullName": "sdfsdfsd"
}
}
]
}
]
}
]
}
]
As you can see, I have several table.
I need to transfer this hierarchy data to a table.
Table1Filed1, Table1Filed2, Table1Filed3, Table1Filed4, Table2Field1, Table2Field2, Table2Field3
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data

Related

How can I create a hierarchical json response in FLASK

I have a single table in database like database table. I want to search a child from database and return a hierarchical JSON to a front end in order to create a tree. How can I do that in FLASK.
My expected JSON for mat should be like expected JSON
Since you have tagged your question with flask, this post assumes you are using Python as well. To format your database values in JSON string, you can query the db and then use recursion:
import sqlite3, collections
d = list(sqlite3.connect('file.db').cursor().execute("select * from values"))
def get_tree(vals):
_d = collections.defaultdict(list)
for a, *b in vals:
_d[a].append(b)
return [{'name':a, **({} if not (c:=list(filter(None, b))) else {'children':get_tree(b)})} for a, b in _d.items()]
import json
print(json.dumps(get_tree(d), indent=4))
Output:
[
{
"name": "AA",
"children": [
{
"name": "BB",
"children": [
{
"name": "EE",
"children": [
{
"name": "JJ",
"children": [
{
"name": "EEV"
},
{
"name": "FFW"
}
]
},
{
"name": "KK",
"children": [
{
"name": "HHX"
}
]
}
]
}
]
},
{
"name": "CC",
"children": [
{
"name": "FF",
"children": [
{
"name": "LL",
"children": [
{
"name": "QQY"
}
]
},
{
"name": "MM",
"children": [
{
"name": "RRV"
}
]
}
]
},
{
"name": "GG",
"children": [
{
"name": "NN",
"children": [
{
"name": "SSW"
}
]
}
]
}
]
},
{
"name": "DD",
"children": [
{
"name": "HH",
"children": [
{
"name": "OO",
"children": [
{
"name": "TTZ"
}
]
}
]
},
{
"name": "II",
"children": [
{
"name": "PP",
"children": [
{
"name": "UUW"
}
]
}
]
}
]
}
]
}
]

How do I pass secret value from keyvault using Azure ARM Template

I am trying to create Azure Application Gateway with ssl certificate from keyvaults. But didn't find any option to add keyvaults to ARM template with .pfx and .cer files. So I have encoded the certificate contents and added as secret in existing keyvault. Now trying to pass the secrets using ARM template. Validation passed but getting error in Deployment stage. Attached the template and parameters I am using.
Getting Error while deploying the resource
Deployment template validation failed: 'Template parameter JToken type is not valid. Expected 'String, Uri'. Actual 'Object'
"additionalInfo": [
{
"type": "TemplateViolation",
"info": {
"lineNumber": 226,
"linePosition": 33,
"path": "properties.template.parameters.appgwfesslcertsecret"
}
}
]
Updated Template:
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"namingSettings": {
"type": "object"
},
"taggingSettings": {
"type": "object"
},
"applicationGatewaySettings": {
"type": "object"
},
"appgwfesslcertsecret": {
"type": "securestring"
},
"appgwbecertsecret": {
"type": "securestring"
}
},
"variables": {
"namePrefix": "[concat(parameters('namingSettings').name.org,'-',parameters('namingSettings').name.cloud,'-',parameters('namingSettings').name.region,'-',parameters('namingSettings').name.businessUnit,'-',parameters('namingSettings').name.account,'-',parameters('namingSettings').name.app,'-',parameters('namingSettings').name.sdlc,'-')]" },
"resources": [
{
"apiVersion": "2018-11-01",
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name)]",
"type": "Microsoft.Network/applicationGateways",
"location": "[resourceGroup().location]",
"copy": {
"name": "appgwCopy",
"count": "[length(parameters('applicationGatewaySettings').settings)]"
},
"tags": "[parameters('taggingSettings').tags]",
"properties": {
"sku": {
"name": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewaySku]",
"tier": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewayTier]",
"capacity": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewayInstanceCount]"
},
"sslPolicy": {
"policyType": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].policyType]",
"policyName": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].policy]"
},
"copy": [
{
"name": "frontendPorts",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendPorts)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendPorts[copyIndex('frontendPorts')].name)]",
"properties": {
"port": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendPorts[copyIndex('frontendPorts')].properties.port]"
}
}
},
{
"name": "gatewayIPConfigurations",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations[copyIndex('gatewayIPConfigurations')].name)]",
"properties": {
"subnet": {
"id": "[resourceId(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations[copyIndex('gatewayIPConfigurations')].properties.subnet.vnetRGName,'microsoft.network/virtualnetworks/subnets', parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations[copyIndex('gatewayIPConfigurations')].properties.subnet.vnetName, parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations[copyIndex('gatewayIPConfigurations')].properties.subnet.subnetName)]"
}
}
}
},
{
"name": "frontendIPConfigurations",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendIPConfigurations)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendIPConfigurations[copyIndex('frontendIPConfigurations')].name)]",
"properties": {
"subnet": {
"id": "[resourceId(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].gatewayIPConfigurations[copyIndex('frontendIPConfigurations')].properties.subnet.vnetRGName,'microsoft.network/virtualnetworks/subnets', parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendIPConfigurations[copyIndex('frontendIPConfigurations')].properties.subnet.vnetName, parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].frontendIPConfigurations[copyIndex('frontendIPConfigurations')].properties.subnet.subnetName)]"
}
}
}
},
{
"name": "backendHttpSettingsCollection",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendHttpSettingsCollection)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendHttpSettingsCollection[copyIndex('backendHttpSettingsCollection')].name)]",
"properties": {
"port": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendHttpSettingsCollection[copyIndex('backendHttpSettingsCollection')].properties.port]",
"protocol": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendHttpSettingsCollection[copyIndex('backendHttpSettingsCollection')].properties.protocol]",
"authenticationCertificates": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendHttpSettingsCollection[copyIndex('backendHttpSettingsCollection')].properties.authenticationCertificates]"
}
}
},
{
"name": "backendAddressPools",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendAddressPools)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].backendAddressPools[copyIndex('backendAddressPools')].name)]"
}
},
{
"name": "httpListeners",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners[copyIndex('httpListeners')].name)]",
"properties": {
"frontendIPConfiguration": {
"id": "[resourceId(resourceGroup().name, 'microsoft.network/applicationGateways/frontendIPConfigurations', concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name),concat(variables('namePrefix'), parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners[copyIndex('httpListeners')].properties.frontendIPConfiguration))]"
},
"frontendPort": {
"id": "[resourceId(resourceGroup().name, 'microsoft.network/applicationGateways/frontendPorts', concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name),concat(variables('namePrefix'), parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners[copyIndex('httpListeners')].properties.frontendPort))]"
},
"protocol": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners[copyIndex('httpListeners')].properties.protocol]",
"sslCertificate": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].httpListeners[copyIndex('httpListeners')].properties.sslCertificate]"
}
}
},
{
"name": "requestRoutingRules",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestRoutingRules)]",
"input": {
"name": "[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestRoutingRules[copyIndex('requestRoutingRules')].name)]",
"properties": {
"httpListener": {
"id": "[resourceId(resourceGroup().name, 'microsoft.network/applicationGateways/httpListeners', concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name),concat(variables('namePrefix'), parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestRoutingRules[copyIndex('requestRoutingRules')].properties.httpListener))]"
},
"backendAddressPool": {
"id": "[resourceId(resourceGroup().name, 'microsoft.network/applicationGateways/backendAddressPools', concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name),concat(variables('namePrefix'), parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestRoutingRules[copyIndex('requestRoutingRules')].properties.backendAddressPool))]"
},
"backendHttpSettings": {
"id": "[resourceId(resourceGroup().name, 'microsoft.network/applicationGateways/backendHttpSettingsCollection', concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name),concat(variables('namePrefix'), parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestRoutingRules[copyIndex('requestRoutingRules')].properties.backendHttpSettings))]"
}
}
}
},
{
"name": "sslCertificates",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].sslCertificates)]",
"input": {
"name": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].sslCertificates[copyIndex('sslCertificates')].name]",
"properties": {
"data": "[parameters('appgwfesslcertsecret')]",
"password": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].sslCertificates[copyIndex('sslCertificates')].properties.password]"
}
}
},
{
"name": "authenticationCertificates",
"count": "[length(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].authenticationCertificates)]",
"input": {
"name": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].authenticationCertificates[copyIndex('authenticationCertificates')].name]",
"properties": {
"data": "[parameters('appgwbecertsecret')]"
}
}
}
],
"probes": [],
"webApplicationFirewallConfiguration": {
"enabled": true,
"firewallMode": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].firewallMode]",
"ruleSetType": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].ruleSetType]",
"ruleSetVersion": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].ruleSetVersion]",
"requestBodyCheck": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestBodyCheck]",
"maxRequestBodySizeInKb": "[if(parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].requestBodyCheck, parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].maxReqBodySize, json('null'))]",
"fileUploadLimitInMb": "[int(100)]"
},
"enableHttp2": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableHTTP2]"
},
"resources": [
{
"type": "providers/diagnosticSettings",
"name": "[concat('Microsoft.Insights/', parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].diagname)]",
"dependsOn": [
"[concat(variables('namePrefix'),parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].name)]"
],
"apiVersion": "2017-05-01-preview",
"properties": {
"name": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].diagname]",
"logs": [
{
"category": "ApplicationGatewayAccessLog",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayAccessLog]",
"retentionPolicy": {
"days": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewayAccessLogRetentionDays]",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayAccessLogRetention]"
}
},
{
"category": "ApplicationGatewayPerformanceLog",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayPerformanceLog]",
"retentionPolicy": {
"days": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewayPerformanceLogRetentionDays]",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayPerformanceLogRetention]"
}
},
{
"category": "ApplicationGatewayFirewallLog",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayFirewallLog]",
"retentionPolicy": {
"days": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].applicationGatewayFirewallLogRetentionDays]",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableApplicationGatewayFirewallLogRetention]"
}
}
],
"metrics": [
{
"category": "AllMetrics",
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableAllMetrics]",
"retentionPolicy": {
"enabled": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].enableAllMetricsRetentionPolicy]",
"days": "[parameters('applicationGatewaySettings').settings[copyIndex('appgwCopy')].allMetricsRetentionDays]"
}
}
]
}
}
]
}
],
"outputs": {}
}
Updated Parameter File:
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"namingSettings": {
"value": {
"name": {
"app": "demo",
"cloud": "azu",
"region": "eus2",
}
}
},
"taggingSettings": {
"value": {
"tags": {
"AppID": "demo",
"Environment": "nonprod",
}
}
},
"applicationGatewaySettings": {
"value": {
"settings": [
{
"name": "appgw-pcs01",
"applicationGatewaySku": "WAF_Medium",
"applicationGatewayTier": "WAF",
"applicationGatewayInstanceCount": 2,
"policyType": "Predefined",
"policy": "AppGwSslPolicy20170401S",
"publicIP": null,
"firewallMode": "Prevention",
"diagname": "Demo-Appgw",
"ruleSetType": "OWASP",
"ruleSetVersion": "3.0",
"requestBodyCheck": true,
"maxReqBodySize": 10,
"enableHTTP2": false,
"enableApplicationGatewayAccessLog": true,
"applicationGatewayAccessLogRetentionDays": 30,
"enableApplicationGatewayAccessLogRetention": true,
"enableApplicationGatewayPerformanceLog": true,
"applicationGatewayPerformanceLogRetentionDays": 30,
"enableApplicationGatewayPerformanceLogRetention": true,
"enableApplicationGatewayFirewallLog": true,
"applicationGatewayFirewallLogRetentionDays": 30,
"enableApplicationGatewayFirewallLogRetention": true,
"enableAllMetrics": true,
"enableAllMetricsRetentionPolicy": true,
"allMetricsRetentionDays": 30,
"frontendPorts": [
{
"name": "feport-80",
"properties": {
"port": 80
}
},
{
"name": "feport-443",
"properties": {
"port": 443
}
}
],
"gatewayIPConfigurations": [
{
"name": "gwipconfig-pcs01",
"properties": {
"subnet": {
"vnetName": "demo-vnet",
"vnetRGName": "demo",
"subnetName": "demo-subgw"
}
}
}
],
"sslCertificates": [
{
"name": "appgwfesslcert",
"properties": {
"data": null,
"password": "password"
}
}
],
"authenticationCertificates": [
{
"name": "appgwbecert",
"properties": {
"data": null
}
}
],
"frontEndIPConfigurations": [
{
"name": "feipcfg-pcs01",
"properties": {
"subnet": {
"vnetName": "demo-vnet",
"vnetRGName": "demo",
"subnetName": "demo-subgw"
}
}
}
],
"httpListeners": [
{
"name": "httplistener-pcs01",
"properties": {
"frontendIPConfiguration": "feipcfg-pcs01",
"frontendPort": "feport-80",
"protocol": "Http",
"sslCertificate": {}
}
},
{
"name": "httpslistener-pcs01",
"properties": {
"frontendIPConfiguration": "feipcfg-pcs01",
"frontendPort": "feport-443",
"protocol": "Https",
"sslCertificate": {
"id": "/subscriptions/105dcee5-gy46-48e3-9046-265c7379e647/resourceGroups/demo/providers/Microsoft.Network/applicationGateways/azu-eus2-nonprod-appgw-pcs01/sslCertificates/appgwfesslcert"
}
}
}
],
"backendHttpSettingsCollection": [
{
"name": "httpsetcol-default",
"properties": {
"protocol": "Http",
"port": 80,
"authenticationCertificates": []
}
},
{
"name": "httpssetcol-default",
"properties": {
"protocol": "Https",
"port": 443,
"authenticationCertificates": [
{
"id": "/subscriptions/105dcee5-gy46-48e3-9046-265c7379e647/resourceGroups/demo/providers/Microsoft.Network/applicationGateways/azu-eus2-nonprod-appgw-pcs01/authenticationCertificates/appgwbecert"
}
]
}
}
],
"backendAddressPools": [
{
"name": "beap-pcs01"
}
],
"requestRoutingRules": [
{
"name": "httpreqrtrule-pcs01",
"properties": {
"httpListener": "httplistener-pcs01",
"backendAddressPool": "beap-pcs01",
"backendHttpSettings": "httpsetcol-default"
}
},
{
"name": "httpsreqrtrule-pcs01",
"properties": {
"httpListener": "httpslistener-pcs01",
"backendAddressPool": "beap-pcs01",
"backendHttpSettings": "httpssetcol-default"
}
}
]
}
]
}
},
"appgwfesslcertsecret": {
"value": {
"reference": {
"keyVault": {
"id": "/subscriptions/105dcee5-gy46-48e3-9046-265c7379e647/resourceGroups/demo/providers/Microsoft.KeyVault/vaults/demo-kv-new"
},
"secretName": "appgwfesslcert"
}
}
},
"appgwbecertsecret": {
"value": {
"reference": {
"keyVault": {
"id": "/subscriptions/105dcee5-gy46-48e3-9046-265c7379e647/resourceGroups/demo/providers/Microsoft.KeyVault/vaults/demo-kv-new"
},
"secretName": "appgwbecert"
}
}
}
}
}
you can only reference KV secrets in the parameters section of the template (or parameters file). You cannot use it in a random place in the template
https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/template-tutorial-use-key-vault

Azure Data Factory V2 Copy Activity with Rest API giving one row for nested JSON

I am trying to flatten a nested JSON returned from a Rest source. The pipeline code is as follows.
The problem here is this pipeline returns only first object from JSON dataset and skips all the rest of the rows.
Can you please guide me on how to iterate over nested objects.
Thanks
Sameet
{
"name": "STG_NCR2",
"properties": {
"activities": [
{
"name": "Copy data1",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "RestSource",
"httpRequestTimeout": "00:01:40",
"requestInterval": "00.00:00:00.010",
"requestMethod": "GET",
"additionalHeaders": {
"OData-MaxVersion": "4.0",
"OData-Version": "4.0",
"Prefer": "odata.include-annotations=*"
}
},
"sink": {
"type": "AzureSqlSink"
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"mappings": [
{
"source": {
"path": "$['value'][0]['tco_ncrid']"
},
"sink": {
"name": "NCRID"
}
},
{
"source": {
"path": "['tco_name']"
},
"sink": {
"name": "EquipmentSerialNumber"
}
}
],
"collectionReference": "$['value'][0]['tco_ncr_tco_equipment']"
}
},
"inputs": [
{
"referenceName": "Rest_PowerApps_NCR",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "Prestaging_PowerApps_NCREquipments",
"type": "DatasetReference"
}
]
}
],
"annotations": []
}
}
The JSON is in the following format
[
{
"value":[
{
"tco_ncrid":"abc-123",
"tco_ncr_tco_equipment":[
{
"tco_name":"abc"
}
]
},
{
"tco_ncrid":"abc-456",
"tco_ncr_tco_equipment":[
{
"tco_name":"xyz"
},
{
"tco_name":"yzx"
}
}
]
]
}
]
This can be resolved by amending the translator property as follows.
"translator": {
"type": "TabularTranslator",
"mappings": [
{
"source": {
"path": "$.['value'][0].['tco_ncrid']"
},
"sink": {
"name": "NCRID",
"type": "String"
}
},
{
"source": {
"path": "$.['value'][0].['tco_text_id']"
},
"sink": {
"name": "EquipmentDescription",
"type": "String"
}
},
{
"source": {
"path": "['tco_name']"
},
"sink": {
"name": "EquipmentSerialNumber",
"type": "String"
}
}
],
"collectionReference": "$.['value'][*].['tco_ncr_tco_equipment']"
}
This code forces the pipeline to iterate over nested array but as you can see that the NCRID is hardcoded to first element of the value array. This is not exactly what I want as I am looking for all Equipment Serial Numbers against every NCRID. Still researching...

How to parse Json response and truncate child nodes

This is the JSON response I am trying to parse:
{
"data": {
"Content": {
"id": 26,
"name": "Dashboard1"
},
"List": [
{
"ListContent": {
"id": 178,
"name": "Card-144"
},
"cards": [
{
"id": 1780,
"configuration": {
"id": 7178,
"name": "Emp"
}
}
]
},
{
"ListContent": {
"id": 179,
"name": "Card-14"
},
"cards": [
{
"id": 1798,
"configuration": {
"id": 1789,
"name": "RandomColumns"
}
}
]
},
{
"ListContent": {
"id": 180,
"name": "Card-1"
},
"cards": [
{
"id": 18080,
"configuration": {
"id": 1080,
"allow": true
}
}
]
},
{
"ListContent": {
"id": 181,
"name": "Card-14"
},
"cards": [
{
"id": 18081,
"configuration": {
"id": 1881,
"name": "Functions"
}
}
]
},
{
"ListContent": {
"id": 182,
"name": "Card-1443"
},
"cards": [
{
"id": 1782,
"configuration": {
"id": 1802,
"name": "Emp-O"
}
}
]
}
]
}
}
From the Json, I need to extract "id"s under the "ListContent" nodes and store it in an array. Also, will need to ignore "id"s under the child nodes.
Here is a groovy script I am trying to achieve this with,
def CList = ""
import groovy.json.JsonSlurper
def jsonRespData = context.expand( '${TestStep#Response#$.data.List}' )
def outputResp = new JsonSlurper().parseText(jsonRespData)
outputResp.id.each()
{log.info( ":"+ it)
CList=CList.concat(it.toString()).concat(',')}
log.info (CList)
So, the array that I am expecting is CList [178,179,180,181,182]
but I am currently getting null.
What should be the correct groovy to only read "id" from "ListContent" and write it to an array?
Any help would be really appreciated.
Thanks in advance.
You can just use the (implicit) spread operator like this:
def json = new groovy.json.JsonSlurper().parse('/tmp/x.json' as File)
//
def i = json.data.List.ListContent.id
assert i == [178, 179, 180, 181, 182]
// with explicit spread operator
def e = json.data.List*.ListContent*.id
assert e == [178, 179, 180, 181, 182]
def str = '''
{
"data": {
"Content": {
"id": 26,
"name": "Dashboard1"
},
"List": [
{
"ListContent": {
"id": 178,
"name": "Card-144"
},
"cards": [
{
"id": 1780,
"configuration": {
"id": 7178,
"name": "Emp"
}
}
]
},
{
"ListContent": {
"id": 179,
"name": "Card-14"
},
"cards": [
{
"id": 1798,
"configuration": {
"id": 1789,
"name": "RandomColumns"
}
}
]
},
{
"ListContent": {
"id": 180,
"name": "Card-1"
},
"cards": [
{
"id": 18080,
"configuration": {
"id": 1080,
"allow": true
}
}
]
},
{
"ListContent": {
"id": 181,
"name": "Card-14"
},
"cards": [
{
"id": 18081,
"configuration": {
"id": 1881,
"name": "Functions"
}
}
]
},
{
"ListContent": {
"id": 182,
"name": "Card-1443"
},
"cards": [
{
"id": 1782,
"configuration": {
"id": 1802,
"name": "Emp-O"
}
}
]
}
]
}
}
'''
def outputResp = new groovy.json.JsonSlurper().parseText(str)
outputResp.data.List.collect { it.ListContent.id }
As you already have List from (context.expand( '${TestStep#Response#$.data.List}' )) , you can simply do:
outputResp.collect { it.ListContent.id }
Above returns an ArrayList.

Mongodb - Finding GeoNear within nested JSON objects

I need to get the closest place near a certain point using this data structure:
[
{
"data_id": "127",
"actual_data": [
{
"id": "220",
"value": "shaul"
},
{
"id": "221",
"value": "3234324"
},
{
"id": "222",
"value": {
"lngalt": [
13.7572225,
-124.0429047
],
"street_number": null,
"political": null,
"country": null,
}
},
{
"id": "223",
"value": "dqqqf1222fs3d7ddd77#Dcc11cS2112s.com"
},
{
"id": "224",
"value": "123123"
},
{
"id": "225",
"value": "lala1"
},
....
},
{
"data_id": "133",
"actual_data": [
{
"id": "260",
"value": {
"lngalt": [
1.7572225,
32.0429047
],
"street_number": null,
"political": null,
"country": null,
}
},
{
"id": "261",
"value": -122.25
}
],
}
]
I used the following query in order to get what I need:
{
"actual_data": {
"$elemMatch": {
"id": "260",
"value.lngalt": {
"$near": {
"$geometry": {
"type": "Point",
"coordinates": [
-73.9667,
40.78
]
},
"$minDistance": 1000,
"$maxDistance": 5000
}
}
}
}
}
But after queering it, I get "Can't canonicalize query: BadValue geoNear must be top-level expr" (error code: 17287). It's strange since I do get the right results when I query without the $near but with $elemMatch only in order to get the exact object with particular value.
Thanks.
SOLVED!
for future refarance I used $geoWithin instead $near function. So my query looks like :
{
"actual_data": {
"$elemMatch": {
"id": "260",
"value.lnglat": {
"$geoWithin": {
"$centerSphere": [
[
-71.07651880000003,
42.353068
],
0.001
]
}
}
}
}
}
PEACE!