I'm reading a JSON file in spark but the dataframe output keeps giving me a table with all null values when I use df.show()....................... I was just wondering if there's something with my structure schema? Or maybe I'm missing something? I've also attached the json data below. Any help would be appreciate it.....................................
discount_type = StructType([StructField("amount", IntegerType(), True),
StructField("description", StringType(), True)
])
child_item_type = StructType([StructField("InsuranceNumber", StringType(), True),
StructField("InsuranceLabel", StringType(), True),
StructField("Insurancequantity", DoubleType(), True),
StructField("Insuranceprice", IntegerType(), True),
StructField("discountsreceived",discount_type , True),
])
item_type = StructType([StructField("InsuranceNumber", StringType(), True),
StructField("InsuranceLabel", StringType(), True),
StructField("Insurancequantity", IntegerType(), True),
StructField("Insuranceprice", IntegerType(), True),
StructField("discountsreceived", discount_type, True),
StructField("childItems",child_item_type , True),
])
order_paid_type = StructType([StructField("Insuranceid", StringType(), True),
StructField("Insurancedesc", StringType(), True),
StructField("purchaseditems", item_type, True),
])
message_type = StructType([StructField("PaidIn", order_paid_type, True)])
data_type = StructType([StructField("Client", message_type, True)])
body_type = StructType([StructField("id", StringType(), True),
StructField("InsuranceProvider", StringType(), True),
StructField("Type", data_type, True),
StructField("eventTime", StringType(), True),
])
[
df = spark.read.schema(body_type).json(INPUT_FILE)
[
{
"id": "164651478631223455788978942317",
"InsuranceProvider": "Embroker",
"Type": {
"Client": {
"PaidIn": {
"Insuranceid": "97331549875122744335422",
"Insurancedesc": "Magic happens here",
"purchaseditems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "DNO",
"Insurancequantity": 1,
"Insuranceprice": 345,
"discountsreceived": [
{
"amount": 495,
"description": "Item 1, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 2495,
"description": "Insurance item 1, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "2",
"InsuranceLabel": "LPL",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": -295,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 495,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "3",
"InsuranceLabel": "LPL",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 295,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 400,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "4",
"InsuranceLabel": "LPL",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 295,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 335,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "5",
"InsuranceLabel": "Employment Practices Liability",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 1295,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 195,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "6",
"InsuranceLabel": "Employment Practices Liability",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 805,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 501,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "7",
"InsuranceLabel": "Employment Practices Liability",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 521,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 533,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "8",
"InsuranceLabel": "Employment Practices Liability",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 422,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 333,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "9",
"InsuranceLabel": "Employment Practices Liability",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 444,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 666,
"description": "Insurance item 2, Discount 1"
}
]
}
]
},
{
"InsuranceNumber": "10",
"InsuranceLabel": "DNO",
"Insurancequantity": 2,
"Insuranceprice": 945,
"discountsreceived": [
{
"amount": 10,
"description": "Item 2, Discount 1"
}
],
"childItems": [
{
"InsuranceNumber": "1",
"InsuranceLabel": "Cyber",
"Insurancequantity": 1,
"Insuranceprice": 0,
"discountsreceived": [
{
"amount": 63,
"description": "Insurance item 2, Discount 1"
}
]
}
]
}
]
}
}
},
"eventTime": "2020-05-19T01:59:10.379Z"
}
]
This question already has answers here:
How to flatten a nested JSON recursively, with flatten_json
(1 answer)
Python Pandas - Flatten Nested JSON
(1 answer)
Closed 9 months ago.
I am trying to flatten the following JSON and flatten it hierarchically:
https://justpaste.it/6e60p
I am using pandas json_normalize function to do this but I am bit stuck.
pd.json_normalize(test_json['result'])
Gives me 2 columns with nested dicts.
I am trying to use record_path=['userDetails'] but then it opens only the user part.
Please advise what is the best practice to flatten the JSON hierarchically?
test_json = {
"success": True,
"error": 'null',
"errorObject": None,
"result": [
{
"requestId": 45,
"userIds": [
4
],
"userDetails": [
{
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
],
"requestColumnGrants": [
{
"id": 57,
"creationTime": 1652083406187,
"lastUpdateTime": 1652083407853,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 1,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "nationkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "nationkey",
"description": None,
"tags": [
{
"id": 1,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "pii",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 0,
"ordinalOverride": None
},
"operationType": "FULL_MASKING",
"operationPayload": "{\"letter\":\"*\"}",
"policyRecommendedOperationType": "FULL_MASKING",
"policyRecommendedOperationPayload": "{\"letter\":\"*\"}",
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 56,
"creationTime": 1652083406165,
"lastUpdateTime": 1652083406165,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 2,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "name",
"rawType": "STRING",
"type": "UNKNOWN",
"trinoName": "name",
"description": None,
"tags": [
{
"id": 2,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "first_name",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 1,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 55,
"creationTime": 1652083406141,
"lastUpdateTime": 1652083406141,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 3,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "regionkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "regionkey",
"description": None,
"tags": [
{
"id": 5,
"creationTime": 1652080042000,
"lastUpdateTime": 1652080042000,
"name": "region",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 2,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
}
],
"workflowValues": [
{
"requestId": 45,
"requestWorkflowProgressionId": 106,
"name": "f3624cb8-cebd-4608-9d9a-60c5b21adbd3.firstName",
"value": "dcasd",
"type": "String",
"source": "Form",
"id": 1,
"creationTime": 1652083422000,
"lastUpdateTime": 1652083422000
},
{
"requestId": 45,
"requestWorkflowProgressionId": 106,
"name": "f3624cb8-cebd-4608-9d9a-60c5b21adbd3.lastNam",
"value": "dsadw",
"type": "String",
"source": "Form",
"id": 2,
"creationTime": 1652083422000,
"lastUpdateTime": 1652083422000
}
]
},
{
"requestId": 44,
"userIds": [
4
],
"userDetails": [
{
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
],
"requestColumnGrants": [
{
"id": 54,
"creationTime": 1652082684949,
"lastUpdateTime": 1652082686821,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 1,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "nationkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "nationkey",
"description": None,
"tags": [
{
"id": 1,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "pii",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 0,
"ordinalOverride": None
},
"operationType": "FULL_MASKING",
"operationPayload": "{\"letter\":\"*\"}",
"policyRecommendedOperationType": "FULL_MASKING",
"policyRecommendedOperationPayload": "{\"letter\":\"*\"}",
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 53,
"creationTime": 1652082684927,
"lastUpdateTime": 1652082684927,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 2,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "name",
"rawType": "STRING",
"type": "UNKNOWN",
"trinoName": "name",
"description": None,
"tags": [
{
"id": 2,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "first_name",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 1,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 52,
"creationTime": 1652082684905,
"lastUpdateTime": 1652082684905,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 3,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "regionkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "regionkey",
"description": None,
"tags": [
{
"id": 5,
"creationTime": 1652080042000,
"lastUpdateTime": 1652080042000,
"name": "region",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 2,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
}
],
"workflowValues": []
},
{
"requestId": 40,
"userIds": [
4
],
"userDetails": [
{
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
],
"requestColumnGrants": [
{
"id": 42,
"creationTime": 1651412063835,
"lastUpdateTime": 1651412065727,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 1,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "nationkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "nationkey",
"description": None,
"tags": [
{
"id": 1,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "pii",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 0,
"ordinalOverride": None
},
"operationType": "FULL_MASKING",
"operationPayload": "{\"letter\":\"*\"}",
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 41,
"creationTime": 1651412063807,
"lastUpdateTime": 1651412063807,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 2,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "name",
"rawType": "STRING",
"type": "UNKNOWN",
"trinoName": "name",
"description": None,
"tags": [
{
"id": 2,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "first_name",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 1,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 40,
"creationTime": 1651412063779,
"lastUpdateTime": 1651412063779,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 3,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "regionkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "regionkey",
"description": None,
"tags": [
{
"id": 5,
"creationTime": 1652080042000,
"lastUpdateTime": 1652080042000,
"name": "region",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 2,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
}
],
"workflowValues": []
},
{
"requestId": 39,
"userIds": [
4
],
"userDetails": [
{
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
],
"requestColumnGrants": [
{
"id": 39,
"creationTime": 1651411857024,
"lastUpdateTime": 1651411858669,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 1,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "nationkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "nationkey",
"description": None,
"tags": [
{
"id": 1,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "pii",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 0,
"ordinalOverride": None
},
"operationType": "FULL_MASKING",
"operationPayload": "{\"letter\":\"*\"}",
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 38,
"creationTime": 1651411856994,
"lastUpdateTime": 1651411856994,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 2,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "name",
"rawType": "STRING",
"type": "UNKNOWN",
"trinoName": "name",
"description": None,
"tags": [
{
"id": 2,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "first_name",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 1,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 37,
"creationTime": 1651411856970,
"lastUpdateTime": 1651411856970,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 3,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "regionkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "regionkey",
"description": None,
"tags": [
{
"id": 5,
"creationTime": 1652080042000,
"lastUpdateTime": 1652080042000,
"name": "region",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 2,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
}
],
"workflowValues": []
},
{
"requestId": 38,
"userIds": [
4
],
"userDetails": [
{
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
],
"requestColumnGrants": [
{
"id": 36,
"creationTime": 1651411738066,
"lastUpdateTime": 1651411739933,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 1,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "nationkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "nationkey",
"description": None,
"tags": [
{
"id": 1,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "pii",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 0,
"ordinalOverride": None
},
"operationType": "FULL_MASKING",
"operationPayload": "{\"letter\":\"*\"}",
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 35,
"creationTime": 1651411738036,
"lastUpdateTime": 1651411738036,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 2,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "name",
"rawType": "STRING",
"type": "UNKNOWN",
"trinoName": "name",
"description": None,
"tags": [
{
"id": 2,
"creationTime": 1651048111000,
"lastUpdateTime": 1651048111000,
"name": "first_name",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 1,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
},
{
"id": 34,
"creationTime": 1651411738012,
"lastUpdateTime": 1651411738013,
"schemaInfoDetails": {
"id": 1,
"displayName": "adam-test-collibra"
},
"tableInfoDetails": {
"id": 1,
"displayName": "nation"
},
"columnInfoDetails": {
"id": 3,
"creationTime": 1651047171000,
"lastUpdateTime": 1652080042000,
"name": "regionkey",
"rawType": "INT64",
"type": "UNKNOWN",
"trinoName": "regionkey",
"description": None,
"tags": [
{
"id": 5,
"creationTime": 1652080042000,
"lastUpdateTime": 1652080042000,
"name": "region",
"tagLevel": "COLUMN",
"riskFactor": "NONE"
}
],
"ordinal": 2,
"ordinalOverride": None
},
"operationType": "NONE",
"operationPayload": None,
"policyRecommendedOperationType": None,
"policyRecommendedOperationPayload": None,
"riskFactor": "NONE",
"creatorUserId": 4,
"creatorUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
},
"lastUpdateUserId": 4,
"lastUpdateUserDetails": {
"displayName": "Admin Demo",
"profileImageUrl": None,
"roles": [
"ADMIN"
],
"countryIso": "IL",
"id": 4,
"creationTime": 1650970312000,
"lastUpdateTime": 1651046585000
}
}
],
"workflowValues": []
}
],
"paging": {
"start": 0,
"limit": 20,
"total": 5,
"hasMore": False
}
}
flatten_json is a library now, so you can do this. It'll give you 160 columns
from flatten_json import flatten
dic_flattened = (flatten(d, '.') for d in test_json['result'])
df = pd.DataFrame(dic_flattened)
df.shape
(5, 160)
Please tell me the ffmpeg cammand line code to get information of mp4 file in json format. i want details of video(in mp4) to json format. The details include images, layers , background, audio information etc just as shown in code below.(The output is show below i need input command line code).Thanks in advance
{
"images":[
{
"name":"img_0.jpg",
"w":540,
"h":960,
"prefix":[
"-i"
],
"postfix":[
]
},
{
"name":"img_1.jpg",
"w":540,
"h":960,
"prefix":[
"-i"
],
"postfix":[
]
},
{
"name":"img_2.jpg",
"w":540,
"h":960,
"prefix":[
"-i"
],
"postfix":[
]
},
{
"name":"img_3.jpg",
"w":540,
"h":960,
"prefix":[
"-i"
],
"postfix":[
]
},
{
"name":"img_4.jpg",
"w":540,
"h":960,
"prefix":[
"-i"
],
"postfix":[
]
}
],
"static_inputs":[
{
"name":"background_video.mp4",
"prefix":[
"-i"
],
"postfix":[
]
}
],
"video":{
"h":960,
"w":1080,
"duration":30
},
"e":[],
"m":[
"-{pythoncomplex}"
],
"r":["[5]split=2[color][alpha]; [color]crop=iw/2:ih:0:0[color]; [alpha]crop=iw/2:ih:iw/2:0[alpha];
[color][alpha]{pythonmerge}[ovrly]; [0]scale=540:960,setsar=1[0_scalled];
[1]scale=540:960,setsar=1[1_scalled]; [2]scale=540:960,setsar=1[2_scalled];
[3]scale=540:960,setsar=1[3_scalled]; [4]scale=540:960,setsar=1[4_scalled]; [0_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*5:s=540x960,
{pythonf}=t=out:st=6:d=1[v0]; [1_scalled]{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,
{pythonz}-0.0015))':d=25*5:s=540x960,{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=6:d=1[v1];
[2_scalled]{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*7:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=8:d=1[v2]; [3_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*7:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=8:d=1[v3]; [4_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*6:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=7:d=1[v4]; [v0][v1][v2][v3]
[v4]concat=n=5:v=1:a=0,format=yuv420p[concatenated_video]; [concatenated_video][ovrly]
{pythono}=0:0[base_video]; [base_video][6]
{pythono}=enable='between(t,0,15)':x=30:y=30[watermarked_part1]; [watermarked_part1][6]
{pythono}=enable='between(t,15,30)':x=(main_w-{pythono}_w-30):y=(main_h-{pythono}_h-30)"],
"i":["[5]split=2[color][alpha]; [color]crop=iw/2:ih:0:0[color]; [alpha]crop=iw/2:ih:iw/2:0[alpha];
[color][alpha]{pythonmerge}[ovrly]; [0]scale=540:960,setsar=1[0_scalled];
[1]scale=540:960,setsar=1[1_scalled]; [2]scale=540:960,setsar=1[2_scalled];
[3]scale=540:960,setsar=1[3_scalled]; [4]scale=540:960,setsar=1[4_scalled]; [0_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*5:s=540x960,
{pythonf}=t=out:st=6:d=1[v0]; [1_scalled]{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,
{pythonz}-0.0015))':d=25*5:s=540x960,{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=6:d=1[v1];
[2_scalled]{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*7:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=8:d=1[v2]; [3_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*7:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=8:d=1[v3]; [4_scalled]
{pythonz}pan=z='if(lte({pythonz},1.0),1.1,max(1.001,{pythonz}-0.0015))':d=25*6:s=540x960,
{pythonf}=t=in:st=0:d=1,{pythonf}=t=out:st=7:d=1[v4]; [v0][v1][v2][v3]
[v4]concat=n=5:v=1:a=0,format=yuv420p[concatenated_video]; [concatenated_video][ovrly]
{pythono}=0:0[base_video]; [base_video][6]
{pythono}=enable='between(t,0,15)':x=30:y=30[watermarked_part1]; [watermarked_part1][6]
{pythono}=enable='between(t,15,30)':x=(main_w-{pythono}_w-30):y=(main_h-{pythono}_h-30)
[final_video]"],
"n":[],
"g":[],
"c":[],
"o":[
"-map",
"[final_video]",
"-map",
"7:a"
],
"d":[
"-t",
"30"
],
"s":[],
"layers": [
{
"ao": 0,
"bm": 1,
"ddd": 0,
"ind": 1,
"ip": 0,
"ks": {
"a": {
"a": 0,
"ix": 1,
"k": [
270,
480,
0
]
},
"o": {
"a": 0,
"ix": 11,
"k": 18.039
},
"p": {
"a": 0,
"ix": 2,
"k": [
270,
480,
0
]
},
"r": {
"a": 0,
"ix": 10,
"k": 0
},
"s": {
"a": 0,
"ix": 6,
"k": [
100,
100,
100
]
}
},
"nm": "all",
"op": 327,
"refId": "image_0",
"sr": 1,
"st": 0,
"ty": 2
},
{
"ao": 0,
"bm": 3,
"ddd": 0,
"ind": 2,
"ip": 76,
"ks": {
"a": {
"a": 0,
"ix": 1,
"k": [
270,
480,
0
]
},
"o": {
"a": 0,
"ix": 11,
"k": 40
},
"p": {
"a": 0,
"ix": 2,
"k": [
270,
480,
0
]
},
"r": {
"a": 0,
"ix": 10,
"k": 0
},
"s": {
"a": 0,
"ix": 6,
"k": [
100,
100,
100
]
}
},
"nm": "lvjing all 3",
"op": 327,
"refId": "image_1",
"sr": 1,
"st": -8,
"ty": 2
},
{
"ao": 0,
"bm": 13,
"ddd": 0,
"ind": 3,
"ip": 76,
"ks": {
"a": {
"a": 0,
"ix": 1,
"k": [
270,
480,
0
]
},
"o": {
"a": 0,
"ix": 11,
"k": 50.196
},
"p": {
"a": 0,
"ix": 2,
"k": [
270,
480,
0
]
},
"r": {
"a": 0,
"ix": 10,
"k": 0
},
"s": {
"a": 0,
"ix": 6,
"k": [
100,
100,
100
]
}
},
"nm": "baohedu all 3",
"op": 327,
"refId": "image_2",
"sr": 1,
"st": -8,
"ty": 2
}
]
}
You can print the video information in a json file using below command
ffprobe -v quiet -print_format json -show_format -show_streams "input.mp4" >output.json
you can see the sample output from this link
let me know if you have anything to know more
Use ffprobe with -print_format json and > to redirect output to a file:
ffprobe -v quiet -print_format json -show_format -show_streams "video.mp4" > ffprobe_output.json
Note that > is a a Linux program, not a function of ffprobe.
I am now struggling for hours but I cannot get my feed import to run properly.
The strange is, that I can import data for the normal fields in Drupal like "Title" or "Language" or "User". But for my custom fields like "field_produkt_beschreibung", no entries are made in Drupal. Even if I supply a Standard Value for the field:
The standard value is not saved in the node.
{
"entity": "node",
"settings": {
"uniq_path": "SKU",
"preprocess": "",
"feed": {
"protect_on_invalid_source": 0,
"protect_on_fewer_items": "0"
},
"processor": {
"name": "default",
"class": "FeedImportProcessor",
"options": {
"items_count": "0",
"skip_imported": 0,
"updates_only": 0,
"reset_cache": "0",
"throw_exception": 1,
"max_reported_errors": "100",
"break_on_undefined_filter": 0,
"skip_defined_functions_check": 0,
"uniq_callback": "",
"after_save": "",
"before_combine": "",
"after_combine": "",
"before_create": "",
"before_save": ""
}
},
"reader": {
"name": "sql",
"class": "SQLFIReader",
"options": {
"dsn": "mysql:dbname=d****7;host=localhost",
"user": "d******7",
"pass": "*****",
"query": "SELECT * FROM mytable",
"params": ""
}
},
"hashes": {
"name": "sql",
"class": "FeedImportSQLHashes",
"options": {
"ttl": 0,
"insert_chunk": 300,
"update_chunk": 300,
"group": "produkte_h_lter"
}
},
"filter": {
"name": "default",
"class": "FeedImportMultiFilter",
"options": {
"param": "[field]",
"include": ""
}
},
"fields": {
"title": {
"field": "title",
"column": false,
"paths": [
"Produktname"
],
"default_action": 0,
"default_value": "Standardwert",
"update_mode": 0,
"filters": {
"\u00fc": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ü",
"\u00fc",
"1"
]
},
"\u00e4": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ä",
"\u00e4",
"1"
]
},
"\u00d6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"Ö",
"\u00d6",
"1"
]
},
"\u00f6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ö",
"\u00f6",
"1"
]
},
"\u00df": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ß",
"\u00df",
"1"
]
},
"R": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"®",
" ",
"1"
]
}
},
"prefilters": []
},
"field_produkt_hersteller_raw": {
"field": "field_produkt_hersteller_raw",
"column": true,
"paths": [
"//Hersteller"
],
"default_action": 0,
"default_value": "kein Hersteller",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_link": {
"field": "field_produkt_link",
"column": true,
"paths": [
"Link"
],
"default_action": 0,
"default_value": "Standard",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_artikelnummer": {
"field": "field_produkt_artikelnummer",
"column": true,
"paths": [
"Artikelnummer"
],
"default_action": 0,
"default_value": "Standardwert",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_beschreibung": {
"field": "field_produkt_beschreibung",
"column": true,
"paths": [
"Beschreibung"
],
"default_action": 0,
"default_value": "Text",
"update_mode": 0,
"filters": {
"\u00fc": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ü",
"\u00fc",
"1"
]
},
"\u00e4": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ä",
"\u00e4",
"1"
]
},
"\u00d6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"Ö",
"\u00d6",
"1"
]
},
"\u00f6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ö",
"\u00f6",
"1"
]
},
"\u00df": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ß",
"\u00df",
"1"
]
},
"R": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"®",
" ",
"1"
]
}
},
"prefilters": []
}
},
"static_fields": {
"type": "produkt",
"uid": "3",
"language": "de",
"field_produkt_shop": {
"tid": "20"
},
"title": "Test"
},
"functions": []
}
}
I really would appreciate a hint, about what might be wrong with my setup. Thank you very much,
Simon
It was missing filters and umlaute (ä, ö...) in the text fields. It is working now