how to convert this array in one? - json

I am playing around with jq at https://jqplay.org/
The JSON I currently have is :
{
"license": {
"type": "permissive",
"url": "http://en.wikipedia.org/wiki/MIT_License",
"name": "MIT"
},
"lib": "libxml2",
"vuln-count": {
"exact": 2,
"total": 3,
"historical": 1
},
"vulns": [
{
"exact": false,
"timestamp-objects": [],
"vuln": {
"published-epoch": "1072839600",
"cvss": 9.3,
"summary": "libxml2, possibly before 2.5.0.\"",
"published": "2003-12-31T03:00:00",
"cve": "CVE-2003-1564",
"modified": "2008-10-24T01:30:02",
"modified-epoch": "1224811802"
}
},
{
"exact": true,
"timestamp-objects": [
"libxml2-2.8.0-1.x86_64.cpio:/usr/lib64/libxml2.so.2.8.0"
],
"vuln": {
"published-epoch": "1356061574",
"cvss": 5,
"summary": "libxml2 before 2.8.0 computes hash values.",
"published": "2012-12-21T03:46:14",
"cve": "CVE-2012-0841",
"modified": "2014-01-28T02:42:55",
"modified-epoch": "1390876975"
}
},
{
"exact": true,
"timestamp-objects": [
"libxml2-2.8.0-1.x86_64.cpio:/usr/lib64/libxml2.so.2.8.0"
],
"vuln": {
"published-epoch": "1346432101",
"cvss": 6.8,
"summary": "libxml2 2.9.0-rc1 and earlier.",
"published": "2012-08-31T16:55:01",
"cve": "CVE-2012-2871",
"modified": "2014-01-28T02:45:36",
"modified-epoch": "1390877136"
}
}
]
}
my current filter is
if .["vuln-count"].exact >0
then {num_of_vulns: .["vuln-count"].exact, lib: .lib, license: .license.type, vuln: .vulns[]|select (.exact==true)|.vuln.cve}
else empty end
And the result is
{
"num_of_vulns": 2,
"lib": "libxml2",
"license": "permissive",
"vuln": "CVE-2012-0841"
}
{
"num_of_vulns": 2,
"lib": "libxml2",
"license": "permissive",
"vuln": "CVE-2012-2871"
}
What is the filter to use in order to get the following output?
{
"num_of_vulns": 2,
"lib": "libxml2",
"license": "permissive",
"vulns": ["CVE-2012-0841", "CVE-2012-2871"]
}
Thanks to Santiago, here is the working filter:
if .["vuln-count"].exact >0 then {
num_of_vulns: ."vuln-count".exact,
lib: .lib,
license: .license.type,
vulns: .vulns | map(select(.exact).vuln.cve)
} else empty end

Your expected result is not meaningful as a JSON! On a JSON object, the property keys must be unique. When property keys are repeated, JSON parsers will usually only keep the last one's value, disregarding the rest.
You can, however, obtain something like this:
{
"num_of_vulns": 2,
"lib": "libxml2",
"license": "permissive",
"vulns": ["CVE-2012-0841", "CVE-2012-2871"]
}
Here's a sample script that does so (I omitted your wrapping if for empty vulns, but it could be added around it):
{
num_of_vulns: ."vuln-count".exact,
lib: .lib,
license: .license.type,
vulns: .vulns | map(select(.exact).vuln.cve)
}

Related

How to transcode MP4 video with SRT subtitle on AWS Elemental MediaConvert

I have a MP4 video with SRT captions and I need to transcode them with media convert. In media convert I set automatic ABR and I specified the SRT origin path.
At the moment, I have tested the following:
I set SRT file in one output and video/audio in another
I set SRT, video and audio in the same output
For the first test, the job finish successfully, but on the S3 bucket there isnt any .SRT file. For the second test, the job fails with "aption destination type [SRT] requires a raw muxer." message
This is my JSON for the first test
{
"Queue": "arn:aws:mediaconvert:us-east-1:{{ACCOUNT-NUMBER}}:queues/Default",
"UserMetadata": {},
"Role": "arn:aws:iam::{{ACCOUNT-NUMBER}}:role/{{MY-ROLE-NAME}}",
"Settings": {
"TimecodeConfig": {
"Source": "ZEROBASED"
},
"OutputGroups": [
{
"Name": "DASH ISO",
"Outputs": [
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"ScalingBehavior": "DEFAULT",
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"ScanTypeConversionMode": "INTERLACED",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"GopClosedCadence": 1,
"GopSize": 90,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"EntropyEncoding": "CABAC",
"FramerateControl": "INITIALIZE_FROM_SOURCE",
"RateControlMode": "QVBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"MinIInterval": 0,
"AdaptiveQuantization": "AUTO",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "MULTI_PASS_HQ",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"AudioDescriptions": [
{
"AudioTypeControl": "FOLLOW_INPUT",
"AudioSourceName": "Audio Selector 1",
"CodecSettings": {
"Codec": "AAC",
"AacSettings": {
"AudioDescriptionBroadcasterMix": "NORMAL",
"Bitrate": 96000,
"RateControlMode": "CBR",
"CodecProfile": "LC",
"CodingMode": "CODING_MODE_2_0",
"RawFormat": "NONE",
"SampleRate": 48000,
"Specification": "MPEG4"
}
},
"StreamName": "latino",
"LanguageCodeControl": "FOLLOW_INPUT",
"LanguageCode": "SPA"
}
]
},
{
"ContainerSettings": {
"Container": "MPD"
},
"CaptionDescriptions": [
{
"CaptionSelectorName": "Captions Selector 1",
"DestinationSettings": {
"DestinationType": "SRT"
},
"LanguageCode": "SPA",
"LanguageDescription": "latino"
}
]
}
],
"OutputGroupSettings": {
"Type": "DASH_ISO_GROUP_SETTINGS",
"DashIsoGroupSettings": {
"SegmentLength": 30,
"MinFinalSegmentLength": 0,
"Destination": "s3://{{BUCKET-NAME}}/streaming15/dash-iso/",
"FragmentLength": 2,
"SegmentControl": "SINGLE_FILE",
"MpdProfile": "ON_DEMAND_PROFILE",
"HbbtvCompliance": "NONE"
}
},
"AutomatedEncodingSettings": {
"AbrSettings": {
"MaxAbrBitrate": 8000000,
"MinAbrBitrate": 600000
}
}
}
],
"AdAvailOffset": 0,
"Inputs": [
{
"AudioSelectors": {
"Audio Selector 1": {
"Offset": 0,
"DefaultSelection": "DEFAULT",
"ProgramSelection": 1
}
},
"VideoSelector": {
"ColorSpace": "FOLLOW",
"Rotate": "DEGREE_0",
"AlphaBehavior": "DISCARD"
},
"FilterEnable": "AUTO",
"PsiControl": "USE_PSI",
"FilterStrength": 0,
"DeblockFilter": "DISABLED",
"DenoiseFilter": "DISABLED",
"InputScanType": "AUTO",
"TimecodeSource": "ZEROBASED",
"CaptionSelectors": {
"Captions Selector 1": {
"SourceSettings": {
"SourceType": "SRT",
"FileSourceSettings": {
"SourceFile": "s3://{{BUCKET-NAME}}/PROMO_CAP_01.srt"
}
}
}
},
"FileInput": "s3://{{BUCKET-NAME}}/PROMO_CAP_01.mp4"
}
]
},
"AccelerationSettings": {
"Mode": "DISABLED"
},
"StatusUpdateInterval": "SECONDS_60",
"Priority": 0
}
What I am missing?
According to the AWS Elemental MediaConvert user guide, SRT is not a supported output for a DASH-ISO output group when the input caption type is SRT.
Here's a link to that guide (reference page 176):
https://docs.aws.amazon.com/mediaconvert/latest/ug/mediaconvert-guide.pdf
The supported caption outputs for SRT input in DASH-ISO are:
Burn in
IMSC (as sidecar .fmp4)
IMSC (as sidecar .xml)
TTML (as sidecar .fmp4)
TTML (as sidecar .ttml)
Additionally, there is a gap in the documentation. SRT->DASH-ISO+WebVTT is supported, even though it is not listed. The documentation will be corrected, but I wanted to share that with you in case it helps.
If you must send SRT to the output destination, then you could create a separate output group where the caption is in a track with no container (see pages 192-196 in the document).

Swift, JSON model

I have a question regarding building a JSON model namely, I should declare a date that will be different each day - in this case: "2020-11-19" as below.
This is a response for the current day.
{
"dates": {
"2020-11-19": {
"countries": {
"Poland": {
"date": "2020-11-19",
"id": "poland",
"links": [
{
"href": "/api/2020-11-19/country/poland",
"rel": "self",
"type": "GET"
}
],
"name": "Poland",
"name_es": "Polonia",
"name_it": "Polonia",
"regions": [],
"source": "John Hopkins University",
"today_confirmed": 796798,
"today_deaths": 12088,
"today_new_confirmed": 23975,
"today_new_deaths": 637,
"today_new_open_cases": 4335,
"today_new_recovered": 19003,
"today_open_cases": 422824,
"today_recovered": 361886,
"today_vs_yesterday_confirmed": 0.03102262743215456,
"today_vs_yesterday_deaths": 0.05562832940354556,
"today_vs_yesterday_open_cases": 0.010358695210626712,
"today_vs_yesterday_recovered": 0.055421236981710864,
"yesterday_confirmed": 772823,
"yesterday_deaths": 11451,
"yesterday_open_cases": 418489,
"yesterday_recovered": 342883
}
},
"info": {
"date": "2020-11-19 00:00CET",
"date_generation": "2020-11-19 22:34",
"yesterday": "2020-11-18 00:00CET"
}
}
},
"metadata": {
"by": "Narrativa & AppliedXL",
"url": [
"wwww.narrativa.com",
"www.appliedxl.com"
]
},
"total": {
"date": "2020-11-19",
"name": "Total",
"name_es": "Total",
"name_it": "Total",
"rid": "#total",
"source": "Narrativa",
"today_confirmed": 56684618,
"today_deaths": 1356365,
"today_new_confirmed": 525111,
"today_new_deaths": 8186,
"today_new_open_cases": 273944,
"today_new_recovered": 242981,
"today_open_cases": 19082735,
"today_recovered": 36245518,
"today_vs_yesterday_confirmed": 0.009350349175964112,
"today_vs_yesterday_deaths": 0.00607189401407382,
"today_vs_yesterday_open_cases": 0.014564678824917632,
"today_vs_yesterday_recovered": 0.00674899660543371,
"yesterday_confirmed": 56159507,
"yesterday_deaths": 1348179,
"yesterday_open_cases": 18808791,
"yesterday_recovered": 36002537
},
"updated_at": "2020-11-19 21:34UTC"
}
How I should build a model so that this property would be changed every day? (of course, this date needs to have a proper format "yyyy-MM-dd")
I would probably do:
var dates: [String: YourModel]
if it will potentially have multiple dates in that field.. otherwise, I'd probably do a custom object that overrides init(from decoder: and parses out that info

How to build a parent key based on a sub-string of another value using jq?

I'm re-shaping a JSON file which base content is the output of an AWS API call. From the raw output, I'm currently extracting the fields that I'm interested in the format I want except for a small detail that motivated this question.
Specifically, my input comes from the output of aws rds describe-db-instances command and contains the info of multiple RDS instances. Taking an example with 2 instances, this is how it looks:
{
"DBInstances": [
{
"DBInstanceIdentifier": "db1-name",
"DBInstanceClass": "db.m5.xlarge",
"Engine": "oracle-ee",
"DBInstanceStatus": "available",
"MasterUsername": "user",
"DBName": "RANDOM",
"Endpoint": {
"Address": "some-endpoint.rds.amazonaws.com",
"Port": 5698,
"HostedZoneId": "GHDSFHFSDHSDH"
},
"AllocatedStorage": 4000,
"InstanceCreateTime": "2018-07-23T23:21:42.361000+00:00",
"PreferredBackupWindow": "09:30-07:00",
"BackupRetentionPeriod": 14,
"DBSecurityGroups": [],
"VpcSecurityGroups": [
{
"VpcSecurityGroupId": "sg-xxxxxxxxxxxxxxxxx",
"Status": "active"
},
{
"VpcSecurityGroupId": "sg-xxxxxxxxxxxxxxxxx",
"Status": "active"
}
],
"DBParameterGroups": [
{
"DBParameterGroupName": "DB1-parameter",
"ParameterApplyStatus": "in-sync"
}
],
"AvailabilityZone": "ZONE1",
"DBSubnetGroup": {
"DBSubnetGroupName": "dbsubnetgroup-1",
"DBSubnetGroupDescription": "dbsubnetgroup-1",
"VpcId": "vpc-xxxxxxxxxxxxxxxxx",
"SubnetGroupStatus": "Complete",
"Subnets": [
{
"SubnetIdentifier": "subnet-xxxxxxxxxxxxxxxxx",
"SubnetAvailabilityZone": {
"Name": "az1"
},
"SubnetStatus": "Active"
},
{
"SubnetIdentifier": "subnet-xxxxxxxxxxxxxxxxx",
"SubnetAvailabilityZone": {
"Name": "az2"
},
"SubnetStatus": "Active"
}
]
},
"PreferredMaintenanceWindow": "sat:07:00-sat:07:30",
"PendingModifiedValues": {},
"LatestRestorableTime": "2020-03-27T18:54:25+00:00",
"MultiAZ": false,
"EngineVersion": "X.X.X",
"AutoMinorVersionUpgrade": false,
"ReadReplicaDBInstanceIdentifiers": [],
"LicenseModel": "bring-your-own-license",
"Iops": 5000,
"OptionGroupMemberships": [
{
"OptionGroupName": "optiongroupName",
"Status": "in-sync"
}
],
"CharacterSetName": "WE8ISO8859P15",
"PubliclyAccessible": false,
"StorageType": "io1",
"DbInstancePort": 0,
"StorageEncrypted": true,
"KmsKeyId": "someKey",
"DbiResourceId": "db-xxxxxxxxxxxxxxxxxxxxxxxxx",
"CACertificateIdentifier": "rds-ca-2019",
"DomainMemberships": [],
"CopyTagsToSnapshot": true,
"MonitoringInterval": 0,
"DBInstanceArn": "someARN",
"IAMDatabaseAuthenticationEnabled": false,
"PerformanceInsightsEnabled": false,
"DeletionProtection": false,
"AssociatedRoles": []
},
{
"DBInstanceIdentifier": "db2-name",
"DBInstanceClass": "db.m5.large",
"Engine": "oracle-ee",
"DBInstanceStatus": "available",
"MasterUsername": "user2",
"DBName": "XXXX",
"Endpoint": {
"Address": "endpoint2.rds.amazonaws.com",
"Port": 8974,
"HostedZoneId": "FASDFDS54FSA"
},
"AllocatedStorage": 100,
"InstanceCreateTime": "2020-04-23T21:38:53.023000+00:00",
"PreferredBackupWindow": "01:00-05:30",
"BackupRetentionPeriod": 35,
"DBSecurityGroups": [],
"VpcSecurityGroups": [
{
"VpcSecurityGroupId": "sg-xxxxxxxxxxxxxxxxx",
"Status": "active"
}
],
"DBParameterGroups": [
{
"DBParameterGroupName": "default",
"ParameterApplyStatus": "in-sync"
}
],
"AvailabilityZone": "AZ-2",
"DBSubnetGroup": {
"DBSubnetGroupName": "subnet-group",
"DBSubnetGroupDescription": "",
"VpcId": "vpc-xxxxxxxxxxxxxxxxx",
"SubnetGroupStatus": "Complete",
"Subnets": [
{
"SubnetIdentifier": "subnet-xxxxxxxxxxxxxxxxx",
"SubnetAvailabilityZone": {
"Name": "AZ-1"
},
"SubnetStatus": "Active"
},
{
"SubnetIdentifier": "subnet-xxxxxxxxxxxxxxxxx",
"SubnetAvailabilityZone": {
"Name": "AZ-2"
},
"SubnetStatus": "Active"
},
{
"SubnetIdentifier": "subnet-xxxxxxxxxxxxxxxxx",
"SubnetAvailabilityZone": {
"Name": "AZ-3"
},
"SubnetStatus": "Active"
}
]
},
"PreferredMaintenanceWindow": "sun:08:39-sun:09:09",
"PendingModifiedValues": {},
"LatestRestorableTime": "2020-07-27T18:53:18+00:00",
"MultiAZ": false,
"EngineVersion": "X.X.X",
"AutoMinorVersionUpgrade": false,
"ReadReplicaDBInstanceIdentifiers": [],
"LicenseModel": "bring-your-own-license",
"Iops": 2000,
"OptionGroupMemberships": [
{
"OptionGroupName": "optiongroup-name",
"Status": "in-sync"
}
],
"CharacterSetName": "AL32UTF8",
"PubliclyAccessible": false,
"StorageType": "io1",
"DbInstancePort": 0,
"StorageEncrypted": true,
"KmsKeyId": "someARN",
"DbiResourceId": "db-xxxxxxxxxxxxxxxxx",
"CACertificateIdentifier": "rds-ca-2019",
"DomainMemberships": [],
"CopyTagsToSnapshot": false,
"MonitoringInterval": 0,
"DBInstanceArn": "someARN",
"IAMDatabaseAuthenticationEnabled": false,
"PerformanceInsightsEnabled": false,
"DeletionProtection": false,
"AssociatedRoles": []
}
]
}
This is my current output:
[
{
"DBInstancePrefix": {
"DBInstanceClass": "db.m5.xlarge",
"DBInstanceIdentifier": "db1-name",
"DBName": "RANDOM",
"DBParameterGroupName": "DB1-parameter",
"DBSubnetGroupName": "dbsubnetgroup-1",
"KmsKeyId": "someKey",
"OptionGroupName": "optiongroupName",
"VpcSecurityGroupIds": [
"sg-xxxxxxxxxxxxxxxxx",
"sg-xxxxxxxxxxxxxxxxx"
]
}
},
{
"DBInstancePrefix": {
"DBInstanceClass": "db.m5.large",
"DBInstanceIdentifier": "db2-name",
"DBName": "XXXX",
"DBParameterGroupName": "default",
"DBSubnetGroupName": "subnet-group",
"KmsKeyId": "someARN",
"OptionGroupName": "optiongroup-name",
"VpcSecurityGroupIds": [
"sg-xxxxxxxxxxxxxxxxx"
]
}
}
]
This is my current JQ filter:
. | [.[] | map(.) | .[] | {DBInstancePrefix: {DBInstanceClass: .DBInstanceClass, DBInstanceIdentifier: .DBInstanceIdentifier, DBName: .DBName, DBParameterGroupName:.DBParameterGroups[].DBParameterGroupName, DBSubnetGroupName: .DBSubnetGroup.DBSubnetGroupName, KmsKeyId:.KmsKeyId, OptionGroupName: .OptionGroupMemberships[].OptionGroupName, VpcSecurityGroupIds: [.VpcSecurityGroups | .[] | .VpcSecurityGroupId] }}]
You can verify it on this snippet on jqplay.org.
What I need is to turn the parent key "DBInstancePrefix" dynamic based on a substring from "DBInstanceIdentifier" key. So for the example names I wrote would be:
"db1-name" >>> "db1"
"db2-name" >>> "db2"
So, my desired output would be:
[
{
"db1": {
"DBInstanceClass": "db.m5.xlarge",
"DBInstanceIdentifier": "db1-name",
"DBName": "RANDOM",
"DBParameterGroupName": "DB1-parameter",
"DBSubnetGroupName": "dbsubnetgroup-1",
"KmsKeyId": "someKey",
"OptionGroupName": "optiongroupName",
"VpcSecurityGroupIds": [
"sg-xxxxxxxxxxxxxxxxx",
"sg-xxxxxxxxxxxxxxxxx"
]
}
},
{
"db2": {
"DBInstanceClass": "db.m5.large",
"DBInstanceIdentifier": "db2-name",
"DBName": "XXXX",
"DBParameterGroupName": "default",
"DBSubnetGroupName": "subnet-group",
"KmsKeyId": "someARN",
"OptionGroupName": "optiongroup-name",
"VpcSecurityGroupIds": [
"sg-xxxxxxxxxxxxxxxxx"
]
}
}
]
Any ideas or suggestions? Thanks for reading.
To manipulate the object key-name as you desire, you can apply the filter operation inside (..). Any operation done inside it, the result is preserved "literally".
Your case demands the DBInstanceIdentifier to be split by - and using the first element in the resultant array.
With that and few other trivial modifications, you need
.DBInstances |
map
(
{
( .DBInstanceIdentifier | split("-")[0] ): {
DBInstanceClass,
DBInstanceIdentifier,
DBName,
DBParameterGroupName:.DBParameterGroups[].DBParameterGroupName,
DBSubnetGroupName: .DBSubnetGroup.DBSubnetGroupName,
KmsKeyId,
OptionGroupName: .OptionGroupMemberships[].OptionGroupName,
VpcSecurityGroupIds: [.VpcSecurityGroups[] | .VpcSecurityGroupId ]
}
}
)
jqplay - Demo
I took the long path and since I couldn't resolve it on the same filter I did another one that takes the output of the first one and thus achieved the desired result. But it is awful compared with the answer selected as the solution.
Based on Inian answer, I did a small modification since my real DBInstanceIdentifier values have the following format:
<name>-db-<environment> and I need DBInstancePrefix becomes <name>-db.
So, my final filter is:
.DBInstances |
map
(
{
( .DBInstanceIdentifier|split("-")[0:2] | join("-") ): {
DBInstanceClass,
DBInstanceIdentifier,
DBName,
DBParameterGroupName:.DBParameterGroups[].DBParameterGroupName,
DBSubnetGroupName: .DBSubnetGroup.DBSubnetGroupName,
KmsKeyId,
OptionGroupName: .OptionGroupMemberships[].OptionGroupName,
VpcSecurityGroupIds: [.VpcSecurityGroups[] | .VpcSecurityGroupId ]
}
}
)

What is wrong with my composer.json file?

I am getting a composer error and I don't understand what is wrong. The json file look fine to me.
[Seld\JsonLint\ParsingException]
"./composer.json" does not contain valid JSON
Parse error on line 88:
...] } "patches": {
--------------------^
Expected one of: 'EOF', '}', ',', ']'
composer.json in the gist below
https://gist.github.com/MrPaulDriver/89d6573e654815d75368cae121076acb
You are missing a , before the patches section, adjust to:
{
"name": "drupal-composer/drupal-project",
"description": "Project template for Drupal 8 projects with composer",
"type": "project",
"license": "GPL-2.0+",
"authors": [
{
"name": "",
"role": ""
}
],
"repositories": [
{
"type": "composer",
"url": "https://packages.drupal.org/8"
}
],
"require": {
"composer/installers": "^1.2",
"cweagans/composer-patches": "^1.6",
"drupal-composer/drupal-scaffold": "^2.2",
"drupal/admin_toolbar": "^1.19",
"drupal/block_class": "1.x-dev",
"drupal/coffee": "1.x-dev",
"drupal/console": "~1.0",
"drupal/core": "~8.0",
"drupal/devel": "1.x-dev",
"drupal/ds": "^3.1",
"drupal/easy_breadcrumb": "^1.6",
"drupal/field_group": "3.x-dev",
"drupal/flexslider": "2.x-dev",
"drupal/google_analytics": "^2.1",
"drupal/metatag": "^1.1",
"drupal/neato": "^1.6",
"drupal/pathauto": "^1.0",
"drupal/responsive_menu": "^2.4",
"drupal/smart_trim": "1.x-dev",
"drupal/token": "^1.0",
"drush/drush": "~8.0",
"webflo/drupal-finder": "^0.3.0",
"webmozart/path-util": "^2.3"
},
"require-dev": {
"behat/mink": "~1.7",
"behat/mink-goutte-driver": "~1.2",
"jcalderonzumba/gastonjs": "~1.0.2",
"jcalderonzumba/mink-phantomjs-driver": "~0.3.1",
"mikey179/vfsstream": "~1.2",
"phpunit/phpunit": ">=4.8.28 <5",
"symfony/css-selector": "~2.8"
},
"conflict": {
"drupal/drupal": "*"
},
"minimum-stability": "dev",
"prefer-stable": true,
"config": {
"sort-packages": true
},
"autoload": {
"classmap": [
"scripts/composer/ScriptHandler.php"
]
},
"scripts": {
"drupal-scaffold": "DrupalComposer\\DrupalScaffold\\Plugin::scaffold",
"pre-install-cmd": [
"DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
],
"pre-update-cmd": [
"DrupalProject\\composer\\ScriptHandler::checkComposerVersion"
],
"post-install-cmd": [
"DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
],
"post-update-cmd": [
"DrupalProject\\composer\\ScriptHandler::createRequiredFiles"
]
},
"extra": {
"installer-paths": {
"web/core": ["type:drupal-core"],
"web/libraries/{$name}": ["type:drupal-library"],
"web/modules/contrib/{$name}": ["type:drupal-module"],
"web/profiles/contrib/{$name}": ["type:drupal-profile"],
"web/themes/contrib/{$name}": ["type:drupal-theme"],
"drush/contrib/{$name}": ["type:drupal-drush"]
},
"patches": {
"drupal/responsive_menu": {
"Disable keyboard input": "https://www.drupal.org/files/issues/off_canvas_menu-2826965-11.patch"
}
}
}
}
Note You might want to use something like https://jsonlint.com, or a proper IDE that will let you know if something is wrong with a JSON file.

Serialize the response from backend to store ember store

My response from backend is not in form which ember store. I am not able to serialize the response.
response.json
[{
"pk": 127,
"url": "http://example.com/api/galleries/127/",
"gallery_name": "Faces",
"thumbnail_url": "https://example.cloud.net/galleryThumbs/2656a05c-4ec7-3eea-8c5e-d8019454d443.jpg",
"time": "1 month ago",
"description": "Created by user",
"is_following": true,
"feedPhotos": [{
"pk": 624,
"url": "http://example.com/api/photos/624/",
"profilePic": "https://example.cloud.net/userDPs/50906ce2-394d-39c8-9261-8cf78e3611c2.jpg",
"userName": "Nabeela",
"userKarma": 915,
"caption": "Old woman spinning her 'chhos-khor' ...a rotation of which is equivalent to the recitation of a mantra.",
"numComments": 0,
"owner": "http://example.com/api/users/44/",
"time": "1 month ago",
"photo_url": "https://example.cloud.net/photos/9cbd6423-3bc5-36e0-b8b4-d725efb3249a.jpg",
"comments_url": "http://example.com/api/photos/624/comments/",
"numFives": 4,
"fivers_url": "http://example.com/api/photogalleries/1362/fivers/",
"fivers_pk": 1362,
"fullphoto_url": "http://example.com/api/photogalleries/1362/photo/",
"fullphoto_pk": 1362,
"is_fived": true,
"hiFiveKarma": 1,
"owner_pk": 44,
"userFirstName": "Nabeela",
"is_bookmarked": false
}, {
"pk": 574,
"url": "http://example.com/api/photos/574/",
"profilePic": "https://example.cloud.net/userDPs/b6f69e4e-980d-3cc3-8b3e-3eb1a7f21350.jpg",
"userName": "Rohini",
"userKarma": 194,
"caption": "Life # Myanmar!",
"numComments": 0,
"owner": "http://example.com/api/users/45/",
"time": "2 months ago",
"photo_url": "https://example.cloud.net/photos/eeae72d5-d6af-391e-a218-b442c0c7e34e.jpg",
"comments_url": "http://example.com/api/photos/574/comments/",
"numFives": 2,
"fivers_url": "http://example.com/api/photogalleries/1303/fivers/",
"fivers_pk": 1303,
"fullphoto_url": "http://example.com/api/photogalleries/1303/photo/",
"fullphoto_pk": 1303,
"is_fived": false,
"hiFiveKarma": 0,
"owner_pk": 45,
"userFirstName": "Rohini",
"is_bookmarked": false
}
]
}, {
"pk": 65,
"url": "http://example.com/api/galleries/65/",
"gallery_name": "Royal",
"thumbnail_url": "https://example.cloud.net/galleryThumbs/d8a900af-1f1d-3977-8cc8-b8bb36e32be5.jpg",
"time": "2 months ago",
"description": "This is a gallery about Royal",
"is_following": false,
"feedPhotos": [{
"pk": 347,
"url": "http://example.com/api/photos/347/",
"profilePic": "https://example.cloud.net/userDPs/50906ce2-394d-39c8-9261-8cf78e3611c2.jpg",
"userName": "Nabeela",
"userKarma": 915,
"caption": "I cannot forget the name of this palace - Moti Mahal (translation: Pearl Palace). Indescribably beautiful, ainnit! at Mehrangarh fort, Jodhp",
"numComments": 0,
"owner": "http://example.com/api/users/44/",
"time": "2 months ago",
"photo_url": "https://example.cloud.net/photos/958ed406-708e-3f01-a2f4-9467cd709fdd.jpg",
"comments_url": "http://example.com/api/photos/347/comments/",
"numFives": 4,
"fivers_url": "http://example.com/api/photogalleries/759/fivers/",
"fivers_pk": 759,
"fullphoto_url": "http://example.com/api/photogalleries/759/photo/",
"fullphoto_pk": 759,
"is_fived": false,
"hiFiveKarma": 0,
"owner_pk": 44,
"userFirstName": "Nabeela",
"is_bookmarked": false
}, {
"pk": 593,
"url": "http://example.com/api/photos/593/",
"profilePic": "https://example.cloud.net/userDPs/95ac6974-f7df-338c-ab84-99fa1df7514c.jpg",
"userName": "Vikanshu",
"userKarma": 932,
"caption": "Marvelous architecture!! in Florence, Italy",
"numComments": 0,
"owner": "http://example.com/api/users/48/",
"time": "1 month ago",
"photo_url": "https://example.cloud.net/photos/7a86eb37-6c68-3d6c-b6cf-2e3b74d330dd.jpg",
"comments_url": "http://example.com/api/photos/593/comments/",
"numFives": 4,
"fivers_url": "http://example.com/api/photogalleries/1363/fivers/",
"fivers_pk": 1363,
"fullphoto_url": "http://example.com/api/photogalleries/1363/photo/",
"fullphoto_pk": 1363,
"is_fived": false,
"hiFiveKarma": 0,
"owner_pk": 48,
"userFirstName": "Vikanshu",
"is_bookmarked": false
}]
}]
How do I serialize this using JSONPISerailizer or any other serializer in ember-cli so that it gets stored in ember store
Reference jsonapi.org
++++Top Level:
Root:
A JSON object must be root of every JSON API request response.
A document must contain at least one top-level members:
1. data: documents "primary data"
2. errors: an array of error objects (id,status,code,title....)
3. meta: a meta object that contains non-standard meta-information (copyright,author...)
member data and errors must not co-exist together.
"data"{}
+++++Resource Objects
1. A resource object MUST contain atleast following top-level member
*id
*type
```
//structure-1
//for galleries
{
"data": {
"type": "galleries",
"id": "1"
}
}
//for photos
{
"data": {
"type": "photos",
"id": "1"
}
}
```
In addition, a resource object may contain any of these top-level members
*attributes
*relationship
*links
*meta
//added attributes first
```
//structure-2
//for galleries
{
"data": {
"type": "galleries",
"id": "1",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
}
}
}
//for photos
{
"data": {
"type": "photos",
"id": "1",
"attributes":{
userName: "Nabeela",
userKarma: 915
}
}
}
```
//Adding relationship
Relationship object must contain atleast one of the following
*links (containing atleast one of "self" or "related" resource link
*data
*meta
//link in relationship (minimum one required from link,data,meta).
//
```
//structure-3
//for galleries
{
"data":[{ //Array(square bracket as adding relationship one more item to data
"type": "galleries",
"id": "1",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
},
"relationships":{
"links":{
"self": "https://example.cloud.net/photos/9cbd6423.jpg //"photo_url" in your payload
},
}]
}
}
```
//data in relationship
```
//structure-4
//for galleries
{
"data": [{
"type": "galleries",
"id": "1",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
},
"relationships":{ //This has all your photo stuff
"links":{
"self": "https://example.cloud.net/photos/9cbd6423.jpg //"photo_url" in your payload
},
"data": { //picked it up from structure-1
"type": "photos",
"id": "77"
}
}]
}
}
```
//Adding related resource "included"
```
//for galleries
{
"data": [{
"type": "galleries",
"id": "1",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
},
"relationships":{ //This has all your photo stuff
"links":{
"self": "https://example.cloud.net/photos/9cbd6423.jpg //"photo_url" in your payload
},
"data": { //picked it up from structure-1
"type": "photos",
"id": "77"
}
}],
"included":[{
"type": "photos",
"id": "77",
"attributes":{
userName: "Nabeela",
userKarma: 915
},
{
"type": "photos",
"id": "78",
"attributes":{
userName: "Nabeela",
userKarma: 915
}
}]
}
}
```
For collections. I am not confident but try this
Now for collection of galleries.
//for galleries
{
"data": [{
"type": "galleries",
"id": "1",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
},
"relationships":{ //This has all your photo stuff
"links":{
"self": "https://example.cloud.net/photos/9cbd6423.jpg //"photo_url" in your payload
},
"data": { //picked it up from structure-1
"type": "photos",
"id": "77"
}
},{
"type": "galleries",
"id": "2",
"attributes": {
"galleryName": "Faces"
"thumbnailUrl:"https://example.cloud.net/galleryThumbs/2656a05c-4ec7.jpg",
"description": "Created by user",
},
"relationships":{ //This has all your photo stuff
"links":{
"self": "https://example.cloud.net/photos/9cbd6423.jpg //"photo_url" in your payload
},
"data": { //picked it up from structure-1
"type": "photos",
"id": "79"
}
}],
"included":[{
"type": "photos",
"id": "77",
"attributes":{
userName: "Nabeela",
userKarma: 915
},{
"type": "photos",
"id": "78",
"attributes":{
userName: "Nabeela",
userKarma: 915
},{
"type": "photos",
"id": "79",
"attributes":{
userName: "Nabeela",
userKarma: 915
}
}]
}
}
============Implementation part =================================
JSONSerializer normalization process follows these steps
*normalizeResponse : entry method.
*normalizeCreateRecordResponse : a normalizeResponse for specific operation.
*normalizeSingleResponse|normalizeArrayResponse:
- for methods like createRecord. we expect a single record back.
- for methods like findAll we expect multiple records back.
+normalize =
normalizeArray iterates and calls normalize for each of it's records
normalizeSingle call its once.
+extractID | extractAttributes | extractRelationships
= normalize delegates to these method to turn record payload into jsonAPI format
Starting with normalizeResponse method. If you open and see normalizeResponse method
in json-serializer
link normalizeResponse: https://github.com/emberjs/data/blob/v2.2.1/packages/ember-
data/lib/serializers/json-serializer.js#L192
you with find a switch case switch(requestType). If requestType if
"findRecord" then "normalizeFindRecordResponse" is called
"queryRecord" then "normalizeQueryRecordResponse" is called
"findAll" then "normalizeFindAllResponse" is called
...so on and so forth.
if you notice the parameter passed to all the methods are same as that of normalize
(...arguments) :)
**Lets start for findAll
i.e normalizeResponse -> normalizeFindAllResponse -> normalizeArrayResponse
as normalizeFindAllResponse method has only one line that call
normalizeArrayResponse.
normalizeFindAllResponse
normalizeResponse -> normalizeFindAllResponse -> normalizeArrayResponse ->
_normalizeResponse{ extractMeta,normalize }
extractMeta [extract meta information like pagination and stuff ]
if single: normalize []
example of normalize method in emberjs docs
```
import DS from 'ember-data';
export default DS.JSONSerializer.extend({
normalize: function(typeClass, hash) {
var fields = Ember.get(typeClass, 'fields');
fields.forEach(function(field) {
var payloadField = Ember.String.underscore(field);
if (field === payloadField) { return; }
hash[field] = hash[payloadField];
delete hash[payloadField];
});
return this._super.apply(this, arguments);
}
});
```
"normalizeArrayResponse calls `return this._normalizeResponse
(store,primaryModelClass,payload,id,requestType,false).
so isSingle is false for _normalizeResponse method. so we will have to push all the
related records of included array
in our case the photos which is done by below snippet from "_normalizeRespose"
method.
_normalizeResponse
```
else{
documentHash.data = payload.map((item) => {
let { data, included } = this.normalize(primaryModelClass,item);
if(included){
documentHash.included.push(...included);
}
return data;
});
return documentHash;
}
```
Things are still unclear in the context of our JSON reponse from server
but atleast we know the flow now.
Lets try to apply it for findAll ( as per the flow above).
run "ember g serializer application" //assuming you are using ember-cli and you
intend to make this serializer generic for application.
As of now I have no information how and when normalizeResponse is called. :(
I just scanned through and guess on recieving data from server the store calls
normalizeResponseHelpers which in turn calls normalizeResponse.
In any case "normalizeResponse" is going to send payload and other necessar
information to normalizeFindAllResponse(...arguments) which in turn will call
normalizeArrayResponse(...arguments) which in turn will call "_normalizeRespone".
Here is where we need to take action
for extraMeta and normalize.
+extraMeta
I am not sure if there is any meta information in you json response.
in case there is you can refer to the example mentioned in docs
extractMeta
So I guess you can directly user the normalize method from example in your application ;).
please try and check. Since i am learning ember myself I cannot guarantee it will work but it should. the lonngggg explation is my thought while i was learning the problem/solution
//app/serializers/application.js
+normalize
```
import DS from 'ember-data';
export default DS.JSONSerializer.extend({
normalize: function(typeClass, hash) {
var fields = Ember.get(typeClass, 'fields');
fields.forEach(function(field) {
var payloadField = Ember.String.underscore(field);
if (field === payloadField) { return; }
hash[field] = hash[payloadField];
delete hash[payloadField];
});
return this._super.apply(this, arguments);
}
});
```
The primary key in the JSON from server is pk. You will have to mention that too
http://emberjs.com/api/data/classes/DS.JSONSerializer.html#property_primaryKey
app/serializers/application.js
import DS from 'ember-data';
export default DS.JSONSerializer.extend({
primaryKey: 'pk'
});