Trying to build complex nested JSON using JOLT - json

I am trying to build complex nested array of JSON objects. I am struggling to get my expected structure using JOLT. Any help would be appreciated.
I am trying to understand JOLT from the appspot and it is hard for me to grasp about recursively iterating over data set.
The "JobId" should be same to milliseconds even if there are 1000 objects in array that is why i am generating "JobId" one time rather than in every object and try to move to right place in next iteration unsuccessfully.
The Input JSON is:
{
"type": "FeatureCollection",
"totalFeatures": 2,
"features": [
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
13.429916995511855,
52.54516530881942
]
},
"properties": {
"feature_type": "BLDG",
"feature_geometry": "point",
"discipline": "LOC",
"activity": "AC",
"be_number": 12313,
"category": 47400,
"condition": "RDY",
"review_date": "2018-03-28T21:36:32.325Z",
"role_type": "INSG",
"symbol_code": "-"
}
},
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
16.429916995511855,
91.54516530881942
]
},
"properties": {
"feature_type": "Bridge",
"feature_geometry": "point",
"discipline": "LOC",
"activity": "AC",
"co_number": 12313,
"category": 47400,
"condition": "RDY",
"review_date": "2018-03-28T21:36:32.325Z",
"role_type": "KLDV",
"symbol_code": "-"
}
}
]
}
The Spec I came up with so far:
[{
"operation": "shift",
"spec": {
"type": "type",
"features": {
"*": {
"geometry": {
"coordinates": {
"0": "FacilityComposite[&3].Facility.Ilat",
"1": "FacilityComposite[&3].Facility.Ilon"
}
},
"properties": {
"activity": "FacilityComposite[&2].Facility.Activity",
"be_number": "FacilityComposite[&2].Facility.BeNumber",
"category": "FacilityComposite[&2].Facility.Category",
"condition": "FacilityComposite[&2].Facility.Condition",
"role_type": ["FacilityComposite[&2].Facility.RoleType", "FacilityComposite[&2].FacForm.RoleType", "FacilityComposite[&2].FacTie[&2].RoleType"],
"review_date": ["FacilityComposite[&2].Facility.ReviewDate", "FacilityComposite[&2].FacForm.ReviewDate", "FacilityComposite[&2].FacTie[&2].ReviewDate"]
}
}
}
}
}, {
"operation": "default",
"spec": {
"JobId": "${now():toNumber()}",
"FacilityComposite[]": {
"*": {
"Facility": {
"FacSk": "-1",
"ClientId": "DISHNET",
"ReviewDate": "${now()}"
},
"FacForm": {
"FacSk": "-1",
"ClientId": "DISHNET"
},
"FacTie": [{
"FacTieSk": "-1",
"TieFromSk": "-1",
"TieToSk": "-1",
"ClientId": "DISHNET"
}]
}
}
}
}
]
The out put with the spec is:
{
"type" : "FeatureCollection",
"FacilityComposite" : [ {
"Facility" : {
"Ilat" : 13.429916995511855,
"Ilon" : 52.54516530881942,
"Activity" : "AC",
"BeNumber" : 12313,
"Category" : 47400,
"Condition" : "RDY",
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"ClientId" : "DISHNET",
"FacSk" : "-1"
},
"FacForm" : {
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"FacSk" : "-1",
"ClientId" : "DISHNET"
},
"FacTie" : [ {
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z"
} ]
}, {
"Facility" : {
"Ilat" : 16.429916995511856,
"Ilon" : 91.54516530881942,
"Activity" : "AC",
"Category" : 47400,
"Condition" : "RDY",
"RoleType" : "KLDV",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"ClientId" : "DISHNET",
"FacSk" : "-1"
},
"FacForm" : {
"RoleType" : "KLDV",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"FacSk" : "-1",
"ClientId" : "DISHNET"
},
"FacTie" : [ null, {
"RoleType" : "KLDV",
"ReviewDate" : "2018-03-28T21:36:32.325Z"
} ]
} ],
"JobId" : "${now():toNumber()}"
}
Expected output is:
{
"type" : "FeatureCollection",
[
"FacilityComposite" : {
"Facility" : {
"JobId" : "${now():toNumber()}"
"Ilat" : 13.429916995511855,
"Ilon" : 52.54516530881942,
"Activity" : "AC",
"BeNumber" : 12313,
"Category" : 47400,
"Condition" : "RDY",
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"ClientId" : "DISHNET",
"FacSk" : "-1"
},
"FacForm" : {
"JobId" : "${now():toNumber()}"
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"FacSk" : "-1",
"ClientId" : "DISHNET"
},
"FacTie" : [ {
"FacTieSk": "-1",
"TieFromSk": "-1",
"TieToSk": "-1",
"JobId" : "${now():toNumber()}"
"ClientId": "DISHNET"
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z"
} ]
},
"FacilityComposite" :{
"Facility" : {
"JobId" : "${now():toNumber()}"
"Ilat" : 16.429916995511856,
"Ilon" : 91.54516530881942,
"Activity" : "AC",
"Category" : 47400,
"Condition" : "RDY",
"RoleType" : "KLDV",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"ClientId" : "DISHNET",
"FacSk" : "-1"
},
"FacForm" : {
"JobId" : "${now():toNumber()}"
"RoleType" : "KLDV",
"ReviewDate" : "2018-03-28T21:36:32.325Z",
"FacSk" : "-1",
"ClientId" : "DISHNET"
},
"FacTie" : [ {
"TieFromSk": "-1",
"TieToSk": "-1",
"JobId" : "${now():toNumber()}"
"ClientId": "DISHNET"
"RoleType" : "INSG",
"ReviewDate" : "2018-03-28T21:36:32.325Z"
} ]
} ],
}

This is the spec That gave me what i am looking for:
[{
"operation": "shift",
"spec": {
"type": "type",
"features": {
"*": {
"geometry": {
"coordinates": {
"0": "FacilityComposite[&3].Facility.Ilat",
"1": "FacilityComposite[&3].Facility.Ilon"
}
},
"properties": {
"activity": "FacilityComposite[&2].Facility.Activity",
"be_number": "FacilityComposite[&2].Facility.BeNumber",
"category": "FacilityComposite[&2].Facility.Category",
"condition": "FacilityComposite[&2].Facility.Condition",
"role_type": ["FacilityComposite[&2].Facility.RoleType", "FacilityComposite[&2].FacForm.RoleType", "FacilityComposite[&2].FacTie[&2].RoleType"],
"review_date": ["FacilityComposite[&2].Facility.ReviewDate", "FacilityComposite[&2].FacForm.ReviewDate", "FacilityComposite[&2].FacTie[&2].ReviewDate"]
}
}
}
}
}, {
"operation": "default",
"spec": {
"JobId": "${now():toNumber()}",
"FacilityComposite[]": {
"*": {
"Facility": {
"FacSk": "-1",
"ClientId": "DISHNET",
"ReviewDate": "${now()}"
},
"FacForm": {
"FacSk": "-1",
"ClientId": "DISHNET"
}
}
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"FacilityComposite": {
"*": {
"FacTie": {
"*": {
"FacTieSk": "-1",
"TieFromSk": "-1",
"TieToSk": "-1",
"ClientId": "DISHNET"
}
}
}
}
}
}
]

Related

Selectively get certain arrays from inside an array in JSON using JOLT

I've been trying to get some fields out of a very long and complicated json format but not getting the output I want.
MY current spec obtains all events from the event array and lists them in the output. I'm unsure how to select specific events and only output those. Am not quite sure of the syntax
My JSON:
{
"rootid": "19718",
"clloadm": "2021-06-01T22:40:02",
"clload": "2021-06-01T21:21:39",
"date": "2021-05-25T21:52:30",
"events": [
{
"done": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"id": "e0_event",
"started": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"id": "e1_event",
"started": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"status": "complete"
},
{
"done": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"id": "e2_event",
"started": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"id": "e3_event",
"started": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"id": "e4_event",
"started": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
}
],
"ids": [
{
"id": "id",
"source": "source",
"value": "value"
},
{
"id": "new_id",
"source": "new_source",
"value": "value"
}
]
}
My Jolt Spec that gets all events for now:
[
{
"operation": "shift",
"spec": {
"rootid": "rootid",
"clloadm": "clloadm",
"clload": "clload",
"date": "date",
"events": {
"*": {
"*": {
"#value": "#id"
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]
The output I get:
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e1" : [ "2021-05-27T02:20:25", "2021-05-27T02:20:25" ],
"e2" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e3" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
The output I would like
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
You can write the individual keys e0 and e4 as conditional cases for #id key while rewriting the rest of the key-value pairs through "*":"&" representation such as
[
{
"operation": "shift",
"spec": {
"*": "&",
"events": {
"*": {
"*": {
"#id": {
"e0": { "#(2,value)": "&" },
"e4": { "#(2,value)": "&" }
}
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]

NiFi, Jolt's operation "substring" doesn't work

I have some datetime marks, and what I pretend is to group it in the quarter of hour just before it.
I have, e.g., this data:
[
{
"id": "123",
"dateTime": "2020-07-08T08:49:50+02:00",
"value": "1"
},
{
"id": "123",
"dateTime": "2020-07-08T13:14:57+02:00",
"value": "1"
},
{
"id": "123",
"dateTime": "2020-07-08T13:15:15+02:00",
"value": "1"
},
{
"id": "123",
"dateTime": "2020-07-08T13:36:39+02:00",
"value": "1"
}
]
After some JOLT Transformations, I have this data:
[ {
"id" : "123",
"dateHour" : "2020-07-08T08",
"minutes" : "45",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "0",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "15",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "30",
"value" : "1"
} ]
here, I have a problem. With the webpage http://jolt-demo.appspot.com/, I have managed to convert that "0" in "00" with the following JOLT code
[
{ "operation": "modify-overwrite-beta",
"spec": {
"*": {
"minutes": "=concat('0',#(0))"
}
}
},
{ "operation": "modify-overwrite-beta",
"spec": {
"*": {
"minutes": "=substring(#(1,minutes),1,3)"
}
}
}
]
Expected output (and achived with the webpage):
[ {
"id" : "123",
"dateHour" : "2020-07-08T08",
"minutes" : "45",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "00",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "15",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "30",
"value" : "1"
} ]
Real output in NiFi:
[ {
"id" : "123",
"dateHour" : "2020-07-08T08",
"minutes" : "045",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "00",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "015",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "030",
"value" : "1"
} ]
For some reason, "=substring()" function doesn't work as I expected in NiFi. Does anyone have any knack to take the last 2 items of the string?
You can convert to number and leftPad it. See if it helps
[
{
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"minutes": ["=toInteger", 0]
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"minutes": "=toString"
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"minutes": "=leftPad(#(1,minutes),2,'0')"
}
}
}
]
Input:
[
{
"id": "123",
"dateHour": "2020-07-08T08",
"minutes": "045",
"value": "1"
},
{
"id": "123",
"dateHour": "2020-07-08T13",
"minutes": "00",
"value": "1"
},
{
"id": "123",
"dateHour": "2020-07-08T13",
"minutes": "015",
"value": "1"
},
{
"id": "123",
"dateHour": "2020-07-08T13",
"minutes": "030",
"value": "1"
}
]
output:
[ {
"id" : "123",
"dateHour" : "2020-07-08T08",
"minutes" : "45",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "00",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "15",
"value" : "1"
}, {
"id" : "123",
"dateHour" : "2020-07-08T13",
"minutes" : "30",
"value" : "1"
} ]

Jolt transforming to array

I am getting null in my output json. Please find my spec and details below. The input json can have n numbers of COMPINFO . pls suggest
my input.json is
{
"valid": "true",
"message": "",
"data": {
"COMPINFO": [
{
"ORGID": "",
"SITEID": "BWDEMO",
"COMPID": "C2014",
"COMP_DESC": "Cherokee High School",
"ASSETTYPE": "MANUFACTURING",
"BUILDING": "Main",
"FLR_LEVEL": "Ground",
"ROOM_SPCE": "100"
},
{
"ORGID": "",
"SITEID": "BWDEMO",
"COMPID": "9001B",
"COMP_DESC": "Sludge Pump",
"ASSETTYPE": "FACILITY",
"BUILDING": "Main",
"FLR_LEVEL": "Production",
"ROOM_SPCE": "100"
}
]
}
}
my Spec.json is
[
{
"operation": "shift",
"spec": {
"data": {
"COMPINFO": {
"*": {
"COMPID": "[&1].COMPID",
"ORGID": "[&1].ORGID",
"COMP_DESC": "[&1].DESCRIPTION",
"BUILDING": "[&1].LOCATIONS.[&1].Building",
"FLR_LEVEL": "[&1].LOCATIONS.[&1].Floor_Level",
"ROOM_SPCE": "[&1].LOCATIONS.[&1].ROOM_SPCE",
"SITEID": "[&1].SITEID",
"ASSETTYPE": "[&1].ASSETTYPE"
}
}
}
}
}
]
expected output should be :
[
{
"COMPID" : "C2014",
"ORGID" : "",
"DESCRIPTION" : "Cherokee High School",
"LOCATIONS" : [ {
"Building" : "Main",
"Floor_Level" : "Ground",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "MANUFACTURING"
}, {
"COMPID" : "9001B",
"ORGID" : "",
"DESCRIPTION" : "Sludge Pump",
"LOCATIONS" : [{
"Building" : "Main",
"Floor_Level" : "Production",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "FACILITY"
}
]
but getting null (highlighted (Bold)) :
[
{
"COMPID" : "C2014",
"ORGID" : "",
"DESCRIPTION" : "Cherokee High School",
"LOCATIONS" : [ {
"Building" : "Main",
"Floor_Level" : "Ground",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "MANUFACTURING"
}, {
"COMPID" : "9001B",
"ORGID" : "",
"DESCRIPTION" : "Sludge Pump",
"LOCATIONS" : [ null, {
"Building" : "Main",
"Floor_Level" : "Production",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "FACILITY"
}
]
can someone help quickly ??
Thanks in advance .ssssssssssssssssssssss
It looks like you there is only one "location" per input item, and that you just want them to alway be the first element in a location array.
If so that is easy. If you are wanting to "group" your data and have some of the locations array have multiple items, that is a harder transform.
Spec for the simple version
[
{
"operation": "shift",
"spec": {
"data": {
"COMPINFO": {
"*": {
"COMPID": "[&1].COMPID",
"ORGID": "[&1].ORGID",
"COMP_DESC": "[&1].DESCRIPTION",
"BUILDING": "[&1].LOCATIONS[0].Building",
"FLR_LEVEL": "[&1].LOCATIONS[0].Floor_Level",
"ROOM_SPCE": "[&1].LOCATIONS[0].ROOM_SPCE",
"SITEID": "[&1].SITEID",
"ASSETTYPE": "[&1].ASSETTYPE"
}
}
}
}
}
]
There are examples around for the harder version.

opsworks parameters and resources in the cloudformation template

I'm working on a cloudformation template that has many parameters, like the ID of the elastic file system and the DNS of the MountTarget. I want to retreive these parameters from the existing resources, but if they are not already created I want to create them in the template :
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "AWS CloudFormation Template : Stack OpsWorks Slave pour deployer les instances script",
"Mappings": {
"Region2Principal": {
"us-east-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"us-west-2": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"us-west-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"eu-west-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
}
},
"AWSInstanceType2Arch" : {
"t1.micro" : { "Arch" : "PV64" },
"t2.micro" : { "Arch" : "HVM64" },
"m1.small" : { "Arch" : "PV64" },
"m1.large" : { "Arch" : "PV64" },
"m4.large" : { "Arch" : "HVM64" },
"m4.xlarge" : { "Arch" : "HVM64" },
"m4.2xlarge" : { "Arch" : "HVM64" },
"m4.4xlarge" : { "Arch" : "HVM64" },
"m4.10xlarge" : { "Arch" : "HVM64" },
"m4.16xlarge" : { "Arch" : "HVM64" },
"c3.large" : { "Arch" : "HVM64" },
"c3.xlarge" : { "Arch" : "HVM64" },
"c3.2xlarge" : { "Arch" : "HVM64" },
"c3.4xlarge" : { "Arch" : "HVM64" },
"c3.8xlarge" : { "Arch" : "HVM64" },
"c4.large" : { "Arch" : "HVM64" },
"c4.xlarge" : { "Arch" : "HVM64" },
"c4.2xlarge" : { "Arch" : "HVM64" },
"c4.4xlarge" : { "Arch" : "HVM64" },
"c4.8xlarge" : { "Arch" : "HVM64" }
},
"AWSRegionArch2AMI" : {
"us-east-1" : { "PV64" : "ami-5fb8c835", "HVM64" : "ami-60b6c60a" },
"us-west-1" : { "PV64" : "ami-56ea8636", "HVM64" : "ami-d5ea86b5" },
"eu-west-1" : { "PV64" : "ami-95e33ce6", "HVM64" : "ami-bff32ccc" },
"us-west-2" : { "PV64" : "ami-d93622b8", "HVM64" : "ami-f0091d91" }
}
},
"Parameters": {
"OpsWorksStackColor": {
"Description": "RGB Color to use for OpsWorks Stack",
"Type": "String",
"Default": "rgb(38, 146, 168)"
},
"Region" : {
"Type":"String",
"Description": "Region location of the template resources",
"Default": "eu-west-1",
"AllowedValues" : [ "us-east-1", "us-west-1", "us-west-2", "eu-west-1" ]
},
"SecurityGroupIds": {
"Description": "Security groups that can be used to access the EC2 instances, do not select more than 5 SG",
"Type": "List<AWS::EC2::SecurityGroup::Id>",
"ConstraintDescription": "must be list of EC2 security group ids"
},
"VpcId": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VPC associated with the provided subnets",
"Default": "vpc-69e3320c",
"ConstraintDescription": "must be an existing VPC ID"
},
"SubnetId": {
"Type": "String",
"Default": "subnet-6820eb31",
"ConstraintDescription": "must be an existing subnet ID"
},
"InstanceType": {
"Type": "String",
"Default": "c3.large",
"AllowedValues" : ["t2.micro", "m1.small", "m1.large","m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large" , "c4.xlarge" ,"c4.2xlarge" , "c4.4xlarge","c4.8xlarge" , "c3.large" , "c3.xlarge", "c3.2xlarge", "c3.4xlarge" ,"c3.8xlarge"],
"ConstraintDescription": "must be a valid EC2 instance type"
},
"KeyPairName": {
"Type": "AWS::EC2::KeyPair::KeyName",
"Default": "test-generic-ec2",
"ConstraintDescription": "must be the name of an existing EC2 KeyPair"
},
"CookbookS3" : {
"Type": "String",
"Default": "https://s3-eu-west-1.amazonaws.com/mybucket.test.cookbooks/cookbook-v1.tar.gz",
"ConstraintDescription": "the Url to the cookbook"
},
"CookbookS3AccessID": {
"Type": "String",
"ConstraintDescription": "username to the appropriate IAM access key ID"
},
"CookbookS3AccessKey": {
"Type": "String",
"NoEcho" : "true",
"ConstraintDescription": "password to the appropriate IAM secret access key"
},
"MountPoint" : {
"Description" : "The Linux mount point for the EFS volume",
"Type": "String",
"MinLength": "1",
"Default": "efs-file-appli-tmp"
},
"MountPointDNS" : {
"Description" : "Mount target DNS name",
"Type" : "String",
"Default" : "eu-west-1a.fs-c2388dc0b.efs.eu-west-1.amazonaws.com"
},
"FileSystem" : {
"Description" :"The Id of the FileSystem",
"Type": "String",
"Default": "fs-c2388dc0b"
},
"NewRelicLicence" : {
"Description": "The licence key of newrelic",
"Type": "String"
},
"Environnement" : {
"Description": "The Environnement variable ",
"Type": "String",
"Default": "test",
"AllowedValues" : ["dev", "test", "int", "prod"]
}
},
"Conditions" : {
"CreateProdResources" : { "Fn::Not" : [{ "Fn::Equals" : [ {"Ref" : "Environnement"},"test" ] }] }
},
"Resources": {
"MyStack": {
"Type": "AWS::OpsWorks::Stack",
"Properties": {
"AgentVersion" : "LATEST",
"Name": { "Ref": "AWS::StackName" },
"Attributes": { "Color": { "Ref": "OpsWorksStackColor" } },
"ChefConfiguration": {},
"ConfigurationManager": { "Name": "Chef", "Version": "12" },
"CustomCookbooksSource": {
"Type": "s3",
"Password" : { "Ref": "CookbookS3AccessKey" },
"Username" : { "Ref": "CookbookS3AccessID" },
"Url": { "Ref": "CookbookS3" }
},
"CustomJson": {
"mount" : { "mountdir" : {
"default" : { "dirname" : { "Ref" : "MountPoint" } } }
},
"mountadd" : { "mountdns" : {
"default" : { "mdns" : { "Ref" : "MountPointDNS" }}}
}
},
"DefaultInstanceProfileArn": { "Fn::GetAtt": [ "OpsWorksInstanceProfile","Arn" ] },
"DefaultOs": "Ubuntu 14.04 LTS",
"DefaultRootDeviceType": "ebs",
"DefaultSshKeyName": { "Ref": "KeyPairName" },
"DefaultSubnetId" : {"Ref" : "SubnetId" },
"ServiceRoleArn": { "Fn::GetAtt": ["OpsWorksServiceRole", "Arn"] },
"UseCustomCookbooks": true,
"UseOpsworksSecurityGroups" : true,
"VpcId" : { "Ref" : "VpcId" }
}
},
"MyLayer": {
"Type": "AWS::OpsWorks::Layer",
"DependsOn" : "OpsWorksServiceRole",
"Properties": {
"AutoAssignElasticIps" : false,
"AutoAssignPublicIps" : true,
"CustomRecipes" : {
"Setup" : ["cassandra-php-driver::setup","awscli::setup","crontab::setup","prometheus-server::setup","awslogs::setup","newrelic::php_agent","settings::setup"],
"Configure" : ["cassandra-php-driver::configure","security::configure","settings::default"],
"Deploy": ["imports::deploy"]
},
"CustomSecurityGroupIds" : { "Ref" : "SecurityGroupIds" },
"EnableAutoHealing" : true,
"InstallUpdatesOnBoot": false,
"LifecycleEventConfiguration": {
"ShutdownEventConfiguration": {
"DelayUntilElbConnectionsDrained": false,
"ExecutionTimeout": 120 }
},
"Name": "script-node",
"Shortname" : "node",
"StackId": { "Ref": "MyStack" },
"Type": "custom",
"UseEbsOptimizedInstances": true,
"VolumeConfigurations": [ {
"Iops": 10000,
"MountPoint": "/dev/sda1",
"NumberOfDisks": 1,
"Size": 20,
"VolumeType": "gp2"
}]
}
},
"OpsWorksServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Principal": { "Service": [ { "Fn::FindInMap": [ "Region2Principal",{ "Ref": "AWS::Region" },"OpsWorksPrincipal" ] } ] },
"Action" : [ "sts:AssumeRole" ]
} ]
},
"Path": "/",
"Policies": [ {
"PolicyName": "opsworks-service",
"PolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Action": "*",
"Resource": "*"
} ]
}
} ]
}
},
"OpsWorksInstanceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Principal": { "Service" : [ { "Fn::FindInMap": [ "Region2Principal", { "Ref": "AWS::Region" },"EC2Principal" ] } ] },
"Action" : [ "sts:AssumeRole" ]
}]
},
"Path": "/",
"Policies": [{
"PolicyName": "aws-opsworks-instance",
"PolicyDocument": {
"Statement": [{
"Effect": "Allow",
"Action": "*",
"Resource": "*"
}]
}
}]
}
},
"OpsWorksInstanceProfile": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Path": "/",
"Roles": [ { "Ref": "OpsWorksInstanceRole" } ]
}
},
"MyInstance": {
"Type": "AWS::OpsWorks::Instance",
"Properties": {
"Hostname": "Script",
"RootDeviceType": "ebs",
"StackId": {"Ref": "MyStack"},
"LayerIds": [{"Ref": "MyLayer"}],
"InstanceType": {"Ref" : "InstanceType"}
}
},
"MyApp": {
"Type": "AWS::OpsWorks::App",
"Properties": {
"AppSource" : {
"Type" : "git",
"Url" : "git://github.com:globlW/My-imports.git",
"Revision" : "develop"
},
"Description": "Dataimport and connectors",
"Name" : "app-Imports",
"Shortname" : "app_imports",
"StackId" : {"Ref": "MyStack"},
"Type" : "other"
}
},
"MyFileSystem" : {
"Type" : "AWS::EFS::FileSystem",
"Condition" : "CreateProdResources",
"Properties" : {
"FileSystemTags" : [{
"Key" : "Name",
"Value" : {"Ref" : "MountPoint"}
}]
}
},
"MountTarget": {
"Type": "AWS::EFS::MountTarget",
"Condition" : "CreateProdResources",
"Properties": {
"FileSystemId": { "Ref": "MyFileSystem" },
"SubnetId": { "Ref": "SubnetId" },
"SecurityGroups": [ { "Ref": "SecurityGroupIds" } ]
}
}
}
}
As you see, I use often the { "Ref" : "FileSystem" } , { "Ref" : "MountPointDNS" } and { "Ref" : "MountPoint" }, I specially need them in the CustomJson. The problem is how to link the parameters with the recent created resources and how to name them ? can I give the same name so that I can always get a value to customjson ?
I hope my issue is clear.
Thank you
It is possible to conditionally use an existing resource using Condition Functions. See the Conditionally use an existing resource example in the documentation, which demonstrates the following pattern:
Parameters:
ExistingResource:
Description: An existing resource (optional).
Default: NONE
Type: String
Conditions:
CreateNewResource: !Equals [!Ref ExistingResource, NONE]
Resources:
NewResource:
Condition: CreateNewResource
Type: # Resource type
Properties: # Resource properties
Outputs:
ResourceId:
Description: Reference to either a newly-created or existing resource.
Value: !If [CreateNewResource, !Ref NewResource, !Ref ExistingResource]

Set an environment variables in the custom json in opsworks

I am working on a AWS cloudformation template that creates an opsworks stack.
In my stack, I will have one instance (Ubuntu 14.04) , all the applications and packages that will be installed in the instance are configured in the chef cookbook provided from an S3 bucket. I need to set an environment variable in my instance and its value is to be set by a parameter of the cloudformation template, this is the template:
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "AWS CloudFormation Template : Stack OpsWorks Slave pour deployer les instances script",
"Mappings": {
"Region2Principal": {
"us-east-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"us-west-2": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"us-west-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
},
"eu-west-1": {
"EC2Principal": "ec2.amazonaws.com",
"OpsWorksPrincipal": "opsworks.amazonaws.com"
}
},
"AWSInstanceType2Arch" : {
"t1.micro" : { "Arch" : "PV64" },
"t2.micro" : { "Arch" : "HVM64" },
"m1.small" : { "Arch" : "PV64" },
"m1.large" : { "Arch" : "PV64" },
"m4.large" : { "Arch" : "HVM64" },
"m4.xlarge" : { "Arch" : "HVM64" },
"m4.2xlarge" : { "Arch" : "HVM64" },
"m4.4xlarge" : { "Arch" : "HVM64" },
"m4.10xlarge" : { "Arch" : "HVM64" },
"m4.16xlarge" : { "Arch" : "HVM64" },
"c3.large" : { "Arch" : "HVM64" },
"c3.xlarge" : { "Arch" : "HVM64" },
"c3.2xlarge" : { "Arch" : "HVM64" },
"c3.4xlarge" : { "Arch" : "HVM64" },
"c3.8xlarge" : { "Arch" : "HVM64" },
"c4.large" : { "Arch" : "HVM64" },
"c4.xlarge" : { "Arch" : "HVM64" },
"c4.2xlarge" : { "Arch" : "HVM64" },
"c4.4xlarge" : { "Arch" : "HVM64" },
"c4.8xlarge" : { "Arch" : "HVM64" }
},
"AWSRegionArch2AMI" : {
"us-east-1" : { "PV64" : "ami-5fb8c835", "HVM64" : "ami-60b6c60a" },
"us-west-1" : { "PV64" : "ami-56ea8636", "HVM64" : "ami-d5ea86b5" },
"eu-west-1" : { "PV64" : "ami-95e33ce6", "HVM64" : "ami-bff32ccc" },
"us-west-2" : { "PV64" : "ami-d93622b8", "HVM64" : "ami-f0091d91" }
}
},
"Parameters": {
"OpsWorksStackColor": {
"Description": "RGB Color to use for OpsWorks Stack",
"Type": "String",
"Default": "rgb(38, 146, 168)"
},
"Region" : {
"Type":"String",
"Description": "Region location of the template resources",
"Default": "eu-west-1",
"AllowedValues" : [ "us-east-1", "us-west-1", "us-west-2", "eu-west-1" ]
},
"SecurityGroupIds": {
"Description": "Security groups that can be used to access the EC2 instances, do not select more than 5 SG",
"Type": "List<AWS::EC2::SecurityGroup::Id>",
"ConstraintDescription": "must be list of EC2 security group ids"
},
"VpcId": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VPC associated with the provided subnets",
"Default": "vpc-69e3320c",
"ConstraintDescription": "must be an existing VPC ID"
},
"SubnetId": {
"Type": "String",
"Default": "subnet-6820eb31",
"ConstraintDescription": "must be an existing subnet ID"
},
"InstanceType": {
"Type": "String",
"Default": "c3.large",
"AllowedValues" : ["t2.micro", "m1.small", "m1.large","m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large" , "c4.xlarge" ,"c4.2xlarge" , "c4.4xlarge","c4.8xlarge" , "c3.large" , "c3.xlarge", "c3.2xlarge", "c3.4xlarge" ,"c3.8xlarge"],
"ConstraintDescription": "must be a valid EC2 instance type"
},
"KeyPairName": {
"Type": "AWS::EC2::KeyPair::KeyName",
"Default": "test-generic-ec2",
"ConstraintDescription": "must be the name of an existing EC2 KeyPair"
},
"CookbookS3" : {
"Type": "String",
"Default": "https://s3-eu-west-1.amazonaws.com/MybucketJRBLO.cookbooks/cookbook-v2.tar.gz",
"ConstraintDescription": "the Url to the cookbook"
},
"CookbookS3AccessID": {
"Type": "String",
"ConstraintDescription": "username to the appropriate IAM access key ID"
},
"CookbookS3AccessKey": {
"Type": "String",
"NoEcho" : "true",
"ConstraintDescription": "password to the appropriate IAM secret access key"
},
"Environnement" : {
"Description": "The Environnement variable ",
"Type": "String",
"Default": "test",
"AllowedValues" : ["dev", "test", "int", "prod"]
}
},
"Conditions" : {
"CreateProdResources" : { "Fn::Not" : [{ "Fn::Equals" : [ {"Ref" : "Environnement"},"test" ] }] }
},
"Resources": {
"MabStack": {
"Type": "AWS::OpsWorks::Stack",
"Properties": {
"AgentVersion" : "LATEST",
"Name": { "Ref": "AWS::StackName" },
"Attributes": { "Color": { "Ref": "OpsWorksStackColor" } },
"ChefConfiguration": {},
"ConfigurationManager": { "Name": "Chef", "Version": "12" },
"CustomCookbooksSource": {
"Type": "s3",
"Password" : { "Ref": "CookbookS3AccessKey" },
"Username" : { "Ref": "CookbookS3AccessID" },
"Url": { "Ref": "CookbookS3" }
},
"CustomJson": { "awscli": { "profils": {
"default": {"role_arn": { "Fn::GetAtt": [ "OpsWorksInstanceProfile", "Arn" ] } }
}
}
,
"chef_environment": {"Ref" : "Environnement"}
},
"DefaultInstanceProfileArn": { "Fn::GetAtt": [ "OpsWorksInstanceProfile","Arn" ] },
"DefaultOs": "Ubuntu 14.04 LTS",
"DefaultRootDeviceType": "ebs",
"DefaultSshKeyName": { "Ref": "KeyPairName" },
"DefaultSubnetId" : {"Ref" : "SubnetId" },
"ServiceRoleArn": { "Fn::GetAtt": ["OpsWorksServiceRole", "Arn"] },
"UseCustomCookbooks": true,
"UseOpsworksSecurityGroups" : true,
"VpcId" : { "Ref" : "VpcId" }
}
},
"MabLayer": {
"Type": "AWS::OpsWorks::Layer",
"DependsOn" : "OpsWorksServiceRole",
"Properties": {
"AutoAssignElasticIps" : false,
"AutoAssignPublicIps" : true,
"CustomRecipes" : {
"Configure" : ["cassandra-php-driver::configure"],
"Setup" : ["cassandra-php-driver::setup", "security::setup"]
},
"CustomSecurityGroupIds" : { "Ref" : "SecurityGroupIds" },
"EnableAutoHealing" : true,
"InstallUpdatesOnBoot": false,
"LifecycleEventConfiguration": {
"ShutdownEventConfiguration": {
"DelayUntilElbConnectionsDrained": false,
"ExecutionTimeout": 120 }
},
"Name": "script-node",
"Shortname" : "node",
"StackId": { "Ref": "MabStack" },
"Type": "custom",
"UseEbsOptimizedInstances": true,
"VolumeConfigurations": [ {
"Iops": 10000,
"MountPoint": "/dev/sda1",
"NumberOfDisks": 1,
"Size": 20,
"VolumeType": "gp2"
}]
}
},
"OpsWorksServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Principal": { "Service": [ { "Fn::FindInMap": [ "Region2Principal",{ "Ref": "AWS::Region" },"OpsWorksPrincipal" ] } ] },
"Action" : [ "sts:AssumeRole" ]
} ]
},
"Path": "/",
"Policies": [ {
"PolicyName": "opsworks-service",
"PolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Action": "*",
"Resource": "*"
} ]
}
} ]
}
},
"OpsWorksInstanceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement": [ {
"Effect": "Allow",
"Principal": { "Service" : [ { "Fn::FindInMap": [ "Region2Principal", { "Ref": "AWS::Region" },"EC2Principal" ] } ] },
"Action" : [ "sts:AssumeRole" ]
}]
},
"Path": "/",
"Policies": [{
"PolicyName": "aws-opsworks-instance",
"PolicyDocument": {
"Statement": [{
"Effect": "Allow",
"Action": "*",
"Resource": "*"
}]
}
}]
}
},
"OpsWorksInstanceProfile": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Path": "/",
"Roles": [ { "Ref": "OpsWorksInstanceRole" } ]
}
},
"MabInstance": {
"Type": "AWS::OpsWorks::Instance",
"Properties": {
"Hostname": "Script",
"RootDeviceType": "ebs",
"StackId": {"Ref": "MabStack"},
"LayerIds": [{"Ref": "MabLayer"}],
"InstanceType": {"Ref" : "InstanceType"}
}
},
"MabApp": {
"Type": "AWS::OpsWorks::App",
"Properties": {
"AppSource" : {
"Type" : "git",
"Url" : "git://github.com:soumab/JRLapplication.git",
"Revision" : "develop"
},
"Description": "Dataimport and connectors",
"Name" : "JRL-App",
"Shortname" : "JRL_app",
"StackId" : {"Ref": "MabStack"},
"Type" : "other"
}
},
}
}
I want my instance to have this environment variable :
export $Apps_ENV = "test" / "int" / "prod"
so the value depends on what the user will select. How to set this variable ? Is it possible through CustomJson ? Or directly in the cookbook ? please help. Thank you.
I successfly acheived what I wanted by the following settings:
I added this to CustomJson in my template :
"settings" : { "var" : {
"default": { "Apps_ENV": {"Ref" : "Environnement"} } }
}
And I added a cookbook as following
settings/recipes/defaults.rb :
template '/etc/environment' do
source 'env.erb'
owner 'root'
group 'root'
mode '0755'
end
and settings/templates/default/env.erb:
<% node['settings']['var'].each do |var_name, var_att| %>
<%= sprintf("[%s]", var_name) %>
<% var_att.each do |attr_name, attr_value| %>
<%= sprintf("%s=%s", attr_name, attr_value) %>
<% end
end %>