{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"VpcId": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VpcId of your existing Virtual Private Cloud (VPC)",
"ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud."
},
"Subnets": {
"Type": "List<AWS::EC2::Subnet::Id>",
"Description": "The list of SubnetIds in your Virtual Private Cloud (VPC)"
},
"InstanceType": {
"Description": "WebServer EC2 instance type",
"Type": "String",
"Default": "t2.small",
"AllowedValues": [
"t1.micro",
"t2.nano",
"t2.micro",
"t2.small",
"t2.medium",
"t2.large",
"m1.small",
"m1.medium",
"cg1.4xlarge"
],
"ConstraintDescription": "must be a valid EC2 instance type."
},
"WebServerCapacity": {
"Default": "2",
"Description": "The initial number of WebServer instances",
"Type": "Number",
"MinValue": "1",
"MaxValue": "10",
"ConstraintDescription": "must be between 1 and 10 EC2 instances."
},
"KeyName": {
"Description": "The EC2 Key Pair to allow SSH access to the instances",
"Type": "AWS::EC2::KeyPair::KeyName",
"ConstraintDescription": "must be the name of an existing EC2 KeyPair."
},
"SSHLocation": {
"Description": "The IP address range that can be used to SSH to the EC2 instances",
"Type": "String",
"MinLength": "9",
"MaxLength": "18",
"Default": "0.0.0.0/0",
"AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
"ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
}
},
"Resources": {
"WebServerScaleUpPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "WebServerGroup"
},
"Cooldown": "60",
"ScalingAdjustment": 1
}
},
"WebServerScaleDownPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "WebServerGroup"
},
"Cooldown": "60",
"ScalingAdjustment": -1
}
},
"CPUAlarmHigh": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmDescription": "Scale-up if CPU > 70% for 5 minutes",
"MetricName": "CPUUtilization",
"Namespace": "AWS/EC2",
"Statistic": "Average",
"Period": 300,
"EvaluationPeriods": 2,
"Threshold": 70,
"AlarmActions": [{
"Ref": "WebServerScaleUpPolicy"
}],
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "WebServerGroup"
}
}],
"ComparisonOperator": "GreaterThanThreshold"
}
},
"CPUAlarmLow": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmDescription": "Scale-down if CPU < 40% for 5 minutes",
"MetricName": "CPUUtilization",
"Namespace": "AWS/EC2",
"Statistic": "Average",
"Period": 300,
"EvaluationPeriods": 2,
"Threshold": 40,
"AlarmActions": [{
"Ref": "WebServerScaleDownPolicy"
}],
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "WebServerGroup"
}
}],
"ComparisonOperator": "LessThanThreshold"
}
},
"ApplicationLoadBalancer": {
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties": {
"Name": "elb-test",
"Scheme": "internet-facing",
"IpAddressType": "ipv4",
"Type": "application",
"Subnets": {
"Ref": "Subnets"
}
}
},
"ALBListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": {
"Ref": "ALBTargetGroup"
}
}],
"LoadBalancerArn": {
"Ref": "ApplicationLoadBalancer"
},
"Port": 80,
"Protocol": "HTTP"
}
},
"ALBTargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": "ELB-Group",
"HealthCheckIntervalSeconds": 30,
"HealthCheckTimeoutSeconds": 5,
"HealthyThresholdCount": 3,
"Port": 80,
"Protocol": "HTTP",
"TargetType": "instance",
"UnhealthyThresholdCount": 5,
"VpcId": {
"Ref": "VpcId"
}
}
},
"WebServerGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"VPCZoneIdentifier": {
"Ref": "Subnets"
},
"HealthCheckGracePeriod": 300,
"LaunchConfigurationName": {
"Ref": "LaunchConfig"
},
"MinSize": "1",
"MaxSize": "8",
"DesiredCapacity": {
"Ref": "WebServerCapacity"
},
"TargetGroupARNs": [{
"Ref": "ALBTargetGroup"
}]
},
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT5M",
"Count": {
"Ref": "WebServerCapacity"
}
}
},
"UpdatePolicy": {
"AutoScalingRollingUpdate": {
"MinInstancesInService": 1,
"MaxBatchSize": 1,
"PauseTime": "PT5M",
"WaitOnResourceSignals": true
}
}
},
"LaunchConfig": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"KeyName": {
"Ref": "KeyName"
},
"ImageId": "ami-00932e4c143f3fdf0",
"SecurityGroups": [{
"Ref": "InstanceSecurityGroup"
}],
"InstanceType": {
"Ref": "InstanceType"
},
"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
"#!/bin/bash -xe\n",
"apt-get update -y\n",
"apt-get install -y python-setuptools\n",
"mkdir -p /opt/aws/bin\n",
"python /usr/lib/python2.7/dist-packages/easy_install.py --script-dir /opt/aws/bin https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz\n",
"/opt/aws/bin/cfn-init -v ",
" --stack ", { "Ref" : "AWS::StackName" },
" --resource EC2Instance ",
" --configsets full_install ",
" --region ", { "Ref" : "AWS::Region" }, "\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ", { "Ref" : "AWS::StackName" },
" --resource EC2Instance ",
" --region ", { "Ref" : "AWS::Region" }, "\n"
]]}}}
},
"InstanceSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Enable SSH access and HTTP from the load balancer only",
"SecurityGroupIngress": [{
"IpProtocol": "tcp",
"FromPort": 22,
"ToPort": 22,
"CidrIp": {
"Ref": "SSHLocation"
}
},
{
"IpProtocol": "tcp",
"FromPort": 80,
"ToPort": 80,
"SourceSecurityGroupId": {
"Fn::Select": [
0,
{
"Fn::GetAtt": [
"ApplicationLoadBalancer",
"SecurityGroups"
]
}
]
}
}
],
"VpcId": {
"Ref": "VpcId"
}
}
}
},
"Outputs": {
"URL": {
"Description": "The URL of the website",
"Value": {
"Fn::Join": [
"",
[
"http://",
{
"Fn::GetAtt": [
"ApplicationLoadBalancer",
"DNSName"
]
}
]
]
}
}
}
}
I am using this template to create auto-scaling with cloud formation and i am using ubuntu-18.04. Every time I am getting same error.
Received 0 SUCCESS signal(s) out of 1. Unable to satisfy 100% MinSuccessfulInstancesPercent requirement
Failed to receive 1 resource signal(s) for the current batch. Each resource signal timeout is counted as a FAILURE.
Please let me know where i am lacking
I have ran your template through cfn-lint and got a lot of problems reported:
W2030 You must specify a valid allowed value for InstanceType (cg1.4xlarge).
Valid values are ['a1.2xlarge', 'a1.4xlarge', 'a1.large', 'a1.medium', 'a1.metal', 'a1.xlarge', 'c1.medium', 'c1.xlarge', 'c3.2xlarge', 'c3.4xlarge', 'c3.8xlarge', 'c3.large', 'c3.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', 'c4.large', 'c4.xlarge', 'c5.12xlarge', 'c5.18xlarge', 'c5.24xlarge', 'c5.2xlarge', 'c5.4xlarge', 'c5.9xlarge', 'c5.large', 'c5.metal', 'c5.xlarge', 'c5d.12xlarge', 'c5d.18xlarge', 'c5d.24xlarge', 'c5d.2xlarge', 'c5d.4xlarge', 'c5d.9xlarge', 'c5d.large', 'c5d.metal', 'c5d.xlarge', 'c5n.18xlarge', 'c5n.2xlarge', 'c5n.4xlarge', 'c5n.9xlarge', 'c5n.large', 'c5n.metal', 'c5n.xlarge', 'cc2.8xlarge', 'cr1.8xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', 'd2.xlarge', 'f1.16xlarge', 'f1.2xlarge', 'f1.4xlarge', 'g2.2xlarge', 'g2.8xlarge', 'g3.16xlarge', 'g3.4xlarge', 'g3.8xlarge', 'g3s.xlarge', 'g4dn.12xlarge', 'g4dn.16xlarge', 'g4dn.2xlarge', 'g4dn.4xlarge', 'g4dn.8xlarge', 'g4dn.metal', 'g4dn.xlarge', 'h1.16xlarge', 'h1.2xlarge', 'h1.4xlarge', 'h1.8xlarge', 'hs1.8xlarge', 'i2.2xlarge', 'i2.4xlarge', 'i2.8xlarge', 'i2.xlarge', 'i3.16xlarge', 'i3.2xlarge', 'i3.4xlarge', 'i3.8xlarge', 'i3.large', 'i3.metal', 'i3.xlarge', 'i3en.12xlarge', 'i3en.24xlarge', 'i3en.2xlarge', 'i3en.3xlarge', 'i3en.6xlarge', 'i3en.large', 'i3en.metal', 'i3en.xlarge', 'm1.large', 'm1.medium', 'm1.small', 'm1.xlarge', 'm2.2xlarge', 'm2.4xlarge', 'm2.xlarge', 'm3.2xlarge', 'm3.large', 'm3.medium', 'm3.xlarge', 'm4.10xlarge', 'm4.16xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.large', 'm4.xlarge', 'm5.12xlarge', 'm5.16xlarge', 'm5.24xlarge', 'm5.2xlarge', 'm5.4xlarge', 'm5.8xlarge', 'm5.large', 'm5.metal', 'm5.xlarge', 'm5a.12xlarge', 'm5a.16xlarge', 'm5a.24xlarge', 'm5a.2xlarge', 'm5a.4xlarge', 'm5a.8xlarge', 'm5a.large', 'm5a.xlarge', 'm5ad.12xlarge', 'm5ad.24xlarge', 'm5ad.2xlarge', 'm5ad.4xlarge', 'm5ad.large', 'm5ad.xlarge', 'm5d.12xlarge', 'm5d.16xlarge', 'm5d.24xlarge', 'm5d.2xlarge', 'm5d.4xlarge', 'm5d.8xlarge', 'm5d.large', 'm5d.metal', 'm5d.xlarge', 'm5dn.12xlarge', 'm5dn.16xlarge', 'm5dn.24xlarge', 'm5dn.2xlarge', 'm5dn.4xlarge', 'm5dn.8xlarge', 'm5dn.large', 'm5dn.metal', 'm5dn.xlarge', 'm5n.12xlarge', 'm5n.16xlarge', 'm5n.24xlarge', 'm5n.2xlarge', 'm5n.4xlarge', 'm5n.8xlarge', 'm5n.large', 'm5n.metal', 'm5n.xlarge', 'p2.16xlarge', 'p2.8xlarge', 'p2.xlarge', 'p3.16xlarge', 'p3.2xlarge', 'p3.8xlarge', 'p3dn.24xlarge', 'r3.2xlarge', 'r3.4xlarge', 'r3.8xlarge', 'r3.large', 'r3.xlarge', 'r4.16xlarge', 'r4.2xlarge', 'r4.4xlarge', 'r4.8xlarge', 'r4.large', 'r4.xlarge', 'r5.12xlarge', 'r5.16xlarge', 'r5.24xlarge', 'r5.2xlarge', 'r5.4xlarge', 'r5.8xlarge', 'r5.large', 'r5.metal', 'r5.xlarge', 'r5a.12xlarge', 'r5a.16xlarge', 'r5a.24xlarge', 'r5a.2xlarge', 'r5a.4xlarge', 'r5a.8xlarge', 'r5a.large', 'r5a.xlarge', 'r5ad.12xlarge', 'r5ad.24xlarge', 'r5ad.2xlarge', 'r5ad.4xlarge', 'r5ad.large', 'r5ad.xlarge', 'r5d.12xlarge', 'r5d.16xlarge', 'r5d.24xlarge', 'r5d.2xlarge', 'r5d.4xlarge', 'r5d.8xlarge', 'r5d.large', 'r5d.metal', 'r5d.xlarge', 'r5dn.12xlarge', 'r5dn.16xlarge', 'r5dn.24xlarge', 'r5dn.2xlarge', 'r5dn.4xlarge', 'r5dn.8xlarge', 'r5dn.large', 'r5dn.metal', 'r5dn.xlarge', 'r5n.12xlarge', 'r5n.16xlarge', 'r5n.24xlarge', 'r5n.2xlarge', 'r5n.4xlarge', 'r5n.8xlarge', 'r5n.large', 'r5n.metal', 'r5n.xlarge', 't1.micro', 't2.2xlarge', 't2.large', 't2.medium', 't2.micro', 't2.nano', 't2.small', 't2.xlarge', 't3.2xlarge', 't3.large', 't3.medium', 't3.micro', 't3.nano', 't3.small', 't3.xlarge', 't3a.2xlarge', 't3a.large', 't3a.medium', 't3a.micro', 't3a.nano', 't3a.small', 't3a.xlarge', 'u-18tb1.metal', 'u-24tb1.metal', 'x1.16xlarge', 'x1.32xlarge', 'x1e.16xlarge', 'x1e.2xlarge', 'x1e.32xlarge', 'x1e.4xlarge', 'x1e.8xlarge', 'x1e.xlarge', 'z1d.12xlarge', 'z1d.2xlarge', 'z1d.3xlarge', 'z1d.6xlarge', 'z1d.large', 'z1d.metal', 'z1d.xlarge']
so.template:27:17
W7001 Mapping 'AWSInstanceType2Arch' is defined but not used
so.template:55:9
W7001 Mapping 'AWSInstanceType2NATArch' is defined but not used
so.template:87:9
E3012 Property Resources/WebServerScaleUpPolicy/Properties/ScalingAdjustment should be of type Integer
so.template:120:17
E3012 Property Resources/WebServerScaleDownPolicy/Properties/ScalingAdjustment should be of type Integer
so.template:131:17
E3012 Property Resources/CPUAlarmHigh/Properties/Period should be of type Integer
so.template:141:17
E3012 Property Resources/CPUAlarmHigh/Properties/EvaluationPeriods should be of type Integer
so.template:142:17
E3012 Property Resources/CPUAlarmHigh/Properties/Threshold should be of type Double
so.template:143:17
E3012 Property Resources/CPUAlarmLow/Properties/Period should be of type Integer
so.template:163:17
E3012 Property Resources/CPUAlarmLow/Properties/EvaluationPeriods should be of type Integer
so.template:164:17
E3012 Property Resources/CPUAlarmLow/Properties/Threshold should be of type Double
so.template:165:17
E3012 Property Resources/ALBListener/Properties/Port should be of type Integer
so.template:202:17
E3002 Invalid Property Resources/ALBTargetGroup/Properties/HealthCheckType
so.template:217:17
E3016 Value for MinInstancesInService must be of type Integer
so.template:251:11
E3016 Value for MaxBatchSize must be of type Integer
so.template:252:11
E3016 Value for WaitOnResourceSignals must be of type Boolean
so.template:254:11
E3012 Property Resources/InstanceSecurityGroup/Properties/SecurityGroupIngress/0/FromPort should be of type Integer
so.template:280:25
E3012 Property Resources/InstanceSecurityGroup/Properties/SecurityGroupIngress/0/ToPort should be of type Integer
so.template:281:25
E3012 Property Resources/InstanceSecurityGroup/Properties/SecurityGroupIngress/1/FromPort should be of type Integer
so.template:288:25
E3012 Property Resources/InstanceSecurityGroup/Properties/SecurityGroupIngress/1/ToPort should be of type Integer
so.template:289:25
I'd suggest you fix these issues first.
This is coming down to HealthCheckType being in the target group resource, it should instead be attached to your autoscaling group.
The fixed template for this error is below
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"VpcId": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VpcId of your existing Virtual Private Cloud (VPC)",
"ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud."
},
"Subnets": {
"Type": "List<AWS::EC2::Subnet::Id>",
"Description": "The list of SubnetIds in your Virtual Private Cloud (VPC)"
},
"InstanceType": {
"Description": "WebServer EC2 instance type",
"Type": "String",
"Default": "t2.small",
"AllowedValues": [
"t1.micro",
"t2.nano",
"t2.micro",
"t2.small",
"t2.medium",
"t2.large",
"m1.small",
"m1.medium",
"cg1.4xlarge"
],
"ConstraintDescription": "must be a valid EC2 instance type."
},
"WebServerCapacity": {
"Default": "2",
"Description": "The initial number of WebServer instances",
"Type": "Number",
"MinValue": "1",
"MaxValue": "10",
"ConstraintDescription": "must be between 1 and 10 EC2 instances."
},
"KeyName": {
"Description": "The EC2 Key Pair to allow SSH access to the instances",
"Type": "AWS::EC2::KeyPair::KeyName",
"ConstraintDescription": "must be the name of an existing EC2 KeyPair."
},
"SSHLocation": {
"Description": "The IP address range that can be used to SSH to the EC2 instances",
"Type": "String",
"MinLength": "9",
"MaxLength": "18",
"Default": "0.0.0.0/0",
"AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
"ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
}
},
"Mappings": {
"AWSInstanceType2Arch": {
"t1.micro": {
"Arch": "HVM64"
},
"t2.nano": {
"Arch": "HVM64"
},
"t2.micro": {
"Arch": "HVM64"
},
"t2.small": {
"Arch": "HVM64"
},
"t2.medium": {
"Arch": "HVM64"
},
"t2.large": {
"Arch": "HVM64"
},
"m1.small": {
"Arch": "HVM64"
},
"m1.medium": {
"Arch": "HVM64"
},
"m1.large": {
"Arch": "HVM64"
},
"d2.xlarge": {
"Arch": "HVM64"
}
},
"AWSInstanceType2NATArch": {
"t1.micro": {
"Arch": "NATHVM64"
},
"t2.nano": {
"Arch": "NATHVM64"
},
"t2.micro": {
"Arch": "NATHVM64"
},
"t2.small": {
"Arch": "NATHVM64"
},
"t2.medium": {
"Arch": "NATHVM64"
},
"t2.large": {
"Arch": "NATHVM64"
},
"m1.small": {
"Arch": "NATHVM64"
}
}
},
"Resources": {
"WebServerScaleUpPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "WebServerGroup"
},
"Cooldown": "60",
"ScalingAdjustment": "1"
}
},
"WebServerScaleDownPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "WebServerGroup"
},
"Cooldown": "60",
"ScalingAdjustment": "-1"
}
},
"CPUAlarmHigh": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmDescription": "Scale-up if CPU > 70% for 5 minutes",
"MetricName": "CPUUtilization",
"Namespace": "AWS/EC2",
"Statistic": "Average",
"Period": "300",
"EvaluationPeriods": "2",
"Threshold": "70",
"AlarmActions": [{
"Ref": "WebServerScaleUpPolicy"
}],
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "WebServerGroup"
}
}],
"ComparisonOperator": "GreaterThanThreshold"
}
},
"CPUAlarmLow": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmDescription": "Scale-down if CPU < 40% for 5 minutes",
"MetricName": "CPUUtilization",
"Namespace": "AWS/EC2",
"Statistic": "Average",
"Period": "300",
"EvaluationPeriods": "2",
"Threshold": "40",
"AlarmActions": [{
"Ref": "WebServerScaleDownPolicy"
}],
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "WebServerGroup"
}
}],
"ComparisonOperator": "LessThanThreshold"
}
},
"ApplicationLoadBalancer": {
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties": {
"Name": "elb-test",
"Scheme": "internet-facing",
"IpAddressType": "ipv4",
"Type": "application",
"Subnets": {
"Ref": "Subnets"
}
}
},
"ALBListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": {
"Ref": "ALBTargetGroup"
}
}],
"LoadBalancerArn": {
"Ref": "ApplicationLoadBalancer"
},
"Port": "80",
"Protocol": "HTTP"
}
},
"ALBTargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": "ELB-Group",
"HealthCheckIntervalSeconds": 30,
"HealthCheckTimeoutSeconds": 5,
"HealthyThresholdCount": 3,
"Port": 80,
"Protocol": "HTTP",
"TargetType": "instance",
"UnhealthyThresholdCount": 5,
"VpcId": {
"Ref": "VpcId"
}
}
},
"WebServerGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"VPCZoneIdentifier": {
"Ref": "Subnets"
},
"HealthCheckType": "ELB",
"HealthCheckGracePeriod": 300
"LaunchConfigurationName": {
"Ref": "LaunchConfig"
},
"MinSize": "1",
"MaxSize": "8",
"DesiredCapacity": {
"Ref": "WebServerCapacity"
},
"TargetGroupARNs": [{
"Ref": "ALBTargetGroup"
}]
},
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT5M",
"Count": {
"Ref": "WebServerCapacity"
}
}
},
"UpdatePolicy": {
"AutoScalingRollingUpdate": {
"MinInstancesInService": "1",
"MaxBatchSize": "1",
"PauseTime": "PT5M",
"WaitOnResourceSignals": "true"
}
}
},
"LaunchConfig": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"KeyName": {
"Ref": "KeyName"
},
"ImageId": "ami-00932e4c143f3fdf0",
"SecurityGroups": [{
"Ref": "InstanceSecurityGroup"
}],
"InstanceType": {
"Ref": "InstanceType"
},
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -x\n",
"# Install the files and packages from the metadata\n",
"/opt/aws/bin/cfn-init -v ",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource MyInstance ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n",
"# Signal the status from cfn-init\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource MyInstance ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n"
]
]
}
}
}
},
"InstanceSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Enable SSH access and HTTP from the load balancer only",
"SecurityGroupIngress": [{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": {
"Ref": "SSHLocation"
}
},
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"SourceSecurityGroupId": {
"Fn::Select": [
0,
{
"Fn::GetAtt": [
"ApplicationLoadBalancer",
"SecurityGroups"
]
}
]
}
}
],
"VpcId": {
"Ref": "VpcId"
}
}
}
},
"Outputs": {
"URL": {
"Description": "The URL of the website",
"Value": {
"Fn::Join": [
"",
[
"http://",
{
"Fn::GetAtt": [
"ApplicationLoadBalancer",
"DNSName"
]
}
]
]
}
}
}
}
So I have been using this logic apps template to hit the Google Analytics API and the response is in this format
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date",
"ga:campaign",
"ga:country",
"ga:browser",
"ga:deviceCategory",
"ga:sourceMedium",
"ga:socialNetwork",
"ga:region"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:users",
"type": "INTEGER"
},
{
"name": "ga:sessions",
"type": "INTEGER"
},
{
"name": "ga:newUsers",
"type": "INTEGER"
},
{
"name": "ga:bounces",
"type": "INTEGER"
},
{
"name": "ga:pageviews",
"type": "INTEGER"
},
{
"name": "ga:sessionDuration",
"type": "TIME"
},
{
"name": "ga:hits",
"type": "INTEGER"
},
{
"name": "ga:goalCompletionsAll",
"type": "INTEGER"
},
{
"name": "ga:goalConversionRateAll",
"type": "PERCENT"
}
]
}
},
"data": {
"rows": [
{
"dimensions": [
"20200312",
"(not set)",
"India",
"Chrome",
"desktop",
"(direct) / (none)",
"(not set)",
"Tamil Nadu"
],
"metrics": [
{
"values": [
"4",
"4",
"4",
"0",
"111",
"5100.0",
"111",
"0",
"0.0"
]
}
]
},
{
"dimensions": [
"20200316",
"(not set)",
"India",
"Chrome",
"desktop",
"(direct) / (none)",
"(not set)",
"Tamil Nadu"
],
"metrics": [
{
"values": [
"1",
"1",
"0",
"0",
"6",
"266.0",
"6",
"0",
"0.0"
]
}
]
},
{
"dimensions": [
"20200318",
"(not set)",
"India",
"Chrome",
"desktop",
"(direct) / (none)",
"(not set)",
"Tamil Nadu"
],
"metrics": [
{
"values": [
"1",
"2",
"0",
"0",
"20",
"135.0",
"20",
"0",
"0.0"
]
}
]
}
],
"totals": [
{
"values": [
"6",
"7",
"4",
"0",
"137",
"5501.0",
"137",
"0",
"0.0"
]
}
],
"rowCount": 3,
"minimums": [
{
"values": [
"1",
"1",
"0",
"0",
"6",
"135.0",
"6",
"0",
"0.0"
]
}
],
"maximums": [
{
"values": [
"4",
"4",
"4",
"0",
"111",
"5100.0",
"111",
"0",
"0.0"
]
}
],
"isDataGolden": true
}
}
]
}
I Want to convert it and bring it in a form that the column header:dimensions and metric header entries name will become column names and their values,ie data.rows.dimensions and metrics.values become corresponding values
ga:date ga:campaign ga:country ga:browser ga:deviceCategory ga:sourceMedium ga:socialNetwork ga:region ga:users ga:sessions ga:newUsers : (column names)
20200316 (not set) India Chrome desktop (direct) / (none) (not set) Tamil Nadu 1 1 1 :(values)
If you can use an Integration account, I suggest to create a flat file schema with the desired structure, and in the logic app you can convert in xml and then apply the Flat File Encoding.
Otherwise a function app should resolve your issue
I am new to this website and came here because I am really struggling with a problem of extracting information from a JSON file. The tricky part is that there are variable number of fields, so I can't get away with simple syntax.
Here's a sample code:
{
"addresses": {
"#count": "1",
"address_name": {
"address_spec": {
"#addr_no": "1",
"full_address": "Tel Aviv Univ, Eitan Berglas Sch Econ, IL-69978 Tel Aviv, Israel",
"organizations": {
"#count": "2",
"organization": [
"Tel Aviv Univ",
{
"#pref": "Y",
"#text": "Tel Aviv University"
}
]
},
"suborganizations": {
"#count": "1",
"suborganization": "Eitan Berglas Sch Econ"
},
"city": "Tel Aviv",
"country": "Israel",
"zip": {
"#location": "BC",
"#text": "IL-69978"
}
}
}
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "1",
"address_name": {
"address_spec": {
"#addr_no": "1",
"full_address": "MIT, Cambridge, MA 02139 USA",
"organizations": {
"#count": "2",
"organization": [
"MIT",
{
"#pref": "Y",
"#text": "Massachusetts Institute of Technology (MIT)"
}
]
},
"city": "Cambridge",
"state": "MA",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "02139"
}
}
}
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "2",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "Univ Kentucky, Lexington, KY 40506 USA",
"organizations": {
"#count": "2",
"organization": [
"Univ Kentucky",
{
"#pref": "Y",
"#text": "University of Kentucky"
}
]
},
"city": "Lexington",
"state": "KY",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "40506"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "Univ Bonn, ZEI, D-5300 Bonn, Germany",
"organizations": {
"#count": "2",
"organization": [
"Univ Bonn",
{
"#pref": "Y",
"#text": "University of Bonn"
}
]
},
"suborganizations": {
"#count": "1",
"suborganization": "ZEI"
},
"city": "Bonn",
"country": "Germany",
"zip": {
"#location": "BC",
"#text": "D-5300"
}
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "1",
"address_name": {
"address_spec": {
"#addr_no": "1",
"full_address": "Harvard Univ, Cambridge, MA 02138 USA",
"organizations": {
"#count": "2",
"organization": [
"Harvard Univ",
{
"#pref": "Y",
"#text": "Harvard University"
}
]
},
"city": "Cambridge",
"state": "MA",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "02138"
}
}
}
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "3",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "Columbia Univ, New York, NY 10027 USA",
"organizations": {
"#count": "2",
"organization": [
"Columbia Univ",
{
"#pref": "Y",
"#text": "Columbia University"
}
]
},
"city": "New York",
"state": "NY",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "10027"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "NYU, New York, NY USA",
"organizations": {
"#count": "2",
"organization": [
"NYU",
{
"#pref": "Y",
"#text": "New York University"
}
]
},
"city": "New York",
"state": "NY",
"country": "USA"
}
},
{
"address_spec": {
"#addr_no": "3",
"full_address": "Univ Pompeu Fabra, Barcelona, Spain",
"organizations": {
"#count": "2",
"organization": [
"Univ Pompeu Fabra",
{
"#pref": "Y",
"#text": "Pompeu Fabra University"
}
]
},
"city": "Barcelona",
"country": "Spain"
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "2",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "Univ Chicago, Chicago, IL 60637 USA",
"organizations": {
"#count": "2",
"organization": [
"Univ Chicago",
{
"#pref": "Y",
"#text": "University of Chicago"
}
]
},
"city": "Chicago",
"state": "IL",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "60637"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "Amer Bar Fdn, Chicago, IL 60611 USA",
"organizations": {
"#count": "1",
"organization": "Amer Bar Fdn"
},
"city": "Chicago",
"state": "IL",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "60611"
}
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "2",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "Ohio State Univ, Columbus, OH 43210 USA",
"organizations": {
"#count": "2",
"organization": [
"Ohio State Univ",
{
"#pref": "Y",
"#text": "Ohio State University"
}
]
},
"city": "Columbus",
"state": "OH",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "43210"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "Harvard Univ, Cambridge, MA 02138 USA",
"organizations": {
"#count": "2",
"organization": [
"Harvard Univ",
{
"#pref": "Y",
"#text": "Harvard University"
}
]
},
"city": "Cambridge",
"state": "MA",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "02138"
}
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "1",
"address_name": {
"address_spec": {
"#addr_no": "1",
"full_address": "Univ Chicago, Chicago, IL 60637 USA",
"organizations": {
"#count": "2",
"organization": [
"Univ Chicago",
{
"#pref": "Y",
"#text": "University of Chicago"
}
]
},
"city": "Chicago",
"state": "IL",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "60637"
}
}
}
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "2",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "Wissensch Zentrum Berlin Sozialforsch, D-1000 Berlin, Germany",
"organizations": {
"#count": "1",
"organization": "Wissensch Zentrum Berlin Sozialforsch"
},
"city": "Berlin",
"country": "Germany",
"zip": {
"#location": "BC",
"#text": "D-1000"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "Harvard Univ, Dept Govt, Cambridge, MA 02138 USA",
"organizations": {
"#count": "2",
"organization": [
"Harvard Univ",
{
"#pref": "Y",
"#text": "Harvard University"
}
]
},
"suborganizations": {
"#count": "1",
"suborganization": "Dept Govt"
},
"city": "Cambridge",
"state": "MA",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "02138"
}
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
{
"addresses": {
"#count": "2",
"address_name": [
{
"address_spec": {
"#addr_no": "1",
"full_address": "NYU, CV Starr Ctr Appl Econ, New York, NY 10003 USA",
"organizations": {
"#count": "2",
"organization": [
"NYU",
{
"#pref": "Y",
"#text": "New York University"
}
]
},
"suborganizations": {
"#count": "1",
"suborganization": "CV Starr Ctr Appl Econ"
},
"city": "New York",
"state": "NY",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "10003"
}
}
},
{
"address_spec": {
"#addr_no": "2",
"full_address": "Princeton Univ, Princeton, NJ 08544 USA",
"organizations": {
"#count": "2",
"organization": [
"Princeton Univ",
{
"#pref": "Y",
"#text": "Princeton University"
}
]
},
"city": "Princeton",
"state": "NJ",
"country": "USA",
"zip": {
"#location": "AP",
"#text": "08544"
}
}
}
]
},
"category_info": {
"headings": {
"#count": "1",
"heading": "Social Sciences"
},
"subjects": {
"#count": "3",
"subject": [
{
"#ascatype": "traditional",
"#text": "Economics"
},
{
"#ascatype": "extended",
"#text": "Business & Economics"
},
{
"#ascatype": "traditional",
"#text": "ECONOMICS"
}
]
}
}
}
What I was hoping to extract is a country for each of the records (some records have more than one country, which seems to be causing the problem). So my naive approach was to say:
.static_data."fullrecord_metadata".addresses.address_name.country
This however gives me several errors (null has no keys, and cannot index array with string). Checking using the keys command:
.static_data."fullrecord_metadata".addresses.address_name | keys
I can see that it's seems there's a problem with the way the data is structured...
So, could you suggest if I can actually extract the list of countries for each entry using jq? Thank you!
For each input top-level JSON entity, the following filter will recursively examine all the objects to see if they have a "country" key, and it will then report the distinct "country" values for that top-level entity:
jq -c '[.. | if type == "object" and has("country")
then .country
else empty end] | unique'
["Israel"]
["USA"]
["Germany","USA"]
["USA"]
["Spain","USA"]
["USA"]
["USA"]
["USA"]
["Germany","USA"]
["USA"]
Here's a filter that will produce the same results in your example, though it is not exactly equivalent:
[.. | .country? // empty] | unique
[Exercise for the interested reader: what is the difference? :-) ]
Here is a solution which uses a function to handle the variation in .address_name
def address_specs:
if type == "array" then .[].address_spec else .address_spec end
;
.addresses | .address_name | [address_specs | .country] | unique