I've got a CloudFormation template that creates an SNS topic and subscription
{
"AWSTemplateFormatVersion": "2010-09-09",
"Resources" : {
"EmailSNSTopic": {
"Type" : "AWS::SNS::Topic",
"Properties" : {
"DisplayName" : "${display_name}"
}
},
"MySubscription": {
"Type": "AWS::SNS::Subscription",
"Properties": {
"TopicArn" : { "Ref" : "EmailSNSTopic" },
"${details}"
}
}
},
"Outputs" : {
"ARN" : {
"Description" : "Email SNS Topic ARN",
"Value" : { "Ref" : "EmailSNSTopic" }
}
}
}
Which I'm trying to call via terrform.
But I keep getting this error
Error: "template_body" contains an invalid JSON: invalid character '{' looking for beginning of object key string
My Terraform configuration looks like this.
provider "aws" {
region = "eu-west-2"
}
data "template_file" "sns_stack" {
template = file("${path.module}/templates/email-sns-stack.json.tpl")
vars = {
display_name = var.display_name
details = join(",", formatlist("{ \"Endpoint\": \"%s\", \"Protocol\": \"%s\" }", var.email_list, var.protocol))
}
}
resource "aws_cloudformation_stack" "sns_topic" {
name = var.stack_name
template_body = data.template_file.sns_stack.rendered
tags = merge(
map("Name", var.stack_name)
)
}
And my variables.tf looks like this
default = "Admin"
}
variable "email_list" {
default = [
"foo#foo.com",
"bar#bar.com"
]
}
variable "protocol" {
default = "email"
}
variable "stack_name" {
default = "sns-test"
}
I expect that ${details} should spit out my Endpoint and Protocol but it doesn't.
What am I doing wrong?
What you want to achieve is rather complex, but doable. You can do this using the following template:
{
"AWSTemplateFormatVersion": "2010-09-09",
"Resources" : ${jsonencode(
merge({for idx, email_address in email_list:
"EmailSubs${idx}" => {
Type = "AWS::SNS::Subscription"
Properties = {
"Endpoint" = email_address
"Protocol" = protocol
"TopicArn" = { "Ref" = "EmailSNSTopic" }
}
}},
{
"EmailSNSTopic" = {
"Type" = "AWS::SNS::Topic",
"Properties" = {
"DisplayName" = "${display_name}"
}
}}
))},
"Outputs" : {
"ARN" : {
"Description" : "Email SNS Topic ARN",
"Value" : { "Ref" : "EmailSNSTopic" }
}
}
}
and TF code:
locals {
template_body = templatefile("./email-sns-stack2.json.tpl", {
display_name = var.display_name
email_list = var.email_list
protocol = var.protocol
})
}
resource "aws_cloudformation_stack" "sns_topic" {
name = var.stack_name
template_body = local.template_body
tags = merge(
map("Name", var.stack_name)
)
}
Related
"open" suppose to take a boolean value and I should be able to do my filter the results with it but I'm ending up with an empty array as result, excuse the messy code please
let queryToMatch = [{$match: {name: project}},{$unwind:"$issues"}];
if(_id != undefined){
queryToMatch.push({$match: {"_id": _id}})
}
if(open != undefined){
queryToMatch.push({$match: {"open": open}})
}
if(issue_title != undefined){
queryToMatch.push({$match:{"issue_title": issue_title}})
}
if(issue_text != undefined){
queryToMatch.push({$match:{"issue_text": issue_text}})
}
if(created_by != undefined){
queryToMatch.push({$match:{"created_by": created_by}})
}
if(assigned_to != undefined){
queryToMatch.push({$match:{"assigned_to": assigned_to}})
}
if(status_text != undefined){
queryToMatch.push({$match:{"status_text": status_text}})
}
console.log(queryToMatch)
res.json("works")
Project.aggregate(queryToMatch, (err, data) => {
//console.log(data)
res.json(data)
})
You have not included the schema, or sample documents you are attempting to query. For this reason my answer may not be as you expect.
If I add the following documement...
db.test.insert(
{
"_id" : ObjectId("60b44ae6af90ae8f4deca589"),
"name": "project",
"open": "open",
"issue_title": "mytitle",
"issue_text": "mytext",
"created_by": "barry",
"assigned_to": "john",
"status_text": "mystatus",
"issues": [
{
"issue_type": "type1",
"issue_date": new Date
},
{
"issue_type": "type2",
"issue_date": new Date
},
{
"issue_type": "type3",
"issue_date": new Date
},
]
}
)
I can run the following mongo shell commands...
var _id = ObjectId("60b44ae6af90ae8f4deca589");
var open = "open";
var issue_title = "mytitle";
var issue_text = "mytext";
var created_by = "barry";
var assigned_to = "john";
var status_text = "mystatus";
let queryToMatch = [{"$match": {"name": "project"}},{$unwind:"$issues"}];
if(_id != undefined){
queryToMatch.push({$match: {"_id": _id}})
}
if(open != undefined){
queryToMatch.push({$match: {"open": open}})
}
if(issue_title != undefined){
queryToMatch.push({$match:{"issue_title": issue_title}})
}
if(issue_text != undefined){
queryToMatch.push({$match:{"issue_text": issue_text}})
}
if(created_by != undefined){
queryToMatch.push({$match:{"created_by": created_by}})
}
if(assigned_to != undefined){
queryToMatch.push({$match:{"assigned_to": assigned_to}})
}
if(status_text != undefined){
queryToMatch.push({$match:{"status_text": status_text}})
}
... then I can issue the following aggregate statement...
db.test.aggregate(queryToMatch).pretty()
... and I get the following results...
{
"_id" : ObjectId("60b44ae6af90ae8f4deca589"),
"name" : "project",
"open" : "open",
"issue_title" : "mytitle",
"issue_text" : "mytext",
"created_by" : "barry",
"assigned_to" : "john",
"status_text" : "mystatus",
"issues" : {
"issue_type" : "type1",
"issue_date" : ISODate("2021-05-31T02:46:04.670Z")
}
}
{
"_id" : ObjectId("60b44ae6af90ae8f4deca589"),
"name" : "project",
"open" : "open",
"issue_title" : "mytitle",
"issue_text" : "mytext",
"created_by" : "barry",
"assigned_to" : "john",
"status_text" : "mystatus",
"issues" : {
"issue_type" : "type2",
"issue_date" : ISODate("2021-05-31T02:46:04.670Z")
}
}
{
"_id" : ObjectId("60b44ae6af90ae8f4deca589"),
"name" : "project",
"open" : "open",
"issue_title" : "mytitle",
"issue_text" : "mytext",
"created_by" : "barry",
"assigned_to" : "john",
"status_text" : "mystatus",
"issues" : {
"issue_type" : "type3",
"issue_date" : ISODate("2021-05-31T02:46:04.670Z")
}
}
It seems your code does work in the context of mongo shell. My advice is to combine the $match stages together first, then perform the $unwind. I assume each of the predicates you are tacking on via a push are intended to be an 'AND' condition, not an 'OR' condition. Here is what the end result of your aggregation looks like after all the logical shenanigans...
[
{
"$match" : {
"name" : "project"
}
},
{
"$unwind" : "$issues"
},
{
"$match" : {
"_id" : ObjectId("60b44ae6af90ae8f4deca589")
}
},
{
"$match" : {
"open" : "open"
}
},
{
"$match" : {
"issue_title" : "mytitle"
}
},
{
"$match" : {
"issue_text" : "mytext"
}
},
{
"$match" : {
"created_by" : "barry"
}
},
{
"$match" : {
"assigned_to" : "john"
}
},
{
"$match" : {
"status_text" : "mystatus"
}
}
]
Perhaps this is a cleaner and easier to read query...
[
{
"$match" : {
"_id" : ObjectId("60b44ae6af90ae8f4deca589"),
"name" : "project",
"open" : "open",
"issue_title" : "mytitle",
"issue_text" : "mytext",
"created_by" : "barry",
"assigned_to" : "john",
"status_text" : "mystatus"
}
},
{
"$unwind" : "$issues"
},
]
Consider instead of using push you create an object for your $match predicate...
var match = new Object();
match.name = "project";
if(_id != undefined){
match._id = _id;
}
if(open != undefined){
match.open = open;
}
if(issue_title != undefined){
match.issue_title = issue_title;
}
if(issue_text != undefined){
match.issue_text = issue_text;
}
if(created_by != undefined){
match.created_by = created_by;
}
if(assigned_to != undefined){
match.assigned_to = assigned_to;
}
if(status_text != undefined){
match.status_text = status_text;
}
let queryToMatch = [{"$match": match},{$unwind:"$issues"}];
db.test.aggregate(queryToMatch).pretty()
{
"root1" : {
"sub1" : null,
"sub2" : {
"subsub1" : {
"key1" : {
},
"key2" : {
},
"key3" : {
},
"key4" : {
}
}
},
"sub3" : {
"subsub2" : {
"key5" : {
}
}
}
},
"root2" : {
"sub1" : null,
"sub2" : {
"subsub1" : {
"key1" : {
},
"key2" : {
},
"key3" : {
},
"key4" : {
}
}
},
"sub3" : {
"subsub2" : {
"key8" : {
}
}
}
}
}
consider the above json.
How to know if 'key8' exists in this json and also find the path where its found in the json.
e.g if searched for 'key8' need to get output similar to :
root2->sub3->subsub2->key8
It's just a straightforward tree traversal. The following returns as soon as a match is found (rather than looking for all matches).
sub key_search {
my $target = $_[1];
my #todo = [ $_[0] ];
while (#todo) {
my ($val, #path) = #{ shift(#todo) };
my $reftype = ref($val);
if (!$reftype) {
# Nothing to do
}
elsif ($reftype eq 'HASH') {
for my $key (keys(%$val)) {
return #path, $target if $key eq $target;
push #todo, [ $val->{$key}, #path, $key ];
}
}
elsif ($reftype eq 'ARRAY') {
for my $i (0..$#$val) {
push #todo, [ $val->[$i], #path, $i ];
}
}
else {
die("Invalid data.\n");
}
}
return;
}
my #path = key_search($data, 'key8')
or die("Not found.\n");
Notes
The result is ambiguous if the data can contain arrays, and if any of the hashes can have integers for keys. Steps can be taken to disambiguate them.
The above doesn't check for cycles, but those can't exist in JSON.
Replace push with unshift to get a depth-first search instead of a breadth-first search.
This is the JSON content I have uploaded in Firebase database.
Now I need to access this in my Angular 2 app.
I am getting undefined and NaN while trying to access the objects.
{
"clients" : {
"clients" : {
"-Kdl_wRRkn7nJxgz4B54" : {
"balance" : "100.00",
"email" : "jdoe#gmail.com",
"firstName" : "John",
"lastName" : "Doe",
"phone" : "555-555-5555"
},
"-KdleehAQm0HgVFYdkUo" : {
"balance" : "350.00",
"email" : "stevesmith#gmail.com",
"firstName" : "Steve",
"lastName" : "Smith",
"phone" : "444-444-4444"
}
}
}
}
I am trying to access this here
export class ClientsComponent implements OnInit {
clients:any[];
totalOwed:number;
constructor(
public clientService:ClientService
) { }
ngOnInit() {
this.clientService.getClients().subscribe(clients => {
this.clients = clients;
console.log(this.clients);
this.getTotalOwed();
});
}
getTotalOwed(){
let total = 0;
for(let i = 0;i < this.clients.length;i++){
console.log(this.clients[i]);
total += parseFloat(this.clients[i].balance);
}
this.totalOwed = total;
console.log(this.totalOwed);
}
}
First off, you should re-structure your JSON, the structure below will be better for looping through, and a better practice. second, the reason you are getting not a number is because you are not accessing the balance key's value.
your code to loop through the data structure below would look something like:
for(let i = 0; i < this.clients.length(); i++) {
console.log(this.clients[i].balance)
}
and the JSON:
{
"clients" : [
{
"id" : "-Kdl_wRRkn7nJxgz4B54",
"balance" : "100.00",
"email" : "jdoe#gmail.com",
"firstName" : "John",
"lastName" : "Doe",
"phone" : "555-555-5555"
},
{
"id" : "-KdleehAQm0HgVFYdkUo",
"balance" : "350.00",
"email" : "stevesmith#gmail.com",
"firstName" : "Steve",
"lastName" : "Smith",
"phone" : "444-444-4444"
}
]
}
I'm trying to convert JSON into Avro using the kite-sdk morphline module. After playing around I'm able to convert the JSON into Avro using a simple schema (no complex data types).
Then I took it one step further and modified the Avro schema as displayed below (subrec.avsc). As you can see the schema consist of a subrecord.
As soon as I tried to convert the JSON to Avro using the morphlines.conf and the subrec.avsc it failed.
Somehow the JSON paths "/record_type[]/alert/action" are not translated by the toAvro function.
The morphlines.conf
morphlines : [
{
id : morphline1
importCommands : ["org.kitesdk.**"]
commands : [
# Read the JSON blob
{ readJson: {} }
{ logError { format : "record: {}", args : ["#{}"] } }
# Extract JSON
{ extractJsonPaths { flatten: false, paths: {
"/record_type[]/alert/action" : /alert/action,
"/record_type[]/alert/signature_id" : /alert/signature_id,
"/record_type[]/alert/signature" : /alert/signature,
"/record_type[]/alert/category" : /alert/category,
"/record_type[]/alert/severity" : /alert/severity
} } }
{ logError { format : "EXTRACTED THIS : {}", args : ["#{}"] } }
{ extractJsonPaths { flatten: false, paths: {
timestamp : /timestamp,
event_type : /event_type,
source_ip : /src_ip,
source_port : /src_port,
destination_ip : /dest_ip,
destination_port : /dest_port,
protocol : /proto,
} } }
# Create Avro according to schema
{ logError { format : "WE GO TO AVRO"} }
{ toAvro { schemaFile : /etc/flume/conf/conf.empty/subrec.avsc } }
# Create Avro container
{ logError { format : "WE GO TO BINARY"} }
{ writeAvroToByteArray { format: containerlessBinary } }
{ logError { format : "DONE!!!"} }
]
}
]
And the subrec.avsc
{
"type" : "record",
"name" : "Event",
"fields" : [ {
"name" : "timestamp",
"type" : "string"
}, {
"name" : "event_type",
"type" : "string"
}, {
"name" : "source_ip",
"type" : "string"
}, {
"name" : "source_port",
"type" : "int"
}, {
"name" : "destination_ip",
"type" : "string"
}, {
"name" : "destination_port",
"type" : "int"
}, {
"name" : "protocol",
"type" : "string"
}, {
"name": "record_type",
"type" : ["null", {
"name" : "alert",
"type" : "record",
"fields" : [ {
"name" : "action",
"type" : "string"
}, {
"name" : "signature_id",
"type" : "int"
}, {
"name" : "signature",
"type" : "string"
}, {
"name" : "category",
"type" : "string"
}, {
"name" : "severity",
"type" : "int"
}
] } ]
} ]
}
The output on { logError { format : "EXTRACTED THIS : {}", args : ["#{}"] } } I output the following:
[{
/record_type[]/alert / action = [allowed],
/record_type[]/alert / category = [],
/record_type[]/alert / severity = [3],
/record_type[]/alert / signature = [GeoIP from NL,
Netherlands],
/record_type[]/alert / signature_id = [88006],
_attachment_body = [{
"timestamp": "2015-03-23T07:42:01.303046",
"event_type": "alert",
"src_ip": "1.1.1.1",
"src_port": 18192,
"dest_ip": "46.231.41.166",
"dest_port": 62004,
"proto": "TCP",
"alert": {
"action": "allowed",
"gid": "1",
"signature_id": "88006",
"rev": "1",
"signature" : "GeoIP from NL, Netherlands ",
"category" : ""
"severity" : "3"
}
}],
_attachment_mimetype=[json/java + memory],
basename = [simple_eve.json]
}]
UPDATE 2017-06-22
you MUST populate the data in the structure in order for this to work, by using addValues or setValues
{
addValues {
micDefaultHeader : [
{
eventTimestampString : "2017-06-22 18:18:36"
}
]
}
}
after debugging the sources of morphline toAvro, it appears that the record is the first object to be evaluated, no matter what you put in your mappings structure.
the solution is quite simple, but unfortunately took a little extra time, eclipse, running the flume agent in debug mode, cloning the source code and lots of coffee.
here it goes.
my schema:
{
"type" : "record",
"name" : "co_lowbalance_event",
"namespace" : "co.tigo.billing.cboss.lowBalance",
"fields" : [ {
"name" : "dummyValue",
"type" : "string",
"default" : "dummy"
}, {
"name" : "micDefaultHeader",
"type" : {
"type" : "record",
"name" : "mic_default_header_v_1_0",
"namespace" : "com.millicom.schemas.root.struct",
"doc" : "standard millicom header definition",
"fields" : [ {
"name" : "eventTimestampString",
"type" : "string",
"default" : "12345678910"
} ]
}
} ]
}
morphlines file:
morphlines : [
{
id : convertJsonToAvro
importCommands : ["org.kitesdk.**"]
commands : [
{
readJson {
outputClass : java.util.Map
}
}
{
addValues {
micDefaultHeader : [{}]
}
}
{
logDebug { format : "my record: {}", args : ["#{}"] }
}
{
toAvro {
schemaFile : /home/asarubbi/Development/test/co_lowbalance_event.avsc
mappings : {
"micDefaultHeader" : micDefaultHeader
"micDefaultHeader/eventTimestampString" : eventTimestampString
}
}
}
{
writeAvroToByteArray {
format : containerlessJSON
codec : null
}
}
]
}
]
the magic lies here:
{
addValues {
micDefaultHeader : [{}]
}
}
and in the mappings:
mappings : {
"micDefaultHeader" : micDefaultHeader
"micDefaultHeader/eventTimestampString" : eventTimestampString
}
explanation:
inside the code the first field name that is evaluated is micDefaultHeader of type RECORD. as there's no way to specify a default value for a RECORD (logically correct), the toAvro code evaluates this, does not get any value configured in mappings and therefore it fails at it detects (wrongly) that the record is empty when it shouldn't.
however, taking a look at the code, you may see that it requires a Map object, containing no values to please the parser and continue to the next element.
so we add a map object using the addValues and fill it with an empty map [{}]. notice that this must match the name of the record that is causing you an empty value. in my case "micDefaultHeader"
feel free to comment if you have a better solution, as this looks like a "dirty fix"
Given this json structure:
{
"categoryID" : 1,
"categoryName" : "Stupid Questions",
"questions" : [{
"question" : [{
"questionOptions" : [{
"questionOptionID" : 1,
"optionText" : "It's top secret."
}, {
"questionOptionID" : 2,
"optionText" : "Because I am big and your small. I am right and your wrong."
}, {
"questionOptionID" : 3,
"optionText" : "I will gladly pay you Tuesday for a hamburger today."
},
],
"questionType" : "checkbox",
"questionText" : "Why can't we use more abstract table and column names?",
"summary" : "Question of the year"
}
]
}
]
}
I would like to map both the questions and questionOptions to template and templateOptions:
{
"categoryID" : 1,
"categoryName" : "Stupid Questions",
"templates" : [{
"template" : [{
"templateOptions" : [{
"templateOptionID" : 1,
"optionText" : "It is top secret."
}, {
"QuestionOptionID" : 2,
"OptionText" : "Because we are lazy."
}, {
"QuestionOptionID" : 3,
"OptionText" : "I will gladly pay you Tuesday for a hamburger today."
},
],
"QuestionType" : "checkbox",
"QuestionText" : "Why can't we use more abstract table and column names?",
"Summary" : "Question of the year"
}
]
}
]
}
Here is the start to my knockout mapping object:
var templateMapping = {
'templates': {
templates: function(data) {
return ko.utils.unwrapObservable(data.questions);
}
}
//how do I map observable array of question options to observable array of template options here?
};
The key in this mapping is that the sub objects have a different structure (unlike this question - https://stackoverflow.com/a/7535397/466321). It seems like all of the mapping examples I have found don't cover how this may get done, and I have unsuccessfully tried a couple of theories of my own.
#Jeff Mercado is right. The mapper is not intended for this. To accomplish what you intend, it takes a bit of recursive javascript.
function myTransform(string) {
// case-insensitive replace
return string.replace(/question/i,'template');
}
function transformObject(source) {
var result = {}
for( var key in source ) {
if( !source.hasOwnProperty(key) ) continue;
var value = source[key];
var newKey = myTransform(key);
if( Object.prototype.toString.call(value) == "[object Array]" ) {
result[newKey] = [];
for( var i in value ) {
if( !value.hasOwnProperty(i) ) continue;
result[newKey][i] = transformObject(value[i]);
}
}
else if( Object.prototype.toString.call(value) == "[object Object]" ) {
result[newKey] = transformObject(value);
}
else {
result[newKey] = value;
}
}
return result;
}
var wow = transformObject(json);
See this fiddle