Formatting DynamoDB data to normal JSON in AWS Lambda - json

Question

I'm using AWS Lambda to scan data from a DynamoDB table. This is what I get in return:
{
"videos": [
{
"file": {
"S": "file1.mp4"
},
"id": {
"S": "1"
},
"canvas": {
"S": "This is Canvas1"
}
},
{
"file": {
"S": "main.mp4"
},
"id": {
"S": "0"
},
"canvas": {
"S": "this is a canvas"
}
}
]
}
My front-end application is using Ember Data Rest Adapter which does not accepts such response. Is there any way I can get normal JSON format? There is this NPM module called dynamodb-marshaler to convert DynamoDB data to normal JSON. I'm looking for a native solution if possible.

Answer 1

Node.js
Use the unmarshall function from AWSJavaScriptSDK:
const AWS = require("aws-sdk");
exports.handler = function( event, context, callback ) {
const newImages = event.Records.map(
(record) => AWS.DynamoDB.Converter.unmarshall(record.dynamodb.NewImage)
);
console.log('Converted records', newImages);
callback(null, `Success`);
}
Python
Use TypeDeserializer.deserialize from boto3.dynamodb.types:
import json
from boto3.dynamodb.types import TypeDeserializer
def ddb_deserialize(r, type_deserializer = TypeDeserializer()):
return type_deserializer.deserialize({"M": r})
def lambda_handler(event, context):
new_images = [ ddb_deserialize(r["dynamodb"]["NewImage"]) for r in event['Records'] ]
print('Converted records', json.dumps(new_images, indent=2))

Answer 2

I know is a bit old but I had the same problem processing stream data from dynamoDB in node js lambda function. I used the proposed by #churro
import sdk and output converter
var AWS = require("aws-sdk");
var parse = AWS.DynamoDB.Converter.output;
use the parse function with a small hack
exports.handler = function( event, context, callback ) {
var docClient = new AWS.DynamoDB.DocumentClient();
event.Records.forEach((record) => {
console.log(record.eventID);
console.log(record.eventName);
console.log('DynamoDB Record:', parse({ "M": record.dynamodb.NewImage }));
});
callback(null, `Successfully processed ${event.Records.length} records.`);
}
Hope it helps

Answer 3

AWS JavaScript SDK was recently updated with Document Client which does exactly what you need. Check the announce and usage examples here: http://blogs.aws.amazon.com/javascript/post/Tx1OVH5LUZAFC6T/Announcing-the-Amazon-DynamoDB-Document-Client-in-the-AWS-SDK-for-JavaScript

Answer 4

Javascript: AWS SDK provides the unmarshall function
Python: use TypeDeserializer from boto3.dynamodb.types:
from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
def from_dynamodb_to_json(item):
d = TypeDeserializer()
return {k: d.deserialize(value=v) for k, v in item.items()}
## Usage:
from_dynamodb_to_json({
"Day": {"S": "Monday"},
"mylist": {"L": [{"S": "Cookies"}, {"S": "Coffee"}, {"N": "3.14159"}]}
})
# {'Day': 'Monday', 'mylist': ['Cookies', 'Coffee', Decimal('3.14159')]}

Answer 5

Here you can find gist which does that:
function mapper(data) {
let S = "S";
let SS = "SS";
let NN = "NN";
let NS = "NS";
let BS = "BS";
let BB = "BB";
let N = "N";
let BOOL = "BOOL";
let NULL = "NULL";
let M = "M";
let L = "L";
if (isObject(data)) {
let keys = Object.keys(data);
while (keys.length) {
let key = keys.shift();
let types = data[key];
if (isObject(types) && types.hasOwnProperty(S)) {
data[key] = types[S];
} else if (isObject(types) && types.hasOwnProperty(N)) {
data[key] = parseFloat(types[N]);
} else if (isObject(types) && types.hasOwnProperty(BOOL)) {
data[key] = types[BOOL];
} else if (isObject(types) && types.hasOwnProperty(NULL)) {
data[key] = null;
} else if (isObject(types) && types.hasOwnProperty(M)) {
data[key] = mapper(types[M]);
} else if (isObject(types) && types.hasOwnProperty(L)) {
data[key] = mapper(types[L]);
} else if (isObject(types) && types.hasOwnProperty(SS)) {
data[key] = types[SS];
} else if (isObject(types) && types.hasOwnProperty(NN)) {
data[key] = types[NN];
} else if (isObject(types) && types.hasOwnProperty(BB)) {
data[key] = types[BB];
} else if (isObject(types) && types.hasOwnProperty(NS)) {
data[key] = types[NS];
} else if (isObject(types) && types.hasOwnProperty(BS)) {
data[key] = types[BS];
}
}
}
return data;
function isObject(value) {
return typeof value === "object" && value !== null;
}
}
https://gist.github.com/igorzg/c80c0de4ad5c4028cb26cfec415cc600

Answer 6

If you are using python in the lambda you can utilise the dynamodb-json library.
Install library
pip install dynamodb-json
and use the below snippet
from dynamodb_json import json_util as util
def marshall(regular_json):
dynamodb_json = util.dumps(reular_json)
def unmarshall(dynamodb_json):
regular_json = util.loads(dynamodb_json)
Reference
https://pypi.org/project/dynamodb-json/

Answer 7

I think it's just a custom transformation exercise for each app. A simple conversion from DynamoDB's item format to you application format might look like this:
var response = {...} // your response from DynamoDB
var formattedObjects = response.videos.map(function(video) {
return {
"file": video.file.S,
"id": video.id.S,
"canvas": video.canvas.S
};
});
If you want to build a generic system for this, you would have to handle DynamoDB's various AttributeValue types. A function like the one below would do the job, but I've left out the hard work of handling most of DynamoDB's more complex attribute value types:
function dynamoItemToPlainObj(dynamoItem) {
var plainObj = {};
for (var attributeName in dynamoItem) {
var attribute = dynamoItem[attributeName];
var attributeValue;
for (var itemType in attribute) {
switch (itemType) {
case "S":
attributeValue = attribute.S.toString();
break;
case "N":
attributeValue = Number(attribute.N);
break;
// more attribute types...
default:
attributeValue = attribute[itemType].toString();
break;
}
}
plainObj[attributeName] = attributeValue;
}
return plainObj;
}
var formattedObjects = response.videos.map(dynamoItemToPlainObj);

Answer 8

I tried several solutions here but none worked with multi-level data, such as if it includes a list of maps e.g.
{
"item1": {
"M": {
"sub-item1": {
"L": [
{
"M": {
"sub-item1-list-map": {
"S": "value"
Below, adapted from #igorzg's answer (which also has that drawback), fixes that.
Example usage:
dynamodb.getItem({...}, function(err, data) {
if (!err && data && data.Item) {
var converted = ddb_to_json(data.Item);
Here's the conversion function:
function ddb_to_json(data) {
function isObject(value) {
return typeof value === "object" && value !== null;
}
if(isObject(data))
return convert_ddb({M:data});
function convert_ddb(ddbData) {
if (isObject(ddbData) && ddbData.hasOwnProperty('S'))
return ddbData.S;
if (isObject(ddbData) && ddbData.hasOwnProperty('N'))
return parseFloat(ddbData.N);
if (isObject(ddbData) && ddbData.hasOwnProperty('BOOL'))
return ddbData.BOOL;
if (isObject(ddbData) && ddbData.hasOwnProperty('NULL'))
return null;
if (isObject(ddbData) && ddbData.hasOwnProperty('M')) {
var x = {};
for(var k in ddbData.M)
x[k] = convert_ddb(ddbData.M[k])
return x;
}
if (isObject(ddbData) && ddbData.hasOwnProperty('L'))
return ddbData.L.map(x => convert_ddb(x));
if (isObject(ddbData) && ddbData.hasOwnProperty('SS'))
return ddbData.SS;
if (isObject(ddbData) && ddbData.hasOwnProperty('NN'))
return ddbData.NN;
if (isObject(ddbData) && ddbData.hasOwnProperty('BB'))
return ddbData.BB;
if (isObject(ddbData) && ddbData.hasOwnProperty('NS'))
return ddbData.NS;
if (isObject(ddbData) && ddbData.hasOwnProperty('BS'))
return ddbData.BS;
return data;
}
return data;
}

Answer 9

If you need online editor try this
https://2json.net/dynamo