Transforming JSON - json

I am trying to transform JSON data provided by Quandl into a custom JSON format so that I can load it into my database.
The JSON Array is stock market data with Date, High, Low, Open, Close values. I need a flat JSON instead of an array.
I tried the following, but it returns full array instead of individual element. If I use [0][1], [0][2], it returns empty values.
Here is my code
var DataTransform = require("node-json-transform").DataTransform
var myData = {"dataset":{"data":[["2016-01-15",292.5,294.4,267.1,279.9,273.0,64104.0,182.09],["2016-01-14",288.0,302.0,265.0,287.6,288.2,68271.0,199.82],["2016-01-13",303.95,307.65,275.0,290.1,292.75,99921.0,293.08]]}}
var map = {
list : 'dataset.data',
item: {
date: [0],
high: [0][1],
low: [0][0][1]
}
};
var dataTransform = DataTransform(myData, map);
var result = dataTransform.transform();
console.log(result);
======================================================================
Output
[{"date":[["2016-01-15",292.5,294.4,267.1,279.9,273,64104,182.09]],"high":"","low":""},{"date":[["2016-01-14",288,302,265,287.6,288.2,68271,199.82]],"high":"","low":""},{"date":[["2016-01-13",303.95,307.65,275,290.1,292.75,99921,293.08]],"high":"","low":""}]

You should use the built-in Array functions for transforming data. e.g. map, reduce, filter, sort, etc. In this case map will do the job perfectly. e.g.
var ds = {"dataset":{"data":[["2016-01-15",292.5,294.4,267.1,279.9,273.0,64104.0,182.09],["2016-01-14",288.0,302.0,265.0,287.6,288.2,68271.0,199.82],["2016-01-13",303.95,307.65,275.0,290.1,292.75,99921.0,293.08]]}}
var transformed = ds.dataset.data
.map(function (d) {
return {
date : d[0],
high : d[1],
low : d[2],
etc:
}
})
// output in format : [{date:"2016-01-15",high:"294.4",low:"267.1"},...]

Related

JSON parse when top node is a number

I connect to various crypto public API's and get values using 3 steps:
var response = UrlFetchApp.fetch("API URL",{muteHttpExceptions: true});
var json = JSON.parse(response.getContentText());
And then depending on the output format, I do one of the following:
Tickers:
0:
last_trade:
will result in me using:
var rate1 = json.tickers[0].last_trade;
result:
XXRPXXBT:
a:
0:
will result in me using:
var rate1 = json.result.XXRPXXBT.a[0];
All of the methods I use work fine except when I get this format:
0:
price:
1:
price:
When I try use one of these, it does not work:
var rate1 = json[0].price;
var rate1 = json.[0].price;
var rate1 = json.0.price;
How do I read it when the top node is a number?
When you need to access a key that is a number you should enclose it in quotes like this (although even not enclosing it should work):
json["0"].price
The other 2 ways you tried are not a valid js syntax
Also check this link from MDN : https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Property_accessors#property_names
var json = {
0:{
price: 100
}
}
console.log(json["0"].price)
// Update for your enpoint in my comments
// Assuming that this have been parse and stored in a variable like the following on
let priceData = [
{
"symbol": "ETHBTC",
"price": "0.06643200"
},
{
"symbol": "LTCBTC",
"price": "0.00461600"
}
]
///The above is an array
/// You access it via numeric indexes like that
console.log(priceData[0].price);
// However make sure that you have actually parsed your api response as a json object first

Turn JSON Data Into Int To Perform Calculations

Trying to convert JSON data into int in order to perform calculations, multiply by numbers or percentages (or whichever method is best recommended)
Tried performing calculation on the object (using addition for example), but it only added numbers on to the end of the resulting string. I have seen suggestions on using JSON parse (reviver) but can't seem to get my head round getting the desired data when it is only one specific part of the JSON data required rather than multiple items of data from the JSON link.
var xmlhttp = new XMLHttpRequest();
var url = "https://api.coindesk.com/v1/bpi/currentprice.json";
xmlhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
var json = JSON.parse(this.responseText);
parseJson(json);
}
};
xmlhttp.open("GET", url, true);
xmlhttp.send();
function parseJson(json) {
var gbpValue = "1 BTC equals to £" + json["bpi"]["GBP"]["rate"];
document.getElementById("data").innerHTML =
gbpValue;
As mentioned, have tried performing calculations on the result but it only adds numbers to the end of the string. Thanks for any advice or help.
What I can see in your code is that you are adding your json variable with a string which would always result in string concatenation in below line:
var gbpValue = "1 BTC equals to £" + json["bpi"]["GBP"]["rate"];
First check your json object if it a number or string.
You could use the Number() or parseInt() functions to convert a string to number.
Example:
var num=Number("23");
or
var num=parseInt("23");
Hope it helps. :)

Populate FeatureCollection with values from bands of each individual image in an image collection in Google Earth Engine

In Google Earth Engine, I have loaded in a Featurecollection as a JSON which contains a few polygons. I would like to add columns to this FeatureCollection which gives me the mean values of two bands for each polygon and from each of the multiple images contained within the Image Collection.
Here is the code I have so far.
//Polygons
var polygons = ee.FeatureCollection('ft:1_z8-9NMZnJie34pXG6l-3StxlcwSKSTJFfVbrdBA');
Map.addLayer(polygons);
//Date of interest
var start = ee.Date('2008-01-01');
var finish = ee.Date('2010-12-31');
//IMPORT Landsat IMAGEs
var Landsat = ee.ImageCollection('LANDSAT/LT05/C01/T1') //Landsat images
.filterBounds(polygons)
.filterDate(start,finish)
.select('B4','B3');
//Add ImageCollection to Map
Map.addLayer(Landsat);
//Map the function over the collection and display the result
print(Landsat);
// Empty Collection to fill
var ft = ee.FeatureCollection(ee.List([]))
var fill = function(img, ini) {
// type cast
var inift = ee.FeatureCollection(ini)
// gets the values for the points in the current img
var mean = img.reduceRegions({
collection:polygons,
reducer: ee.Reducer.mean(),
});
// Print the first feature, to illustrate the result.
print(ee.Feature(mean.first()).select(img.bandNames()));
// writes the mean in each feature
var ft2 = polygons.map(function(f){return f.set("mean", mean)})
// merges the FeatureCollections
return inift.merge(ft2)
// gets the date of the img
var date = img.date().format()
// writes the date in each feature
var ft3 = polygons.map(function(f){return f.set("date", date)})
// merges the FeatureCollections
return inift.merge(ft3)
}
// Iterates over the ImageCollection
var newft = ee.FeatureCollection(Landsat.iterate(fill, ft))
// Export
Export.table.toDrive(newft,
"anyDescription",
"anyFolder",
"test")
In the console I get an error message
Element (Error)
Failed to decode JSON.
Error: Field 'value' of object '{"type":"ArgumentRef","value":null}' is missing or null.
Object: {"type":"ArgumentRef","value":null}.
In my csv file which is generated I get a new column called mean but this is populated with and no actual values.
There is no reason to use iterate() here. What you can do is a nested map(). Over polygons and then over images. You can flatten the resulting list of lists to turn it into a single list like this:
// compute mean band values by mapping over polygons and then over images
var results = polygons.map(function(f) {
return images.map(function(i) {
var mean = i.reduceRegion({
geometry: f.geometry(),
reducer: ee.Reducer.mean(),
});
return f.setMulti(mean).set({date: i.date()})
})
})
// flatten
results = results.flatten()
Script: https://code.earthengine.google.com/b65a731c78f78a6f9e08300dcf552dff
The same approach can be used with reduceRegions() as well, mapping over images and then over regions. However, you will have to map over the resulting features to set dates.
images.filterBounds(f) can be probably also added if your features cover a larger area.
PS: your table is not shared

Printing parsed JSON

So, I've parsed a response like this:
var g_rgListingInfo = JSON.parse( response );
response looks like this
{"321242653847396921":{"listingid":"321242653847396921","price":28338,"fee":4249,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2003","steam_fee":1416,"publisher_fee":2833,"asset":{"currency":0,"appid":730,"contextid":"2","id":"3038615825","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"321242653843485871":{"listingid":"321242653843485871","price":30175,"fee":4525,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2003","steam_fee":1508,"publisher_fee":3017,"asset":{"currency":0,"appid":730,"contextid":"2","id":"1730491611","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"782860982384213986":{"listingid":"782860982384213986","price":31305,"fee":4695,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2003","steam_fee":1565,"publisher_fee":3130,"asset":{"currency":0,"appid":730,"contextid":"2","id":"2815962367","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"783987515556891867":{"listingid":"783987515556891867","price":31305,"fee":4695,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2003","steam_fee":1565,"publisher_fee":3130,"asset":{"currency":0,"appid":730,"contextid":"2","id":"3708699202","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"783987515558623437":{"listingid":"783987515558623437","price":30957,"fee":4642,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2003","steam_fee":1547,"publisher_fee":3095,"asset":{"currency":0,"appid":730,"contextid":"2","id":"4462433815","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"718685320959305952":{"listingid":"718685320959305952","price":34000,"fee":5100,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2001","steam_fee":1700,"publisher_fee":3400,"asset":{"currency":0,"appid":730,"contextid":"2","id":"4450043953","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"796369492002647568":{"listingid":"796369492002647568","price":34500,"fee":5175,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2001","steam_fee":1725,"publisher_fee":3450,"asset":{"currency":0,"appid":730,"contextid":"2","id":"4024113558","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D3082226233578562378","name":"Inspect in Game..."}]}},"718684619833530742":{"listingid":"718684619833530742","price":22958,"fee":3442,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2002","steam_fee":1147,"publisher_fee":2295,"asset":{"currency":0,"appid":730,"contextid":"2","id":"4331886445","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"788487401257494747":{"listingid":"788487401257494747","price":34783,"fee":5217,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2001","steam_fee":1739,"publisher_fee":3478,"asset":{"currency":0,"appid":730,"contextid":"2","id":"2315637005","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D1030942533801731526","name":"Inspect in Game..."}]}},"321242020664839911":{"listingid":"321242020664839911","price":34783,"fee":5217,"publisher_fee_app":730,"publisher_fee_percent":"0.10000000149011612","currencyid":"2001","steam_fee":1739,"publisher_fee":3478,"asset":{"currency":0,"appid":730,"contextid":"2","id":"4283078084","amount":"1","market_actions":[{"link":"steam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D6944696178921031564","name":"Inspect in Game..."}]}}}
I put it in here: http://json.parser.online.fr/ this is the
result
The problem I'm having is that I fail to loop through the items. g_rgListingInfo.length is NaN. I tried to use forEach but that failed too.
I want to loop through all these "321242653847396921", "321242653843485871"... wich are always changing and obtain their listingid, price, fee etc.
I am pretty new to node.js so I'm sorry if this is a stupid question.
You have an object, not an array. So, to iterate the results, you have to either convert your object into an array or just iterate it as object.
Converting into array
Depending on what you need, this could be more convenient:
var myData = Object.keys(g_rgListingInfo).map(Number).sort().map(function (c) {
return g_rgListingInfo[c];
});
// Then you can just use the `myData` array
myData.forEach(function (current, index) {
/* do something */
});
// ...or using for loop
for (var i = 0; i < myData.length; ++i) {
var current = myData[i];
/* do something */
}
Iterating the object
You have to get the keys of the object iterate them (optionally you maybe want to sort them first).
// ["3124...", ...]
var numbers = Object.keys(g_rgListingInfo);
// Optional sort
numbers = numbers.map(Number).sort();
// Iterate the object keys
numbers.forEach(function (currentKey) {
var currentObject = g_rgListingInfo[currentKey];
});

Query a JSON list of dict

[{"time":136803,"price":"1.4545","amount":"0.0885","ID":"112969"},
{"time":136804,"price":"2.5448","amount":"0.0568","ID":"5468489"},
{"time":136805,"price":"1.8948","amount":"0.0478","ID":"898489"}]
I have a large JSON file like the one above. It is a list of dictionaries. I want to choose a time and find the value assoaciated with that time. I will not know where in my list the time is located only the value for the time. Is there a way I can say, for time 136804, make x = to price? Or should I loop through each value? I also want to use this value (x) in a mathematical function.
My fist idea is to use brute force by going through each item and checking it for a matching time value in a loop.
Is this the best way?
Take a look at SpahQL http://danski.github.io/spahql/ which we use to query JSON in order to select values and subsequently change them as required.
I did something similar to this recently. JSON file I had to query had around 6000 lines and around 500 JSON objects. My query function given below loops through the each object to select the matching objects, but it can fetch any result within few milliseconds.
var data = '[{"time":136803,"price":"1.4545","amount":"0.0885","ID":"112969"},'+ '{"time":136804,"price":"2.5448","amount":"0.0568","ID":"5468489"},'+ '{"time":136805,"price":"1.8948","amount":"0.0478","ID":"898489"}]';
var data = JSON.parse(data);
var query = function(data, select, andwhere) {
var return_array = [];
$.each(data, function (i, obj) {
var temp_obj = {};
var where = true;
if (andwhere) {
$.each(andwhere, function(j, wh) {
if (obj[wh.col] !== wh.val) {
where = false;
}
});
}
if (where === false) {
return;
}
$.each(obj, function (j, elem) {
if (select.indexOf(j.trim())!==-1) {
temp_obj[j] = elem;
}
});
return_array.push(temp_obj);
});
return return_array;
};
var result = query(data, ['price','amount'],[{"col":"time","val":136804}]);
console.log(JSON.stringify(result));
http://jsfiddle.net/bejgy3sn/1/