I have a big json file which is changed quite often (translation values). I need to check whether properties in this json are sorted alphabetically. Perfectly I'd like to use gulp for that and run this check in CI environment so when properties aren't ordered an error is reported.
I've searched for it but haven't found anything. Are you aware of any existing solution dealing with this problem?
You could do it purely with node.js and hook it into your gulp build process easily enough:
var fs = require('fs');
fs.readFile("data.json", { encoding: 'utf8' }, function(err, data) {
if (err) throw err;
var dataObj = JSON.parse(data).mystuff;
//check if it's ordered
for (var i = 1; i < dataObj.length; i++) {
if (dataObj[i].toLowerCase() < dataObj[i-1].toLowerCase()) { //note that depending on what it is you're ordering, this might need to change...
throw new Error('THIS IS OUT OF ORDER! I TOLD YOU! I TOLD YOU!!!! KEEP IT ORDERED! AND WHAT DID YOU DO?!!! UNACCEPTABLE!!!');
}
}
console.log('Ordered correctly! Good work!');
});
And the data.json that I used to test it, ordered incorrectly:
{
"mystuff": [
"Super",
"Duper",
"Happy",
"Fun"
]
}
And ordered correctly:
{
"mystuff": [
"Duper",
"Fun",
"Happy",
"Super"
]
}
And if you want to take it a step further and have the process automatically re-order the file for you, you could do something like this:
var fs = require('fs');
var fileName = 'data.json';
var fileOptions = { encoding: 'utf8' };
fs.readFile(fileName, fileOptions, function(err, data) {
if (err) throw err;
debugger;
var dataObj = JSON.parse(data);
var dataObjOrdered = orderObj(dataObj);
var jsonOrdered = JSON.stringify(dataObjOrdered, null, 4);
fs.writeFile(fileName, jsonOrdered, fileOptions, function(err) {
if (err) throw err;
console.log('JSON data fields are ordered huzzah!');
});
});
/**
* Returns an object with the same data, but with its fields ordered.
* Note that we leave array values in the same order they're already in
*/
function orderObj(obj) {
if (!isObject(obj)) {
return obj;
}
//arrays are left alone. if you want to order them too, do it inside this block
if (Array.isArray(obj)) {
return obj;
}
//load up an array containing all our object's fields
var fields = [];
for (var field in obj) {
fields.push(field);
}
//sort the fields alphabetically (remember to do this compare case-insensitively since utf-8 capital letters come before lowercase letters)
fields.sort(function (a, b) {
a = a.toLowerCase();
b = b.toLowerCase();
if (a > b) {
return 1;
}
if (a < b) {
return -1;
}
return 0;
});
//return the new object with ordered fields, recursively ordering sub objects' fields
var result = {};
for(var i = 0; i < fields.length; i++) {
var field = fields[i];
var value = obj[field];
result[field] = orderObj(value);
}
return result;
}
function isObject(value) {
// http://jsperf.com/isobject4
return value !== null && typeof value === 'object';
}
Related
The results of my query look like:
{
col1: [values],
col2: [values]
}
Is there a way to get my query to return as a list of rows objects?
The best way is to write a function to convert your query response.
One example:
function transformColumnSchemaToRowSchema(data) {
var keys = _.keys(data);
var arrays = _.values(data);
var arrayOfPropertyLists = _.zip.apply(_, arrays);
var arrayOfObjects = _.map(arrayOfPropertyLists, function(list) {
var obj = {};
_.each(keys, function(key, i) {
obj[key] = list[i];
});
return obj;
});
return arrayOfObjects;
}
Or a more compact version:
var data = {{q_some_query}};
delete data['_response']
var newdata = _.zip.apply(_, _.values(data)).map((val) => _.zipObject(_.keys(data), val));
After you convert to this format, you might want to convert back to the original format (for example, if you want to put the data into a table widget in Slate). This is how that can be done:
var original_fmt = {};
_.forEach(_.keys(newdata[0]), function(k) {
original_fmt[k] = _.map(newdata, k);
});
// use original_fmt
I am working with NodeJS using ExpressJS framework in a mysql backend. I am running a query inside a for loop and my loop and work afterwards depends on the return value of the query. I am not very good with mysql query so I am running it through a for loop.
The problem is, due asynchronous [I guess!], the for loop ends long before the query result comes out.
Here is my code:
function search_people_step2(user_id, search_criteria, user_friend)
{
var first_name_friends = [];
var last_name_friends = [];
for(var i = 0; i < user_friend.length; i++)
{
con.query("SELECT first_name, second_name FROM user WHERE userid = ?", user_friend[i],function(err, rows)
{
if(err)
{
//error;
}
else
{
if(rows.length == 0)
{
//nothing gets returned
}
else {
console.log(rows);
first_name_friends[i] = rows[0].first_name;
last_name_friends[i] = rows[0].second_name;
}
}
});
}
Now,I can get the value (using console.log) inside the query statement, however, on the outside, the value becomes empty (undefined) since the rest of the code has already been computed.
How can I solve this?
Thanks in advance.
The first thing that I find weird in your code is that you are not using an IN statement in your SQL query (not directly related to your problem though) which means you are making as many requests as there are entries in user_friend. The problem is that the SQL library is implemented asynchronously and you cannot avoid it. BUT you can handle it elegantly with Promises which are ES6 features:
(I didn't test the code but I think it should work)
function search_people_step2(user_id, search_criteria, user_friend)
{
return new Promise((resolve,reject)=>{
var first_name_friends = [];
var last_name_friends = [];
var placeHolders=user_friend.map(()=>"?").join(",");
con.query("SELECT first_name, second_name FROM user WHERE userid IN ("+placeHolders+")",user_friend,(err,rows)=>{
if(err)
reject(err);
else{
rows.forEach(row=>{
first_name_friends.push(row.first_name);
last_name_friends.push(row.second_name);
});
resolve({first_name_friends,last_name_friends});
}
});
});
}
And call your function like this :
search_people_step2(id,crit,friends).then(result=>{
//handle result asynchronously as there is no choice
console.log(result.first_name_friends);
console.log(result.last_name_friends);
}).catch(err=>{
//handle error
});
You are right, your problem is the asynchronous nature of the mysql call. You have to provide a callback to your search_people_step2 function.
You may change it like this:
search_people_step2(user_id, search_criteria, user_friend, callback)
In your function body you may use a library called async to handle all the callbacks properly. Here is an example for the usage:
async.eachSeries(user_friend, function(item, eachCb){
con.query("SELECT first_name, second_name FROM user WHERE userid = ?",
user_friend[i],function(err, rows) {
if(err) {
eachCb('error');
}
else {
if(rows.length == 0){
//nothing gets returned
eachCb(null);
}
else {
console.log(rows);
first_name_friends.push(rows[0].first_name);
last_name_friends.push(rows[0].second_name);
eachCb(null);
}
}
}, callback);
});
This calls each query in order on every item of the array and calls the inner callback if finished. When all items are processed or an error occured the outer callback is called. See the async library for further documentation.
simplest solution is
function search_people_step2(user_id, search_criteria, user_friend)
{
var first_name_friends = [];
var last_name_friends = [];
for(var i = 0; i < user_friend.length; i++)
{
con.query("SELECT first_name, second_name FROM user WHERE userid = ?", user_friend[i],function(err, rows)
{
if(err)
{
//error;
}
else
{
if(rows.length == 0)
{
//nothing gets returned
}
else {
console.log(rows);
first_name_friends[i] = rows[0].first_name;
last_name_friends[i] = rows[0].second_name;
}
if(i==user_friend.length-1){
//do your work here which you want to perform in end
}
}
});
}
or use async library
var async = require('async');
var first_name_friends = [];
var last_name_friends = [];
async.series([function(cb){
function search_people_step2(user_id, search_criteria, user_friend)
{
for(var i = 0; i < user_friend.length; i++)
{
con.query("SELECT first_name, second_name FROM user WHERE userid = ?", user_friend[i],function(err, rows)
{
if(err)
{
//error;
}
else
{
if(rows.length == 0)
{
//nothing gets returned
}
else {
console.log(rows);
first_name_friends[i] = rows[0].first_name;
last_name_friends[i] = rows[0].second_name;
}
if(i==user_friend.length-1){
cb()
}
}
});
}
},function(cb){
//do your work here
}],function(err){})
I want to make a dynamic graph based on a json file. I have seen many examples with tsv but I donot how to convert it to json.
That is the part that I want to change from tsv to json but I donot know how!
d3.tsv("data/data.tsv", function(error, data) {
data.forEach(function(d) {
d.date = parseDate(d.date);
d.close = +d.close;
});
when I use
d3.json("data/data.json", function(data) {
data.forEach(function d) {
d.date = parseDate(d.date);
d.close = +d.close;
}
});
it gives this error: Uncaught type error: cannot call method 'forEach' of undefined!
Thanks for your suggestions :)
try to do something like this
d3.json("data/data.json", function(data) {
data.forEach(function d) {
d.date = parseDate(d.date);
d.close = +d.close;
}
});
d3.js have support for json, https://github.com/mbostock/d3/wiki/Requests
The syntax around your forEach is a little off; try this instead:
d3.json("data/data.json", function(data) {
data.forEach(function(d) {
d.date = parseDate(d.date);
d.close = +d.close;
});
});
(As Felix points out, this will only work if your JSON object is defined and is an array)
Here a small code where you'll be able to convert tsv to json. It could help you...
ps : here is typescript, but you can easily convert it to vanilla javascript ;)
// Set bunch of datas into format object
tsvToJson(datas: string): Array<Object>{
// Separate each lines
let array_datas = datas.split(/\r\n|\r|\n/g);
// Separate each values into each lines
var detailed_datas = [];
for(var i = 0; i < array_datas.length; i++){
detailed_datas.push(array_datas[i].split("\t"));
}
// Create index
var index = [];
var last_index = ""; // If the index we're reading is equal to "", it mean it might be an array so we take the last index
for(var i = 0; i < detailed_datas[0].length; i++){
if(detailed_datas[0][i] == "") index.push(last_index);
else {
index.push(detailed_datas[0][i]);
last_index = detailed_datas[0][i];
}
}
// Separate data from index
detailed_datas.splice(0, 1);
// Format data
var formated_datas = [];
for(var i = 0; i < detailed_datas.length; i++){
var row = {};
for(var j = 0; j < detailed_datas[i].length; j++){
// Check if value is empty
if(detailed_datas[i][j] != ""){
if(typeof row[index[j]] == "object"){
// it's already set as an array
row[index[j]].push(detailed_datas[i][j]);
} else if(row[index[j]] != undefined){
// Already have a value, so it might be an array
row[index[j]] = [row[index[j]], detailed_datas[i][j]];
} else {
// It's empty for now, so let's say first that it's a string
row[index[j]] = detailed_datas[i][j];
}
}
}
formated_datas.push(row);
}
console.log(formated_datas); // #TODO : remove this
return formated_datas;
}
I transpile and resume Wetteren's code:
convertTSVtoJSON(tsvData) {
const formattedData = tsvData.split(/\r\n|\r|\n/g).filter(e => !!e).map((parsedEntry) => parsedEntry.split("\t"));
const tsvHeaders = formattedData.shift();
return formattedData.map(formattedEntry => {
{
return tsvHeaders.reduce((jsonObject, heading, position) => {
jsonObject[heading] = formattedEntry[position];
return jsonObject;
}, {});
}
});
}
I have a button on page - when clicked, it passes all the data to the servlet that could update each row data. My question is how to pass the whole store to the servlet as json data? Is there any easy way? Thanks
Here is some code I wrote to get the store to an object. Then it can be converted to JSON using dojo.toJson(obj);. I learned about this from the dojotoolkit website originally. (Give credit where credit is due). I realize this code is huge and nasty. When I looked for a better way about a year back I could not find one.
JsonHelper.storeToObject = function(store) {
var object = [];
var index = -1;
store.fetch({
onItem : function(item, request) {
object[++index] = JsonHelper.itemToObject(store, item);
}
});
return object;
};
JsonHelper.itemToObject = function(store, item) {
// store:
// The datastore the item came from.
// item:
// The item in question.
var obj = {};
if (item && store) {
// Determine the attributes we need to process.
var attributes = store.getAttributes(item);
if (attributes && attributes.length > 0) {
var i;
for (i = 0; i < attributes.length; i++) {
var values = store.getValues(item, attributes[i]);
if (values) {
// Handle multivalued and single-valued attributes.
if (values.length > 1) {
var j;
obj[attributes[i]] = [];
for (j = 0; j < values.length; j++) {
var value = values[j];
// Check that the value isn't another item. If
// it is, process it as an item.
if (store.isItem(value)) {
obj[attributes[i]].push(itemToObject(store,
value));
} else {
obj[attributes[i]].push(value);
}
}
} else {
if (store.isItem(values[0])) {
obj[attributes[i]] = itemToObject(store,
values[0]);
} else {
obj[attributes[i]] = values[0];
}
}
}
}
}
}
return obj;
};
I have a code to do some calculation.
How can I write this code in an asyn way?
When query the database, seems we can not get the results synchronously.
So how to implement this kind of feature?
function main () {
var v = 0, k;
for (k in obj)
v += calc(obj[k].formula)
return v;
}
function calc (formula) {
var result = 0;
if (formula.type === 'SQL') {
var someSql = "select value from x = y"; // this SQL related to the formula;
client.query(someSql, function (err, rows) {
console.log(rows[0].value);
// *How can I get the value here?*
});
result = ? // *How can I return this value to the main function?*
}
else
result = formulaCalc(formula); // some other asyn code
return result;
}
Its not possible to return the result of an asynchronous function, it will just return in its own function scope.
Also this is not possible, the result will always be unchanged (null)
client.query(someSql, function (err, rows) {
result = rows[0].value;
});
return result;
Put a callback in the calc() function as second parameter and call that function in the client.query callback with the result
function main() {
calc(formula,function(rows) {
console.log(rows) // this is the result
});
}
function calc(formula,callback) {
client.query(query,function(err,rows) {
callback(rows);
});
}
Now if you want the main to return that result, you also have to put a callback parameter in the main and call that function like before.
I advice you to check out async its a great library to not have to deal with this kind of hassle
Here is a very crude way of implementing a loop to perform a calculation (emulating an asynchronous database call) by using events.
As Brmm alluded, once you go async you have to go async all the way. The code below is just a sample for you to get an idea of what the process in theory should look like. There are several libraries that make handling the sync process for asynch calls much cleaner that you would want to look into as well:
var events = require('events');
var eventEmitter = new events.EventEmitter();
var total = 0;
var count = 0;
var keys = [];
// Loop through the items
calculatePrice = function(keys) {
for (var i = 0; i < keys.length; i++) {
key = keys[i];
eventEmitter.emit('getPriceFromDb', {key: key, count: keys.length});
};
}
// Get the price for a single item (from a DB or whatever)
getPriceFromDb = function(data) {
console.log('fetching price for item: ' + data.key);
// mimic an async db call
setTimeout( function() {
price = data.key * 10;
eventEmitter.emit('aggregatePrice', {key: data.key, price: price, count: data.count});
}, 500);
}
// Agregate the price and figures out if we are done
aggregatePrice = function(data) {
count++;
total += data.price;
console.log('price $' + price + ' total so far $' + total);
var areWeDone = (count == data.count);
if (areWeDone) {
eventEmitter.emit('done', {key: data.key, total: total});
}
}
// We are done.
displayTotal = function(data) {
console.log('total $ ' + data.total);
}
// Wire up the events
eventEmitter.on('getPriceFromDb', getPriceFromDb);
eventEmitter.on('aggregatePrice', aggregatePrice);
eventEmitter.on('done', displayTotal);
// Kick of the calculate process over an array of keys
keys = [1, 2, 3]
calculatePrice(keys);