I want to be able to only save specific data from the output of the API...
function getGiph() {
giphy.search({
q: 'pratt',
}, function (err, res) {
var gif = res.data;
for (var i = 0; i < gif.length; i++) {
var embed = gif[i].embed_url;
fs.writeFile('./apidata.json', JSON.stringify(embed, undefined, 1), function (err) {
if (err) throw err;
console.log('saved');
});
}
});
}
This is my code, when I console.log(embed); it outputs only the embed_urls of the API in the terminal just like I want, but when I write to the JSON it outputs "https://giphy.com/embed/XqOCxN1WpEzja"MsM0"" and that is all...
If I replace 'embed' with 'gif' in the fs.writeFile(), then of course it writes the entire API data to the JSON.
I have tried JSON.stringify(gif, undefined, 1),
JSON.stringify(embed, undefined, 1).
I honestly searched and searched I hardly ever ask questions because I almost always find a solution but this one has me, I am pretty new to all this as well.
Related
I am having trouble storing a PDF content to MySQL as BLOB and retrieving it back from MySQL and stream it to front end using Node.JS.
I am able to save something to the DB (but not sure if I am saving the correct PDF content in the right manner). When I retrieve the pdf data, it does not open in PDF reader. I tried writing the retrieved data to a file and it also does not seem to work. Clear that either I am not storing the correct data to MySQL or I am not retrieving it in the correct way.
Could someone please help me out if you have come across similar scenario.
Code has been attached for your reference.
Many thanks.
var PDFDocument = require('pdfkit');
var doc = new PDFDocument;
var buffers = [];
doc.on('data', function(buff) {
buffers.push(buff);
});
doc.on('end', function () {
var queryToExec = "insert pdf_contents SET ? ";
var values = {
id : 0,//Auto Incrementing column
pdf: buffers.toString()
};
mysqlConnection.query(queryToExec, values, function(err, rows) {
if(!err) {
console.log("Stored to db successfully");
var selQuery = "select pdf from pdf_contents order by id desc limit 1"
mysqlConnection.query(selQuery, function(err_2, rows_2) {
if(!err_2) {
var myPdfDoc = rows_2[0].pdf;
console.log("Retrieved successfully, but pdf could not be opened!");
} else {
console.log("Error : ", err_2.toString());
}
});
} else {
console.log("Error : ", err.toString());
}
});
});
doc.addPage().text('Some text...', 100, 100);
doc.end();
I have trouble moving certain code outside a test into a function that needs to return a value.
Here is part of my code for the test file
function getCountOfTopics(browser){
var count;
browser.getText('#sumTopics',
function(result){
count = result.value;
console.log(result.value);
}
);
return count;
};
module.exports = {
'Create article' : function(browser){
var noOfThreadsByInlineCode, noOfThreadsByFunction;
browser.getText('#sumTopics',
function(result){
noOfThreadsByInlineCode = result.value;
}
);
noOfThreadsByFunction = getCountOfTopics(browser);
browser.end();
}
}
Now, the variable noOfThreadsByInlineCode indeed gets the value in the DOM, but the variable noOfThreadsByFunction is undefined. The console does indeed print the correct value, so the function does get the correct value out of the DOM.
I would appreciate help in updating the function so that I do get the value returned.
One word answer is Asynchronisity. The code doesn't wait for your callback to get complete, thats what the feature of Node JS is.
If you are in desperately in need for the content inside of the callback you can write this variable into a file and then access it anywhere you want inside your code. Here's a bit of a workaround:
Save something in a file:
var fs = require('fs');
iThrowACallBack(function(response){
fs.writeFile('youCanSaveData.txt', this.response, function(err) {
if (err) throw err;
console.log('Saved!');
browser.pause(5000);
});
});
Access it somewhere else:
iAccessThefile(){
response = fs.readFileSync('youCanSaveData.txt').toString('utf-8');
}
Hope it helps.
You return variable 'count' outside the callback,that is why.You can take a look this topic How to return value from an asynchronous callback function?
function getCountOfTopics(browser){
var count;
browser.getText('#sumTopics',
function(result){
count = result.value;
console.log(result.value);
/// result.value is available in this callback.
}
);
What do you want to do with the 'value'?
ps:do not remember custom_command.I think it is very helpful for this issue.
I wanna know if it's possible do multiple res.send/json in same method. I have a big problem with that because I wanna develop a function into a Web-Worker to call a put request each minute but in 2nd request I get "can't set headers after they are sent.". I know that it's not possible do it but I wanna know if exist some way to run.
exports.update = function (req, res) {
var feed = req.feed;
feed.title = req.body.title;
feed.apifeed = req.body.apifeed;
feed.apikey = req.body.apikey;
feed.active = req.body.active;
if(feed.apifeed && feed.apikey && feed.active){
var t = Threads.create();
t.eval(setInterval(
function(){
async.parallel([
function(callback, data){
var url = 'https://api.xively.com/v2/feeds/' + feed.apifeed + '.json?key=' + feed.apikey;
sensordata.getSensorData(url, function(data){
callback(null, data);
});
}
], function(err, data){
feed.content[0].value = data[0][0].value;
feed.content[0].date = new Date();
feed.save(function (err) {
if (err) {
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.json(feed);
}
});
});
}, 5000)
);
}
};
Thanks for supporting and I wait your responses. Greetings!
You cannot respond to a request more than once, that wouldn't make sense.
If you need to stream data, then you will need to use long polling, WebSockets, Server-sent Events, etc.
Also, assuming Threads.create() does what I think it may be doing, spawning an OS thread just to do what you're currently doing is a waste of resources. Nothing in that block of code is CPU-bound.
I'm currently working on a project that involves querying yahoo-finance for many different ticker symbols. The bottleneck is acquiring the data from yahoo, so I was wondering if there is a way I might go about speeding this up.
If I used multiple machines to query and then aggregated the data, would that help? I only have one physical machine; how might I go about doing that?
Thanks!
EDIT: Currently, I'm using Node.js, yahoo-finance, and Q.deferred to ask yahoo for historical data. Then, once all the promises are fulfilled (for each ticker), I'm doing a Q.all() to persist the data.
var data = [];
tickers = ["goog", "aapl", ...];
...
Q.all(_.map(tickers, function(symbol) {
return getYahooPromise(symbol);
}))
.done( function() { persistData(data) });
getYahooPromise retrieves data for the ticker symbol and pushes it into the data array. Once all promises are resolved, the data is persisted in a MySQL database.
SECOND EDIT:
More code:
var sequentialCalls = [];
for ( var i = 0; i < tickers.length / chunkSize; i++ ) {
sequentialCalls.push( persistYahooChunk );
}
sequentialCalls.push( function(callback) {
connection.end();
callback();
});
async.series( sequentialCalls )
exports.persistYahooChunk = function(callback) {
console.log("Starting yahoo query");
var currentTickers = tickers.slice(currentTickerIndex,currentTickerIndex + chunkSize);
return yahooFinance.historical( {
symbols: currentTickers,
from: "2015-01-28",
to: "2015-02-05"
}).then( function(result) {
console.log("Query " + currentTickerIndex + "/" + tickers.length + "completed");
currentTickerIndex += chunkSize;
//add valid data
var toPersist = _.map(result, function(quotes, symbol) {
return [symbol, quotes.length != 0 ];
});
var query = "INSERT INTO `ticker` (`symbol`, `valid`) VALUES ?";
connection.query(query, [toPersist], function(err, result) {
if (err) {
console.log (err);
}
//console.log(result);
callback();
});
});
}
The bottleneck is because you are doing one query per ticker.
Depending on the data you need to pull, if you could do a single query that includes all your tickers it would be much faster.
Here is an example if you need to get all current prices for a list of tickers, with a single query :
http://finance.yahoo.com/webservice/v1/symbols/A,B,C,D,E/quote?format=json
I'm pretty new to coding so forgive me if my code is unreadable or my question simplistic.
I am trying to create a little server application that (amongst other things) displays the properties of a neo4j node. I am using node.js, Express and Aseem Kishore's Node-Neo4j REST API client, the documentation for which can be found here.
My question stems from my inability to fetch the properties of nodes and paths. I can return a node or path, but they seem to be full of objects with which I cannot interact. I poured through the API documents looking for some examples of how particular methods are called but I found nothing.
Ive been trying to call the #toJSON method like, "db.toJSON(neoNode);" but it tells me that db does not contain that method. I've also tried, "var x = neoNode.data" but it returns undefined.
Could someone please help me figure this out?
//This file accepts POST data to the "queryanode" module
//and sends it to "talkToNeo" which queries the neo4j database.
//The results are sent to "resultants" where they are posted to
//a Jade view. Unfortuantly, the data comes out looking like
// [object Object] or a huge long string, or simply undefined.
var neo4j = require('neo4j');
var db = new neo4j.GraphDatabase('http://localhost:7474');
function resultants(neoNode, res){
// if I console.log(neoNode) here, I now get the 4 digit integer
// that Neo4j uses as handles for nodes.
console.log("second call of neoNode" + neoNode);
var alpha = neoNode.data; //this just doesn't work
console.log("alpha is: " +alpha); //returns undefined
var beta = JSON.stringify(alpha);
console.log("logging the node: ");
console.log(beta);// still undefined
res.render("results",{path: beta});
res.end('end');
}
function talkToNeo (reqnode, res) {
var params = {
};
var query = [
'MATCH (a {xml_id:"'+ reqnode +'"})',
'RETURN (a)'
].join('\n');
console.log(query);
db.query(query, params, function (err, results) {
if (err) throw err;
var neoNode = results.map(function (result){
return result['a']; //this returns a long string, looks like an array,
//but the values cannot be fetched out
});
console.log("this is the value of neoNode");
console.log(neoNode);
resultants(neoNode, res);
});
};
exports.queryanode = function (req, res) {
console.log('queryanode called');
if (req.method =='POST'){
var reqnode = req.body.node; //this works as it should, the neo4j query passes in
talkToNeo(reqnode, res) //the right value.
}
}
EDIT
Hey, I just wanted to answer my own question for anybody googling node, neo4j, data, or "How do I get neo4j properties?"
The gigantic object from neo4j, that when you stringified it you got all the "http://localhost:7474/db/data/node/7056/whatever" urls everywhere, that's JSON. You can query it with its own notation. You can set a variable to the value of a property like this:
var alpha = unfilteredResult[0]["nodes(p)"][i]._data.data;
Dealing with this JSON can be difficult. If you're anything like me, the object is way more complex than any internet example can prepare you for. You can see the structure by putting it through a JSON Viewer, but the important thing is that sometimes there's an extra, unnamed top layer to the object. That's why we call the zeroth layer with square bracket notation as such: unfilteredResult[0] The rest of the line mixes square and dot notation but it works. This is the final code for a function that calculates the shortest path between two nodes and loops through it. The final variables are passed into a Jade view.
function talkToNeo (nodeone, nodetwo, res) {
var params = {
};
var query = [
'MATCH (a {xml_id:"'+ nodeone +'"}),(b {xml_id:"' + nodetwo + '"}),',
'p = shortestPath((a)-[*..15]-(b))',
'RETURN nodes(p), p'
].join('\n');
console.log("logging the query" +query);
db.query(query, params, function (err, results) {
if (err) throw err;
var unfilteredResult = results;
var neoPath = "Here are all the nodes that make up this path: ";
for( i=0; i<unfilteredResult[0]["nodes(p)"].length; i++) {
neoPath += JSON.stringify(unfilteredResult[0]['nodes(p)'][i]._data.data);
}
var pathLength = unfilteredResult[0].p._length;
console.log("final result" + (neoPath));
res.render("results",{path: neoPath, pathLength: pathLength});
res.end('end');
});
};
I would recommend that you look at the sample application, which we updated for Neo4j 2.0
Which uses Cypher to load the data and Node-labels to model the Javascript types.
You can find it here: https://github.com/neo4j-contrib/node-neo4j-template
Please ask more questions after looking at this.