I'm trying to introduce the simplest caching layer in my webapp, all I want to do is cache the results from a few queries for 24 hours as that is how often the DB receieves new data. I'm trying to use node-cache-manager (which looks great) but struggling! I don't think I completely understand how it should be implemented into sequelize. The example is using Mongoose and Mongo by the looks of it:
function responder(res) {
return function respond(err, data) {
var startTime = moment(res.req._startTime);
var diff = moment().diff(startTime, 'ms');
if (err) {
err.status = 500;
res.render('error', {error: err});
} else {
data.requestTime = diff;
res.render('users/show', data);
}
};
}
function fetchUser(id, cb) {
var cacheKey = 'user_' + id;
memoryCache.wrap(cacheKey, function (cacheCb) {
console.log("Fetching user from slow database");
User.get(id, cacheCb);
}, cb);
}
router.get('/:id', function (req, res) {
fetchUser(req.param('id'), responder(res));
});
I'm using MySQL and currently have the example below, but I can see the query hitting the database every time, it does not seem to be persisting or creating the cache...:
function responder(res) {
return function respond(err, data) {
var startTime = moment(res.req._startTime);
var diff = moment().diff(startTime, 'ms');
if (err) {
err.status = 500;
res.json({valuesCount: err});
} else {
data.requestTime = diff;
res.json({valuesCount: res});
}
};
}
function fetchCount(req, cb) {
var cacheKey = 'allDatabaseRecords',
table = sequelize.import('extracts');
memoryCache.wrap(cacheKey, function (cacheCb) {
console.log("Fetching count from slow database");
table.count().then(cacheCb);
}, cb);
}
router.post('/extract-tool/data-filter', function(req, res) {
var table = sequelize.import('extracts');
fetchCount(req, responder(res));
});
I've tried following the documentation and the example, but this as mentioned, this still hits the DB every time! Any help is MASSIVELY appreciated!!!
I could not get this to work as per the examples given, I had to specifically set and get the key value pair to and from the cache each time.
Now it works fine, structured as below:
memoryCache.get(cacheKey, function(err, result) {
if (result !== undefined){res.json({billingAddress12LastName: result});}
});
memoryCache.wrap(cacheKey, function (cacheCb) {
console.log("Fetching count from slow database");
table.count().then(data);
memoryCache.set(cacheKey, data, {ttl: ttl}, function(err) {
res.json({valuesCount: data});
});
}, cb);
Still don't understand how the wrap function works and what its for as I'm explicitly setting and getting the cache values now, would appreciate some comments still :).
Related
I am making a graph based web progrom. I am using nodejs.
I have a list of keys calls map which stores ID of vertices.
I want to retrieve the name of these vertices from MySQL using the ID. I have found a solution but I am not sure if it will work every time. This is my code.
for(var i=0;i<map.length;++i){
con.query('SELECT * FROM station WHERE id='+map[i],function(err,result,field){
if(err)
console.log("ERROR 3");
else{
result.forEach(function(r){
stationName.push(r.name);
})
if(stationName.length==map.length){
console.log(stationName);
res.render('route/showroute.ejs',{stationName: stationName});
}
}
})
}
I was wonder is it possible that my last map query loads before other query which may cause station name to be stored in wrong order. I am new to javascript.
You can do this using async await function
var getDataById = function (id) {
return new Promise(function (resolve, reject) {
con.query('SELECT * FROM station WHERE id=' + id, function (err, result, field) {
if (err) {
console.log("ERROR 3");
reject(err);
} else {
resolve(result);
}
});
});
};
(async function () {
for (var i = 0; i < map.length; ++i) {
var data = await getDataById(map[i]);
data.forEach(function (r) {
stationName.push(r.name);
});
if (stationName.length == map.length) {
console.log(stationName);
}
}
})();
You are worried about the query-function being asynchronous, and it is, but only with regard to what comes after the function. You're putting your stationNames into a new array inside the callback function which will be executed in sequence.
Am trying to retrieve all the database records from a table called post using node js but the problem is that only one record is retrieved instead of all.
In php I can use while() loop to loop through the database record to get all data.
Currently, I do not know how to neatly loop through the database in nodejs to get all the records from database. Some Stackoverflow scholars suggest using await/async method but i do not know to to implement it on the code below to make it work. can someone help me fix the issue.
var connection = require('./config');
module.exports.getpost = function (req, res) {
connection.query('SELECT * FROM posts', function (error, results, fields) {
if (error) {
console.log('error');
res.json({
status : false,
message : 'there are some error with the query'
});
} else {
var postid = results[0].id;
var title = results[0].title;
var content = results[0].content;
var type = -1;
console.log(title);
// Checking user status
connection.query('SELECT count(*) as cntStatus,type FROM like_table WHERE userid= ? and postid=?', [userid,postid], function (error, results, fields) {
if (error) {
console.log('error');
res.json({
status : false,
message : 'there are some error with the query'
});
} else {
var total_count = results[0].cntStatus;
if(total_count > 0){
type = results[0].type;
}
var total_count = results[0].cntStatus;
var result = {
"id" : postid,
"title" : title,
"content" : content,
"type" : type,
"likes" : total_count
};
console.log('query okay');
res.json({
//data:results,
data : result
});
}
});
}
});
}
I'm assuming you're using mysql npm. In that case I'm not sure what is the problem in your case. Results param is an array of rows returned by your select statement. So you can use loop to iterate trough all the rows.
You don't actually need to use async/await (which doesn't have any advantage in terms of functionality but looks cleaner). But if you want to get rid of callbacks you need to wrap connection query into a promise or use mysql2 npm which has promise interface. Here is how you can iterate trough all the rows from your select using async/await instead of callback:
var connection = require('./config');
module.exports.getpost = async function (req, res) {
try {
const queryResult = await query('SELECT * FROM posts');
queryResult.forEach(row => {
console.log(row.title);
})
} catch (err) {
console.log('error');
res.json({
status: false,
message: 'there are some error with the query'
});
}
}
Please note that you need to use nodejs 8 to run the code with async/await.
Also you don't need to do another query inside of your posts query, you can merge those two using SQL join
async waterfall - Runs an array of functions in series, each passing their results to the next in the array. However, if any of the functions pass an error to the callback, the next function is not executed and the main callback is immediately called with the error.
var connection = require('./config');
var async = require('async');
module.exports.getpost = function (req, res) {
var arrayOfFuncs = [];
var func_1 = function(callback) {
connection.query('SELECT * FROM posts', function (error, results, fields) {
if (error) {
console.log('error');
callback(error, null);
} else {
var toPass = {};
toPass.postid = results[0].id;
toPass.title = results[0].title;
toPass.content = results[0].content;
toPass.type = -1;
callback(null, toPass);
}
})
}
arrayOfFuncs.push(func_1);
var func_2 = function(prevData, callback) {
connection.query('SELECT count(*) as cntStatus,type FROM like_table WHERE userid= ? and postid=?', [userid,prevData.postid], function (error, results, fields) {
if (error) {
console.log('error');
callback(error, null);
} else {
var total_count = results[0].cntStatus;
if(total_count > 0){
type = results[0].type;
}
var total_count = results[0].cntStatus;
var result = {
"id" : postid,
"title" : title,
"content" : content,
"type" : type,
"likes" : total_count
};
console.log('query okay');
callback(null, result);
}
});
}
arrayOfFuncs.push(func_2);
async.waterfall(arrayOfFuncs, function(errString, finalResult) {
if(errString) {
return res.send(errString);
} else {
return res.send(finalResult);
}
});
}
I'm quite new to NodeJS and JS globally and I'm in trouble while setting and Object Property through a MySQL query.
I'm using Promise to avoid bad asynchronous effect but apparently I'm doing it wrong, the property of my Agent Obejct is never updated.
Here is the code :
class Agent {
constructor(agentId, agentName, agentCountry) {
this.agentId = agentId;
this.agentName = agentName;
this.agentCountry = agentCountry;
}
setAgentCountry () {
var promise = function(agentID) {
return new Promise(function(resolve, reject) {
var query = "SELECT c.CountryID, c.CountryName FROM AgentCountry ac, Country c WHERE ac.AgentID = '" + agentID + "' AND ac.CountryID = c.CountryID";
connection.query(query, function(err, results) {
if (!err) {
resolve(results);
} else {
console.log('Error while performing Query.');
}
});
});
}
promise(this.agentID).then(function(data) {
var string = JSON.stringify(data);
var json = JSON.parse(string);
//the agent property is never updated !!
this.agentCountry = json;
}.bind(this), function(err) {
console.log(err);
});
}
}
I call the method this way :
var agent = new Agent(1,"John Doe", "France");
console.log(agent.agentCountry); //Displays "France"
agent.setAgentCountry();
console.log(agent.agentCountry); //Did not display the table of countries it should
Could you help me with this ?
Thanks
The main problem is that console.log is being executed before the promise being resolved. Writing a console.log inside the "then" clause will show you the timing.
The promise will be resolved or rejected eventually but nobody is waiting for setAgentCountry.
There are several points of order here:
A promise must always be either (1) resolved or (2) rejected. Your error case logs it to the console without calling reject(), so it's stuck in promise limbo for forever when it errors.
Why do you name a variable, promise, the same as the library, Promise?
I think you will find it more modular to just wrap the mysql_conn.query() callback into a promise():
const mysql_conn = mysql.createConnection({
host: mysql_conf.host,
user: mysql_conf.user,
password: mysql_conf.password
});
mysql_conn.queryPromiser = function(sql, args) {
return new Promise(function(resolve, reject) {
mysql_conn.query(
sql,
args,
function(err, results, fields) {
if (err) {
reject(err);
} else {
resolve( {"results": results, "fields": fields} );
}
}
);
});
};
then you can use it like so:
class Agent {
constructor(agentId, agentName) {
this.agentId = agentId;
this.agentName = agentName;
this.agentCountry = null;
}
configureCountryPromiser() {
var sql = "SELECT country FROM agent_countries WHERE agent_id = ?";
var args = [ this.agentId ];
var that = this;
return mysql_conn.queryPromiser(sql, args)
.then(function(data) {
if (data.results.length) {
that.agentCountry = data.results[0].country;
} else {
// handle case where agent_id is not found in agent_countries
}
});
}
};
agent_instance = new Agent(1, "Benoit Duprat");
agent_instance.configureCountryPromiser()
.then(function() {
console.log("agent country configured to ", agent_instance.agentCountry);
}).catch(console.error);
Please note that I have not tested the class code, but the general idea should be enough to answer your question.
Some time ago I decided to switch from PHP to node. In my first projects I didn't want to use any ORM since I thought that I didn't need to complicate my life so much learning another thing (at the moment I was learning node and angular) therefor I decided to use mysql package without anything else. It is important to say that I have some complex queries and I didn't want to learn from sctratch how to make them work using one of the 9000 ORM node have, This is what I've been doing so far:
thing.service.js
Thing.list = function (done) {
db.query("SELECT * FROM thing...",function (err,data) {
if (err) {
done(err)
} else {
done(null,data);
}
});
};
module.exports = Thing;
thing.controler.js
Thing = require('thing.service.js');
Thing.list(function (err,data) {
if (err) {
res.status(500).send('Error D:');
} else {
res.json(data);
}
});
how can I promisify this kind of functions using bluebird ? I've already tried but .... here I am asking for help. This is what I tried
var Thing = Promise.promisifyAll(require('./models/thing.service.js'));
Thing.list().then(function(){})
I have done this way and it is working fine.
const connection = mysql.createConnection({.....});
global.db = Bluebird.promisifyAll(connection);
db.queryAsync("SELECT * FROM users").then(function(rows){
console.log(rows);});
I have never had much luck with promisifyAll and IMO I prefer to handle my internal checks manually. Here is an example of how I would approach this:
//ThingModule
var Promises = require('bluebird');
Things.list = function(params) {
return new Promises(function(resolve, reject) {
db.query('SELECT * FROM thing...', function(err, data) {
return (err ? reject(err) : resolve(data));
});
});
}
//usage
var thinger = require('ThingModule');
thinger.list().then(function(data) {
//do something with data
})
.error(function(err) {
console.error(err);
})
You can also create a function that fires SQL like this :-
function sqlGun(query, obj, callback) {
mySQLconnection.query(query, obj, function(err, rows, fields) {
if (err) {
console.log('Error ==>', err);
// throw err;
return (err, null);
}
console.log(query)
if (rows.length) {
return callback(null, rows);
} else {
return callback(null, [])
}
});
}
Where mySQLconnection is the connection object you get after mysql.createConnection({}).
After that, you can promisify the function and use the promise like below :-
var promisified = Promise.promisify(sqlGun);
promisified(query, {}).then( function() {} );
Completely new to nodejs, having trouble wrapping my head around asynchronous programming/callbacks. What I'm trying to do is:
On 'post', I want to gather all the words in a table in my database. I call it like so: var lesson_data = init_load_lesson(); This call to init_load_lesson() is from index.js in my 'routers' file made my express.
How do I construct a proper callback so that lesson_data is the results of my query?
var mysql = require('mysql');
var connection = require('./connection');
var data = [];
function init_load_lesson()
{
connection.connect();
var queryString = "SHOW COLUMNS FROM LessonOneVocabulary";
connection.query(queryString, function(err, rows, fields) {
if (err) throw err;
else
{
for (var i in rows)
{
data.push(rows[i].Fields);
console.log(data);
}
console.log(data);
}
});
connection.end();
}
module.exports = function() {
load_lesson();
};
To add a callback (try a few more functions and you'll get the hang of it):-
function init_load_lesson(callback) {
... // connect to database
if (err) {
callback(err);
} else {
// process item
callback(null, data);
}
}
init_load_lesson(function(err2, results) {
if (err2) {
// process error
} else {
// process results
}
});