sails.js run multiple command query in mysql - mysql

I have problem with executing multiple sql query on sails.js
I would like to run a script from a file on sails lift.
I wrote a custom handling inside /config/bootstrap.js
module.exports.bootstrap = function(cb) {
fs.readFile('SQL\\StoredProcedures\\MyProcedure.sql', 'utf8', function (err,data) {
if (err) {
console.log(err);
}
console.log(data);
MyModel.query(data, function(err, records){
if(err){
console.log(err);
}
});
});
// It's very important to trigger this callback method when you are finished
// with the bootstrap! (otherwise your server will never lift, since it's waiting on the bootstrap)
cb();
};
The problem is, that .query() function does not accept multiple queries inside. I mean, it does accept when in my file I have:
DROP PROCEDURE IF EXISTS `MyProcedure`;
But it won't accept while in my file I have:
DROP PROCEDURE IF EXISTS `MyProcedure`;
SELECT * FROM something;
Is there a way to execute this file?

This can be done by setting up your config/datastores.js like this:
module.exports = {
default: {
multipleStatements: true
}
}
By adding this into your configuration, you're letting Sails handle the parsing and execution of your queries.
The issue is that by default, the Node MySQL driver doesn't allow running multiple queries at once. This is to guard against SQL injection.
For more complete explanation, see #sgress454's comment here: https://github.com/balderdashy/sails/issues/4192

You could split the lines in your file, and run all the queries one by one?
var fs = require('fs');
module.exports = function (cb) {
fs.readFile('SQL\\StoredProcedures\\MyProcedure.sql', 'utf8', function (err,data) {
if (err) {
sails.log.error(err);
return cb(); // you have no queries to run
}
sails.log.info(data);
var queries = data.split('\n');
// async is injected into the global scope by sails, so no need to require it
// if you don't need your queries to run in order, then you can use async.each
// instead of async.eachSeries
// https://github.com/caolan/async#each
async.eachSeries(
queries,
function (query, cb) {
MyModel.query(query, function (err, records) {
if (err) {
sails.log.error(err);
return cb();
// If you don't want other queries to execute in case of an error, then
// return cb(err);
}
});
},
function (err) {
if (err) { sails.log.error(err); }
return cb();
}
);
});
};

Related

Why is my AWS Lambda node.js mysql query not returning?

I'm trying to write some external data into some local tables. We'll be looping through an array, writing most of the data in each array element to the main table and the rest to related tables, replacing all the data each time.
I've stripped the code down to the bare bones to show the problem I'm having. The DELETE runs fine, but the INSERT runs only once, and doesn't even return.
I have a screenshot of the output at https://imgur.com/a/zA6Hz8g .
In it, you can see that the code for the DELETE runs fine (ComQueryPacket sent, OkPacket returned) but when it gets to the INSERT, the ComQueryPacket is sent but nothing is returned. And then the code just falls through.
This results in the first row writing successfully, but no subsequent rows get written.
I've tried changing the connection to use pools, but that didn't help either.
Any ideas?
var mysql = require('mysql');
var promise = require('promise');
const con = mysql.createConnection({
<connectionInfo>,
debug: true
});
function connectToDB() {
return new promise((resolve, reject) => {
console.log("IN connectToDB");
con.connect(function(err) {
if (err) {
console.log("ERROR: Could not connect -- " + err);
reject;
}
console.log("Connected!");
resolve();
});
});
}
function deleteExistingMainRow() {
return new promise((resolve, reject) => {
var query = "DELETE FROM my_table";
con.query(query, [],
function(err, result) {
if (err) {
console.log("ERROR in deleteExistingMainRow: " + err);
reject;
}
else {
console.log("DEBUG: Successful delete of main row");
resolve();
}
});
});
}
function writeMainRow(data_row) {
return new promise((resolve, reject) => {
console.log("IN writeMainRow");
var query = 'INSERT INTO my_table SET id = ?';
con.query(query, [data_row.id],
function(err, result) {
console.log("YES we tried to query");
if (err) {
console.log("ERROR in writeMainRow: " + err);
reject(err);
}
else {
console.log("DEBUG: Successful write of main row");
resolve();
}
});
});
}
exports.handler = function(event, context) {
connectToDB().then(function(script) {
deleteExistingMainRow().then(function(script) {
var data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
data.forEach(data_row => {
writeMainRow(data_row).then(function(script) {
console.log("DEBUG: Main row written in forEach");
},
function(err) {
if (err) { console.log("ERR"); } process.exit(0);
}());
});
console.log("DEBUG: Hey we're exiting now");
con.commit;
con.end(function(err) {
console.log("Error on con end: " + err);
});
context.done(null, "DONE");
process.exit(0);
});
});
};
Just a few moths ago AWS made Node.js v 8.10 runtime available in lambda.
Which means, you can use async/await and Promises. So, we can rearrange code to something like this:
exports.handler = async (event, context) => {
const dbConnection = await connectToDB();
await deleteExistingMainRow();
const data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
// use here for...of loop to keep working with async/await behaviour
for(const data_row of data){
await writeMainRow(data_row);
}
}
Also, you can rewrite your code to use native Promises or async/await functions.
And of course, cover logic on try/catch block, I've skipped them for simplicity.
The reason why your code is not behaving as you expect is because of the asynchronous nature of NodeJS.
Your for_each loop spawns several threads that are going to INSERT the data in your database.
As soon as these threads are started, the rest of the code will execute, starting with console.log("DEBUG: Hey we're exiting now");
So the commit happens before all the INSERT calls are done and, more importantly, you're calling Process.exit() in your code. This terminates the runtime, even before the INSERT can finish.
Call callback() instead as per https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
Handling your multiple asynchronous writes can be done differently. First, as grynets commented before me, I would strongly suggest to rewrite your code using async/await to make the call easier to read.
Then, you have to understand that each call to writeMainRow will return its own Promise and your code must wait for ALL promises to complete before to commit() and to callback()
Promise.all(...) will do that for you. See the doc at https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
try using
INSERT INTO table_name(id) VALUES (?);
I know both your query and the above query works the same. Just give it a Try.
And just make sure your for loop is working properly sending values to the writeMainRow(function). It wouldnt show an error you if pass an empty value and make sure you are not passing the same values in the for loop. And i think you have to pass writeMainRow(data_row.id) rather than writeMainRow(data_row).
Hope this helps.
And one more suggestion if you are updating multiple rows there are options in mysql node library like transactions. Using those functions will be more effective and you can roll back the result if you face error. Other option is to write procedures, in which case you mysql server will bear the computation.

Create databases connection on startup - sails.js

I know you can use Waterline and the models approach, but the issue is that I have to access more than one database to generate a response. Besides the models approach makes it difficult to think on a solution due to the fact that the data is joined and infered from different tables under the different databases.
Therefore, I would like to know how can I open a DB connection using the mysql or mysql2 native driver and reuse it all over the app. Where is the most suitable place, a hook, etc.? How can I close them when the app goes down?
TA
A custom hook will be the best for you. For Redis, I created one like this:
api/hooks/redis/index.js
var redisModule = require('redis');
module.exports = function connectToRedis(sails) {
return {
connectToRedis: function (callback) {
var hook = this;
var config = sails.config.connections.redis;
var redisClient = redisModule.createClient(config.port, config.host);
hook.initAdapters();
redisClient.on('connect', function () {
sails.log.verbose('Connection to redis was succesfull!');
// select db
redisClient.select(config.db, function (err) {
if (err) {
return callback(err);
}
sails.adapters.redis = redisClient;
callback();
});
});
redisClient.on('error', function (error) {
sails.log.error(error);
callback();
});
},
initAdapters: function () {
if (sails.adapters === undefined) {
sails.adapters = {};
}
},
// Run automatically when the hook initializes
initialize: function (cb) {
var hook = this;
hook.connectToRedis(function() {
cb();
});
},
};
};

How to promisify a MySql function using bluebird?

Some time ago I decided to switch from PHP to node. In my first projects I didn't want to use any ORM since I thought that I didn't need to complicate my life so much learning another thing (at the moment I was learning node and angular) therefor I decided to use mysql package without anything else. It is important to say that I have some complex queries and I didn't want to learn from sctratch how to make them work using one of the 9000 ORM node have, This is what I've been doing so far:
thing.service.js
Thing.list = function (done) {
db.query("SELECT * FROM thing...",function (err,data) {
if (err) {
done(err)
} else {
done(null,data);
}
});
};
module.exports = Thing;
thing.controler.js
Thing = require('thing.service.js');
Thing.list(function (err,data) {
if (err) {
res.status(500).send('Error D:');
} else {
res.json(data);
}
});
how can I promisify this kind of functions using bluebird ? I've already tried but .... here I am asking for help. This is what I tried
var Thing = Promise.promisifyAll(require('./models/thing.service.js'));
Thing.list().then(function(){})
I have done this way and it is working fine.
const connection = mysql.createConnection({.....});
global.db = Bluebird.promisifyAll(connection);
db.queryAsync("SELECT * FROM users").then(function(rows){
console.log(rows);});
I have never had much luck with promisifyAll and IMO I prefer to handle my internal checks manually. Here is an example of how I would approach this:
//ThingModule
var Promises = require('bluebird');
Things.list = function(params) {
return new Promises(function(resolve, reject) {
db.query('SELECT * FROM thing...', function(err, data) {
return (err ? reject(err) : resolve(data));
});
});
}
//usage
var thinger = require('ThingModule');
thinger.list().then(function(data) {
//do something with data
})
.error(function(err) {
console.error(err);
})
You can also create a function that fires SQL like this :-
function sqlGun(query, obj, callback) {
mySQLconnection.query(query, obj, function(err, rows, fields) {
if (err) {
console.log('Error ==>', err);
// throw err;
return (err, null);
}
console.log(query)
if (rows.length) {
return callback(null, rows);
} else {
return callback(null, [])
}
});
}
Where mySQLconnection is the connection object you get after mysql.createConnection({}).
After that, you can promisify the function and use the promise like below :-
var promisified = Promise.promisify(sqlGun);
promisified(query, {}).then( function() {} );

Node.js returning result from MySQL query

I have the following function that gets a hexcode from the database
function getColour(username, roomCount)
{
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err) throw err;
return result[0].hexcode;
});
}
My problem is that I am returning the result in the callback function but the getColour function doesn't return anything. I want the getColour function to return the value of result[0].hexcode.
At the moment when I called getColour it doesn't return anything
I've tried doing something like
function getColour(username, roomCount)
{
var colour = '';
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err) throw err;
colour = result[0].hexcode;
});
return colour;
}
but of course the SELECT query has finished by the time return the value in colour
You have to do the processing on the results from the db query on a callback only. Just like.
function getColour(username, roomCount, callback)
{
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err)
callback(err,null);
else
callback(null,result[0].hexcode);
});
}
//call Fn for db query with callback
getColour("yourname",4, function(err,data){
if (err) {
// error handling code goes here
console.log("ERROR : ",err);
} else {
// code to execute on data retrieval
console.log("result from db is : ",data);
}
});
If you want to use promises to avoid the so-called "callback hell" there are various approaches.
Here's an example using native promises and the standard MySQL package.
const mysql = require("mysql");
//left here for testing purposes, although there is only one colour in DB
const connection = mysql.createConnection({
host: "remotemysql.com",
user: "aKlLAqAfXH",
password: "PZKuFVGRQD",
database: "aKlLAqAfXH"
});
(async () => {
connection.connect();
const result = await getColour("username", 2);
console.log(result);
connection.end();
})();
function getColour(username, roomCount) {
return new Promise((resolve, reject) => {
connection.query(
"SELECT hexcode FROM colours WHERE precedence = ?",
[roomCount],
(err, result) => {
return err ? reject(err) : resolve(result[0].hexcode);
}
);
});
}
In async functions, you are able to use the await expression which will pause the function execution until a Promise is resolved or rejected. This way the getColour function will return a promise with the MySQL query which will pause the main function execution until the result is returned or a query error is thrown.
A similar but maybe more flexible approach might be using a promise wrapper package of the MySQL library or even a promise-based ORM.

How to structure Node.js script with MySQL so that connection doesn't close prematurely

So I am making my first attempt with Node and I can't really wrap my head around how to work with the MySQL connection. The script is somewhat simplified like this
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '192.168.40.1',
user : 'user',
password : 'password',
database : 'database'
});
function DoSomething(connection, item, callback) {
connection.query(
'SELECT COUNT(*) AS count FROM another_table WHERE field=?',
item.field,
function (err, results) {
if (err) throw err;
if (results.length > 0 && results[0].count >= 1) {
callback(err, connection, item, 'Found something')
}
});
}
function DoSomethingElse(connection, item, callback) {
// Similar to DoSomething()
}
function StoreResult(err, connection, item, reason) {
if (err) throw err;
connection.query(
'INSERT INTO result (item_id, reason) VALUES (?, ?)',
[item.id, reason],
function (err, results) {
if (err) throw err;
});
}
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
});
connection.end();
What I am having trouble with (as far as I can tell) is that since DoSomething() it seems that connection.end() is called before all of the DoSomething()'s have finished causing errors that queries can't be performed when the connection is closed.
I tried playing around with the async library, but I haven't gotten anywhere so far. Anyone with some good advice on how to do this?
The problem with your code is that you're closing the connection synchronously while an asynchronous request is still being handled. You should call connection.end() only after all query callbacks have been called.
Since you are doing multiple queries, this means using some way to wait for all their results. The simplest way is to nest every next call into the callback of the previous one, but that way leads to the pyramid of doom. There are a number of popular libraries that solve this, but my own preference is for async.
Using async I would rewrite your code as follows:
async.waterfall([function(next) {
connection.query('SELECT * FROM table WHERE deleted=?', [0], next); // note the callback
},
function(results, next) {
// asynchronously handle each results. If they should be in order, use forEachSeries
async.forEach(results, function(item, next) {
// handle in parallel
async.parallel([function(done) {
DoSomething(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
});
}, function(done) {
DoSomethingElse(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
}], function(err) {
// this callback is called with an error as soon as it occurs
// or after all callbacks are called without error
next(err);
});
}, function(err) {
// idem
next(err);
});
}], function(err, results) {
// idem
// Do whatever you want to do with the final error here
connection.end();
});
This also allows you to solve a possible issue with the order of your queries in the forEach: They are started in order, but are not guaranteed to finish in order due to their asynchronous nature.
Close your connection after you have done everything you want in the script.
When programming in asynchronous language, keep in mind that the real ending point of your script is the last asynchronous callback, instead of the last line like other scripts (e.g. PHP).
Note that you don't want to simply ignore the connection.end(); as the underlying MySQL driver will keep the connection alive and your script will stuck in the last line forever until you kill it.
This is the modified version of your code.
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
// End your connection here
connection.end();
});