Node.JS and MySQL - queries lock up and execute extremely slowly - mysql

I am getting strange behavior using Node.JS and MySQL with this driver - https://github.com/mysqljs/mysql
Essentially, I have a button on the frontend that triggers an app.get that makes a query in the database and I can happily use the results in my backend.
This works nicely, until I press the button 4-5 times in a second, where as the queries lock up and I have to wait for 2-3 minutes until they continue executing. I have a similar write function that behaves the same way.
Is it possible this is a problem, because I'm trying to execute the exact same query asynchronously? I.e. do I have to limit this from the front end or is it a backend problem?
Any ideas on how to debug what exactly is going on?
// database.js
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit: 100,
host : 'localhost',
user : 'secret',
password : 'secret',
database : 'mydb'
});
exports.getConnection = function(callback) {
pool.getConnection(function(err, connection) {
callback(err, connection);
});
};
// dbrw.js
var con = require('../config/database');
function read(id, done) {
con.getConnection(function(err, connection){
if(!err){
connection.query("SELECT * FROM users WHERE id = ?",[id], function(err, rows) {
connection.release();
if (err)
done(err);
if (rows.length) {
console.log("rows " + JSON.stringify(rows));
done(rows[0].progress);
};
});
}
else {
console.log(err);
}
});
}
exports.read = read;
// routes.js
var dbrw = require('./dbrw.js');
app.get('/read', isLoggedIn, function(req, res) {
dbrw.read(req.user.id, function(result) {
console.log(result);
});
});
// Frontend - angular app.js
$scope.tryread = function() {
$http.get('/read');
}
Thanks in advance for any input.

I see a few issues:
function read(id, done) {
con.getConnection(function(id, connection){...}
}
Notice how you overwrite the id passed to read by giving that same name to an argument of the callback to getConnection.
Also, your Express route doesn't actually end the request by sending back a response, which will make your browser time out the connection. At some point, it will even refuse to send more requests because too many are still pending.
So make sure to end the request:
app.get('/read', isLoggedIn, function(req, res) {
dbrw.read(req.user.id, function(result) {
console.log(result);
res.end(); // or `res.send(result)`
});
});
And a tip: you should use the callback calling convertion for Node, where the first argument represents an error (if there is any) and the second argument represents the return value.

Related

AWS Lambda nodejs mysql query inside loop is not working

module.exports.handler = async (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
let index = 0;
var client = mysql.createConnection({
host: process.env.rds_host,
user: process.env.rds_username,
password: process.env.rds_password,
port: process.env.rds_port
});
client.connect((err) => {
if (err) throw err;
console.log("Connected!");
});
let array = [];
let queries = ["query1", "query2", "query3", "query4"];
queries.map(q => {
array.push(getQueryResult(client, q));
});
Promise.all(array).then(result => {
console.log(result);
});
callback(null, {});
};
const getQueryResult = async (client, query) => {
return new Promise((resolve, reject) => {
client.query(query, function (err, result) {
if (err) {
reject(err);
}
resolve(result);
});
});
};
Above is my lambda scripts to execute multiple query from mysql. The problem is I didn't get any result and error message from above scripts. Please help me something is missing inside my scripts?
The issue is >> code is not waiting to finish Promise
You can resolve by :
Add callback in then
Promise.all(array).then(result => {
console.log(result);
callback(null, {});
});
OR
Use await
let result = await Promise.all(promiseArray);
console.log(result)
callback(null, {});
Note: Use try-catch to handle error in for await
Also, don't use map to loop array instead use For loop.
There are two or three (potential) issues with your code above:
Your if statement does not get evaluated because of the typeof client predicate does not return true.
your mysql port conflicts with your localhost port (assumption)
Change your if block as such:
// check if `dotenv` has been called if you use it
require('dotenv').config();
// access the state property on mysql object
// if you have closed your connection previously
// this will get evaluated by the parser
if (mysql.state === "disconnected") {
var client = mysql.createConnection({
host: process.env.rds_host,
user: process.env.rds_username,
password: process.env.rds_password,
port: process.env.rds_port // is your port the same as your localhost?
// include your database name here
});
// I suggest you to await your client connection
// since mysql executes sequentially, not asynchronously
client.connect(function(err) => {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
console.log("Connected!");
});
}
if the error persists, it means that your enviroment variables are not set correctly so your db configuration should be reviewed (see inline comments).

Save Query result into Variable Alexa Skills Json

I needed a DB for an alexa app, so I set up and and it INSERTS nicely, but when im trying to SELECT and save it to a variable the values saved to the variable are [Object Object] instead of wanted value, I know it can be async problem or parsing problem but i just cant fix the code, some help would be cool,
canHandle(handlerInput) {
return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
&& Alexa.getIntentName(handlerInput.requestEnvelope) === 'buscaIntent';
},
handle(handlerInput) {
const mysql = require('mysql');
const connection = mysql.createConnection
({
host: 'remotemysql.com',
user: 'RBb34534sd',
password: 'xxxxxxxxx',
database: 'RBsdfewrg'
});
var stat = connection.query('SELECT `spe` FROM `prueba` WHERE `nombre` LIKE "raichu" limit 1', function (err, result, fields) {
if (err) throw err;
console.log(result);
return result[0];
});
connection.end();
return handlerInput.responseBuilder
.speak("Busc " + stat)
.reprompt("reprompt buscar")
.getResponse();
}
}; ```
The issue is that you're not waiting for your database query to complete before sending your response to the Alexa service. Requests in node.js are non-blocking, meaning you either need to nest the request with a callback, or leverage Promises / async-await patterns so that the SQL query is processed before the function is fully executed.
You can read more on converting the built-in library for SQL connections to support Promises here, or use a library like this that already has a wrapper in place.
In either scenario, the end result would be refactored to something like this:
canHandle(handlerInput) {
return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
&& Alexa.getIntentName(handlerInput.requestEnvelope) === 'buscaIntent';
},
async handle(handlerInput) {
const mysql = require('mysql2/promise');
const connection = await mysql.createConnection
({
host: 'remotemysql.com',
user: 'RBb34534sd',
password: 'xxxxxxxxx',
database: 'RBsdfewrg'
});
var stat = await connection.execute('SELECT `spe` FROM `prueba` WHERE `nombre` LIKE "raichu" limit 1', function (err, result, fields) {
if (err) throw err;
console.log(result);
return result[0];
});
return handlerInput.responseBuilder
.speak("Busc " + stat)
.reprompt("reprompt buscar")
.getResponse();
}
Another article describing async calls for Alexa requests here.
I think the query is returning an object you can't keep the object in speech. Check what's inside the object and if you have a field that you want inside that object then access by stat.YourField.

mysql + node.js - how to execute every or none queries?

I am currently using mysql and Node.js, with multipleStatements I can do several queries at the same time, but if a query fails it doesn't break as I expected
app.post('/api3/stock5', function (req, res) {
var connection = mysql.createConnection({
host : conf.host,
user : conf.user,
password : conf.pass,
database : conf.database,
multipleStatements: true //<- essential for several queries at the same time
});
var sql ="UPDATE (...)"; INSERT INTO (...)";
var query = connection.query(sql);
query.on('error', function(err) {
console.log("error: /api3/stock5 /n");
console.log(sql);
});
query.on('result', function(result) {
});
query.on('end', function(result) {
});
res.send('OK');
});
Is there any way to make sure every query was performed correctly and only in that case commit? Thanks

How to properly use node-mysql?

I just started using node-mysql2 and I am confused how to use it properly. Examples:
(Implicitly established connection)
var express = require("express"),
app = express();
var mysql = require("mysql2");
var conn = mysql.createConnection({ ... });
app.VERB("/", function(req, res){
conn.query('SELECT 1+1 as test1', function(err, rows) {
// Show data to user...
// No .end() needed?
});
});
app.listen(8000);
second example:
var express = require("express"),
app = express();
var mysql = require("mysql2");
var conn = mysql.createConnection({ ... });
conn.connect(function(err){ ... }); // Is it right place to put it here? Or it has to go inside the callback below?
app.VERB("/", function(req, res){
conn.query("SELECT 1+1 as test1", function(err, rows){
// Show data to user...
conn.end(); // .end() necessary?
});
});
app.listen(8000);
Although I haven’t used node-mysql2, it is compatible with the original node-mysql module, so the same usage patterns apply.
The best way to use it is with connection pooling. That way, the MySQL client will create and destroy connections as needed. Your job is to call connection.release() when you no longer need a connection:
var express = require("express"),
app = express();
var mysql = require("mysql2");
var pool = mysql.createPool({ ... });
app.VERB("/", function(req, res){
pool.getConnection(function(err, conn) {
if (err) { /* handle the error and bail out */ }
conn.query('SELECT 1+1 as test1', function(err, rows) {
conn.release(); /* the connection is released back to the pool */
if (err) { /* handle the error */ }
else { /* show data to user */ }
});
});
});
app.listen(8000);
If your app runs “forever” (a web site for example) then you don’t need to call pool.end(). If you’re not using a connection pool, you don’t need to call connection.end() after each request. You wouldn’t want to: that would give you the overhead of establishing/tearing down the MySQL connection on every request!
If your app does not run “forever” (a command-line utility, for example), then call pool.end() or connection.end() just before you exit back to the command-line.

Node.js MySQL Needing Persistent Connection

I need a persistent MySQL connection for my Node web app. The problem is that this happens about a few times a day:
Error: Connection lost: The server closed the connection.
at Protocol.end (/var/www/n/node_modules/mysql/lib/protocol/Protocol.js:73:13)
at Socket.onend (stream.js:79:10)
at Socket.EventEmitter.emit (events.js:117:20)
at _stream_readable.js:895:16
at process._tickCallback (node.js:415:13)
error: Forever detected script exited with code: 8
error: Forever restarting script for 2 time
info: socket.io started
Here is my connection code:
// Yes I know multipleStatements can be dangerous in the wrong hands.
var sql = mysql.createConnection({
host: 'localhost',
user: 'my_username',
password: 'my_password',
database: 'my_database',
multipleStatements: true
});
sql.connect();
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
sql = mysql.createConnection(connection.config);
handleDisconnect(sql);
sql.connect();
});
}
handleDisconnect(sql);
As you can see, the handleDisconnect code does not work..
Use the mysql connection pool. It will reconnect when a connection dies and you get the added benefit of being able to make multiple sql queries at the same time. If you don't use the database pool, your app will block database requests while waiting for currently running database requests to finish.
I usually define a database module where I keep my queries separate from my routes. It looks something like this...
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
exports.getUsers = function(callback) {
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
var sql = "SELECT id,name FROM users";
connection.query(sql, [], function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err) {
console.log(err);
callback(true);
return;
}
callback(false, results);
});
});
});
I know this is super delayed, but I've written a solution to this that I think might be a bit more generic and usable. I had written an app entirely dependent on connection.query() and switching to a pool broke those calls.
Here's my solution:
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'user',
password : 'secret',
database : 'test',
port : 3306
});
module.exports = {
query: function(){
var sql_args = [];
var args = [];
for(var i=0; i<arguments.length; i++){
args.push(arguments[i]);
}
var callback = args[args.length-1]; //last arg is callback
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
return callback(err);
}
if(args.length > 2){
sql_args = args[1];
}
connection.query(args[0], sql_args, function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err){
console.log(err);
return callback(err);
}
callback(null, results);
});
});
}
};
This instantiates the pool once, then exports a method named query. Now, when connection.query() is called anywhere, it calls this method, which first grabs a connection from the pool, then passes the arguments to the connection. It has the added effect of grabbing the callback first, so it can callback any errors in grabbing a connection from the pool.
To use this, simply require it as module in place of mysql. Example:
var connection = require('../middleware/db');
function get_active_sessions(){
connection.query('Select * from `sessions` where `Active`=1 and Expires>?;', [~~(new Date()/1000)], function(err, results){
if(err){
console.log(err);
}
else{
console.log(results);
}
});
}
This looks just like the normal query, but actually opens a pool and grabs a connection from the pool in the background.
In response to #gladsocc question:
Is there a way to use pools without refactoring everything? I have
dozens of SQL queries in the app.
This is what I ended up building. It's a wrapper for the query function. It will grab the connection, do the query, then release the connection.
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
And I use it like I would normally.
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});