I am using Nodejs MySQL and tried to create database level transaction so that I can execute a bunch of statements in a batch and rollback if there is an error in any step. I tried to follow this tutorial.
My database module is:
let mysql = require('mysql')
let keys = require('../config/keys')
let util = require('util')
let pool = mysql.createPool({
connectionLimit: 20,
host: keys.connection.host,
user: keys.connection.user,
password: keys.connection.password,
database: keys.connection.database,
dateStrings: true
// debug:true //Set this to true for verbose debugging. Leaving this to default for now cause it is creating too many messages at my console
})
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
pool.query = util.promisify(pool.query)
const connection = () => {
return new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) reject(err);
console.log("MySQL pool connected: threadId " + connection.threadId);
const query = (sql, binding) => {
return new Promise((resolve, reject) => {
connection.query(sql, binding, (err, result) => {
if (err) reject(err);
resolve(result);
});
});
};
const release = () => {
return new Promise((resolve, reject) => {
if (err) reject(err);
console.log("MySQL pool released: threadId " + connection.threadId);
resolve(connection.release());
});
};
resolve({
query,
release
});
});
});
};
// const query = (sql, binding) => {
// return new Promise((resolve, reject) => {
// pool.query(sql, binding, (err, result, fields) => {
// if (err) reject(err);
// resolve(result);
// });
// });
// };
module.exports = {
pool,
connection
}
In my route, I am trying to use the connection which should allow transaction:
const mysql = require('../../middleware/database')
async function buildCoreSchemas(){
const connection = await mysql.connection();
try{
await connection.query("START TRANSACTION");
await connection.query(`CREATE TABLE adjustreason (
AdjustID int NOT NULL AUTO_INCREMENT,
AdjustReason varchar(100) NOT NULL,
PRIMARY KEY (AdjustID)
)`)
await connection.query(`insert into adjustreason(AdjustReason) values('sdsds')`)
await connection.query(`insert into adjustreason(FAKECOLUMN) values('sdsds')`)
await connection.query("COMMIT");
}
catch(err){
await connection.query("ROLLBACK");
console.log(err)
return false
}
finally {
await connection.release();
}
As you can see I my second insert statement is wrong as there is no column name called FAKE COLUMN. So, the error gets caught and I get the error message in my console:
Unknown column 'FAKECOLUMN' in 'field list
But when I go and look at my database the transaction is not rollbacked because I can see that the first record is still there. What am I doing wrong?
Ciao, try to modify code in this way:
connection.beginTransaction(function(err) {
if (err) { throw err; }
connection.query(`CREATE TABLE adjustreason (
AdjustID int NOT NULL AUTO_INCREMENT,
AdjustReason varchar(100) NOT NULL,
PRIMARY KEY (AdjustID)
)`, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.query(`insert into adjustreason(AdjustReason) values('sdsds')`, function
(error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.query(`insert into adjustreason(FAKECOLUMN) values('sdsds')`, function
(error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
console.log('success!');
});
});
});
});
});
so you call connection.query inside connection.beginTransaction and if one of those query fails, you call connection.rollback. Otherwise connection.commit
I am trying to recreate mysql Nodejs transaction using async await syntax. This is what the documentation shows without async/await:
connection.beginTransaction(function(err) {
if (err) { throw err; }
connection.query('INSERT INTO posts SET title=?', title, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
var log = 'Post ' + results.insertId + ' added';
connection.query('INSERT INTO log SET data=?', log, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
console.log('success!');
});
});
});
});
I have created an async pool that I can use with async await:
let util=require('util')
let pool = mysql.createPool({
connectionLimit: 20,
host: keys.connection.host,
user: keys.connection.user,
password: keys.connection.password,
database: keys.connection.database,
dateStrings: true
// debug:true //Set this to true for verbose debugging. Leaving this to default for now cause it is creating too many messages at my console
})
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
pool.query = util.promisify(pool.query)
module.exports = pool
To convert the documentation to async/await I tried to do :
const connection = await pool.getConnection();
await connection.beginTransaction();
but I am getting:
TypeError: Cannot read property 'beginTransaction' of undefined
I wanted to know if it is possible to use async await when using transaction with mysql. I created the snippet below using the mysql documentation but the problem below is that if I have multiple queries and I want them to fire one after the another there is no way to specify it without creating a call back hell. Has anyone been able to do it with async await?
pool.getConnection(function(err, connection) {
connection.beginTransaction(function(err) {
if (err) { throw err; }
connection.query('INSERT INTO posts SET title=?', title, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
var log = 'Post ' + results.insertId + ' added';
connection.query('INSERT INTO log SET data=?', log, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
//console.log('success!');
connection.release();
});
});
});
});
})
I am using a connection pool by the way as you can see above. If you want to know how the pool is created here is the code:
let pool = mysql.createPool({
connectionLimit: 20,
host: keys.connection.host,
user: keys.connection.user,
password: keys.connection.password,
database: keys.connection.database,
dateStrings: true
// debug:true //Set this to true for verbose debugging. Leaving this to default for now cause it is creating too many messages at my console
})
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
// if (connection) connection.release()
return
})
pool.query = util.promisify(pool.query)
module.exports = pool
If you don't need the returned value of each query/request. You can use promise in every request and store it in an array and do promise all at the end:
let requestArray = []
// push each request to requestArray
// then do
Promise.all(requestArray).then((values) => {
console.log(values);
});
if you do, you need something like async waterfall : async-waterfall : npmjs.com/package/async-waterfall
Hope that helps
During off hours, the DB goes in the Idle state. So I am reconnecting the data using the following snippet,
var mysql = require('mysql');
var settings = require('./config.js');
var connection;
function handleDisconnect() {
connection = mysql.createConnection({
host: settings.dbhost,
user: settings.user,
password: settings.password,
database: settings.database
});
connection.connect(function (err) {
if (err) {
console.log('Error when connecting to db:', err, settings.database);
setTimeout(handleDisconnect, 2000);
}
else {
console.log("Database is connected ...", settings.database);
}
});
connection.on('error', function (err) {
console.log('db error', err, settings.database);
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
connection
handleDisconnect();
} else {
throw err;
}
});
}
handleDisconnect();
module.exports = connection;
When executing DB query, I am getting the following error:
{
"status": 500,
"Error": {
"code": "PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR",
"fatal": false
}
}
Any ideas on how to fix the issue?
One possible, but imperfect solution is to use sql.ConnectionPool()
I'm trying to figure out how to structure my application to use MySQL most efficent way. I'm using node-mysql module. Other threads here suggested to use connection pooling so i set up a little module mysql.js
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'root',
password : 'root',
database : 'guess'
});
exports.pool = pool;
Now whenever I want to query mysql I require this module and then query the databse
var mysql = require('../db/mysql').pool;
var test = function(req, res) {
mysql.getConnection(function(err, conn){
conn.query("select * from users", function(err, rows) {
res.json(rows);
})
})
}
Is this good approach? I couldn't really find too much examples of using mysql connections besides very simple one where everything is done in main app.js script so I don't really know what the convention / best practices are.
Should I always use connection.end() after each query? What if I forget about it somewhere?
How to rewrite the exports part of my mysql module to return just a connection so I don't have to write getConnection() every time?
It's a good approach.
If you just want to get a connection add the following code to your module where the pool is in:
var getConnection = function(callback) {
pool.getConnection(function(err, connection) {
callback(err, connection);
});
};
module.exports = getConnection;
You still have to write getConnection every time. But you could save the connection in the module the first time you get it.
Don't forget to end the connection when you are done using it:
connection.release();
You should avoid using pool.getConnection() if you can. If you call pool.getConnection(), you must call connection.release() when you are done using the connection. Otherwise, your application will get stuck waiting forever for connections to be returned to the pool once you hit the connection limit.
For simple queries, you can use pool.query(). This shorthand will automatically call connection.release() for you—even in error conditions.
function doSomething(cb) {
pool.query('SELECT 2*2 "value"', (ex, rows) => {
if (ex) {
cb(ex);
} else {
cb(null, rows[0].value);
}
});
}
However, in some cases you must use pool.getConnection(). These cases include:
Making multiple queries within a transaction.
Sharing data objects such as temporary tables between subsequent queries.
If you must use pool.getConnection(), ensure you call connection.release() using a pattern similar to below:
function doSomething(cb) {
pool.getConnection((ex, connection) => {
if (ex) {
cb(ex);
} else {
// Ensure that any call to cb releases the connection
// by wrapping it.
cb = (cb => {
return function () {
connection.release();
cb.apply(this, arguments);
};
})(cb);
connection.beginTransaction(ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table1 ("value") VALUES (\'my value\');', ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table2 ("value") VALUES (\'my other value\')', ex => {
if (ex) {
cb(ex);
} else {
connection.commit(ex => {
cb(ex);
});
}
});
}
});
}
});
}
});
}
I personally prefer to use Promises and the useAsync() pattern. This pattern combined with async/await makes it a lot harder to accidentally forget to release() the connection because it turns your lexical scoping into an automatic call to .release():
async function usePooledConnectionAsync(actionAsync) {
const connection = await new Promise((resolve, reject) => {
pool.getConnection((ex, connection) => {
if (ex) {
reject(ex);
} else {
resolve(connection);
}
});
});
try {
return await actionAsync(connection);
} finally {
connection.release();
}
}
async function doSomethingElse() {
// Usage example:
const result = await usePooledConnectionAsync(async connection => {
const rows = await new Promise((resolve, reject) => {
connection.query('SELECT 2*4 "value"', (ex, rows) => {
if (ex) {
reject(ex);
} else {
resolve(rows);
}
});
});
return rows[0].value;
});
console.log(`result=${result}`);
}
You will find this wrapper usefull :)
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
Require it, use it like this:
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});
I am using this base class connection with mysql:
"base.js"
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit : 10,
host: Config.appSettings().database.host,
user: Config.appSettings().database.username,
password: Config.appSettings().database.password,
database: Config.appSettings().database.database
});
var DB = (function () {
function _query(query, params, callback) {
pool.getConnection(function (err, connection) {
if (err) {
connection.release();
callback(null, err);
throw err;
}
connection.query(query, params, function (err, rows) {
connection.release();
if (!err) {
callback(rows);
}
else {
callback(null, err);
}
});
connection.on('error', function (err) {
connection.release();
callback(null, err);
throw err;
});
});
};
return {
query: _query
};
})();
module.exports = DB;
Just use it like that:
var DB = require('../dal/base.js');
DB.query("select * from tasks", null, function (data, error) {
callback(data, error);
});
When you are done with a connection, just call connection.release() and the connection will return to the pool, ready to be used again by someone else.
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
// Use the connection
connection.query('SELECT something FROM sometable', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
// Don't use the connection here, it has been returned to the pool.
});
});
If you would like to close the connection and remove it from the pool, use connection.destroy() instead. The pool will create a new connection the next time one is needed.
Source: https://github.com/mysqljs/mysql
You can use this format as I used
const mysql = require('mysql');
const { HOST, USERNAME, PASSWORD, DBNAME, PORT } = process.env;
console.log();
const conn = mysql.createPool({
host: HOST,
user: USERNAME,
password: PASSWORD,
database: DBNAME
}, { debug: true });
conn.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('Db is connected - The solution is: ', results[0].solution);
});
module.exports = conn;
Using the standard mysql.createPool(), connections are lazily created by the pool. If you configure the pool to allow up to 100 connections, but only ever use 5 simultaneously, only 5 connections will be made. However if you configure it for 500 connections and use all 500 they will remain open for the durations of the process, even if they are idle!
This means if your MySQL Server max_connections is 510 your system will only have 10 mySQL connections available until your MySQL Server closes them (depends on what you have set your wait_timeout to) or your application closes! The only way to free them up is to manually close the connections via the pool instance or close the pool.
mysql-connection-pool-manager module was created to fix this issue and automatically scale the number of connections dependant on the load. Inactive connections are closed and idle connection pools are eventually closed if there has not been any activity.
// Load modules
const PoolManager = require('mysql-connection-pool-manager');
// Options
const options = {
...example settings
}
// Initialising the instance
const mySQL = PoolManager(options);
// Accessing mySQL directly
var connection = mySQL.raw.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
// Initialising connection
connection.connect();
// Performing query
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
// Ending connection
connection.end();
Ref: https://www.npmjs.com/package/mysql-connection-pool-manager
i always use connection.relase(); after pool.getconnetion like
pool.getConnection(function (err, connection) {
connection.release();
if (!err)
{
console.log('*** Mysql Connection established with ', config.database, ' and connected as id ' + connection.threadId);
//CHECKING USERNAME EXISTENCE
email = receivedValues.email
connection.query('SELECT * FROM users WHERE email = ?', [email],
function (err, rows) {
if (!err)
{
if (rows.length == 1)
{
if (bcrypt.compareSync(req.body.password, rows[0].password))
{
var alldata = rows;
var userid = rows[0].id;
var tokendata = (receivedValues, userid);
var token = jwt.sign(receivedValues, config.secret, {
expiresIn: 1440 * 60 * 30 // expires in 1440 minutes
});
console.log("*** Authorised User");
res.json({
"code": 200,
"status": "Success",
"token": token,
"userData": alldata,
"message": "Authorised User!"
});
logger.info('url=', URL.url, 'Responce=', 'User Signin, username', req.body.email, 'User Id=', rows[0].id);
return;
}
else
{
console.log("*** Redirecting: Unauthorised User");
res.json({"code": 200, "status": "Fail", "message": "Unauthorised User!"});
logger.error('*** Redirecting: Unauthorised User');
return;
}
}
else
{
console.error("*** Redirecting: No User found with provided name");
res.json({
"code": 200,
"status": "Error",
"message": "No User found with provided name"
});
logger.error('url=', URL.url, 'No User found with provided name');
return;
}
}
else
{
console.log("*** Redirecting: Error for selecting user");
res.json({"code": 200, "status": "Error", "message": "Error for selecting user"});
logger.error('url=', URL.url, 'Error for selecting user', req.body.email);
return;
}
});
connection.on('error', function (err) {
console.log('*** Redirecting: Error Creating User...');
res.json({"code": 200, "status": "Error", "message": "Error Checking Username Duplicate"});
return;
});
}
else
{
Errors.Connection_Error(res);
}
});