I'm using node's driver for mysql and need to execute 'n' number of transactions one after the other and not simultaneously.
I've tried using a for/forEach loop but the transactions seem to happen concurrently and that causes my api to crash.Here's the error :-
throw err; // Rethrow non-MySQL errors
^
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
One single transactions seems to work just fine.
Each transaction has 4 queries, req.body is an array of objects:-
router.post('/production/add', (req, res) => {
for (const obj of req.body) {
pool.getConnection(function (err, connection) {
connection.beginTransaction(function (err) {
if (err) throw err;
const query1 = `select qty from production where prc_id = ${obj.prc_id}`;
console.log(query1);
connection.query(query1, function (error, result1, fields) {
if (error) {
return connection.rollback(function () {
res.status(400).send({ query: 1, message: error.sqlMessage, code: error.code, errno: error.errno });
return;
});
}
const new_prod_qty = result1[0].qty - obj.auth_prod_qty;
const query2 = new_prod_qty > 0 ? `update production set qty = ${new_prod_qty} where prc_id = ${obj.prc_id}` : `delete from production where prc_id = ${obj.prc_id}`;
console.log(query2);
connection.query(query2, function (error, results2, fields) {
if (error) {
return connection.rollback(function () {
res.status(400).send({ message: error.sqlMessage, code: error.code, errno: error.errno });
return;
});
}
const query3 = `update prc set auth_prod_qty = ${obj.auth_prod_qty} where prc_id = ${obj.prc_id}`;
console.log(query3);
connection.query(query3, function (error, results3, fields) {
if (error) {
return connection.rollback(function () {
res.status(400).send({ message: error.sqlMessage, code: error.code, errno: error.errno });
return;
});
}
const query4 = "select * from store";
connection.query(query4, function (error, results3, fields) {
if (error) {
return connection.rollback(function () {
res.status(400).send({ message: error.sqlMessage, code: error.code, errno: error.errno });
return;
});
}
connection.commit(function (err) {
if (err) {
return connection.rollback(function () {
res.status(400).send({ message: error.sqlMessage, code: error.code, errno: error.errno });
return;
});
}
res.status(201).send(results2);
});
});
});
});
});
});
});
};
});
Based off some research Sequelize ORM seems to promisify transactions but however I'm hoping to use it as a last resort. Any sort of solution with or without Sequelize would be appreciated!
Thanks in advance!
You need to use async / await to run your txs sequentially. How to do this?
Use npm mysql2 in place of npm mysql. That gets you promisified (awaitable) versions of the APIs when you require('mysql2/promise'). Plus, this is much more fun to program and debug than those miserable nested callbacks. Just don't forget the awaits.
Use this basic outline for your code's data processing loop. Everything will go in order sequentially. The way you create your pool is a little different; read the npm page. This is not debugged.
const mysql = require('mysql2/promise');
router.post('/production/add', async (req, res) => {
const connection = await pool.getConnection()
for (const obj of req.body) {
try {
await connection.beginTransaction()
const query1 = 'whatever'
const result1 = await connection.query(query1)
const query2 = 'something else'
const result 2 = await connection.query(query2)
/* etcetera etcetera */
await connection.commit()
}
catch (error) {
await connection.rollback()
pool.releaseConnection()
res.status(400).send({ something })
}
}
pool.releaseConnection()
}
mysql2/promise is exactly the package I was looking for, works with mysql and uses promise() method to upgrade mysql connection to a promise based mysql2 connection.
router.post('/stock/add', async (req, res) => {
const connection = pool.getConnection(async function (err, connection) {
if (err) {
connection.release();
res.status(400).send(err);
return;
}
else {
for (const obj of req.body) {
try {
await connection.promise().beginTransaction();
const [result1, fields1] = await connection.promise().query(query1)
const [result2, fields2] = await connection.promise().query(query2);
const [result3, fields3] = await connection.promise().query(query3);
const [result4, fields4] = await connection.promise().query(query4);
await connection.promise().commit();
}
catch (error) {
await connection.promise().rollback();
connection.release();
res.status(400).send(error);
return;
}
}
res.status(200).send('Transaction Complete');
}
});
});
Related
I am using Nodejs MySQL and tried to create database level transaction so that I can execute a bunch of statements in a batch and rollback if there is an error in any step. I tried to follow this tutorial.
My database module is:
let mysql = require('mysql')
let keys = require('../config/keys')
let util = require('util')
let pool = mysql.createPool({
connectionLimit: 20,
host: keys.connection.host,
user: keys.connection.user,
password: keys.connection.password,
database: keys.connection.database,
dateStrings: true
// debug:true //Set this to true for verbose debugging. Leaving this to default for now cause it is creating too many messages at my console
})
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
pool.query = util.promisify(pool.query)
const connection = () => {
return new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) reject(err);
console.log("MySQL pool connected: threadId " + connection.threadId);
const query = (sql, binding) => {
return new Promise((resolve, reject) => {
connection.query(sql, binding, (err, result) => {
if (err) reject(err);
resolve(result);
});
});
};
const release = () => {
return new Promise((resolve, reject) => {
if (err) reject(err);
console.log("MySQL pool released: threadId " + connection.threadId);
resolve(connection.release());
});
};
resolve({
query,
release
});
});
});
};
// const query = (sql, binding) => {
// return new Promise((resolve, reject) => {
// pool.query(sql, binding, (err, result, fields) => {
// if (err) reject(err);
// resolve(result);
// });
// });
// };
module.exports = {
pool,
connection
}
In my route, I am trying to use the connection which should allow transaction:
const mysql = require('../../middleware/database')
async function buildCoreSchemas(){
const connection = await mysql.connection();
try{
await connection.query("START TRANSACTION");
await connection.query(`CREATE TABLE adjustreason (
AdjustID int NOT NULL AUTO_INCREMENT,
AdjustReason varchar(100) NOT NULL,
PRIMARY KEY (AdjustID)
)`)
await connection.query(`insert into adjustreason(AdjustReason) values('sdsds')`)
await connection.query(`insert into adjustreason(FAKECOLUMN) values('sdsds')`)
await connection.query("COMMIT");
}
catch(err){
await connection.query("ROLLBACK");
console.log(err)
return false
}
finally {
await connection.release();
}
As you can see I my second insert statement is wrong as there is no column name called FAKE COLUMN. So, the error gets caught and I get the error message in my console:
Unknown column 'FAKECOLUMN' in 'field list
But when I go and look at my database the transaction is not rollbacked because I can see that the first record is still there. What am I doing wrong?
Ciao, try to modify code in this way:
connection.beginTransaction(function(err) {
if (err) { throw err; }
connection.query(`CREATE TABLE adjustreason (
AdjustID int NOT NULL AUTO_INCREMENT,
AdjustReason varchar(100) NOT NULL,
PRIMARY KEY (AdjustID)
)`, function (error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.query(`insert into adjustreason(AdjustReason) values('sdsds')`, function
(error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.query(`insert into adjustreason(FAKECOLUMN) values('sdsds')`, function
(error, results, fields) {
if (error) {
return connection.rollback(function() {
throw error;
});
}
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
console.log('success!');
});
});
});
});
});
so you call connection.query inside connection.beginTransaction and if one of those query fails, you call connection.rollback. Otherwise connection.commit
I am trying to trigger csv file upload in s3 and insert the data from the file to database using lambda.
Most of the times code executes successfully if i run the code back to back in couple of seconds gap.
But sometimes the problem i face is the code stops execution at console console.log('about to get the data'); and ignore rest of the code and sometimes mysql connection gets time out.
I can find that the problem occurs only when i test the lambda code with more than 20 seconds of gap. So, i guess this is a cold start problem.
I don't want to miss even a single s3 trigger. So, i need help to find flaw in my code that is causing this problem.
const AWS = require('aws-sdk');
const s3 = new AWS.S3({region: 'ap-south-1', apiVersion: '2006-03-01'});
var mysql= require('mysql');
var conn = mysql.createPool({
connectionLimit: 50,
host: 'HOST',
user: 'USER',
password: 'PASSWORD',
database: 'DATABASE'
})
async function mainfunc (event, context, callback) {
console.log("Incoming Event: ", JSON.stringify(event));
const bucket = event.Records[0].s3.bucket.name;
const filename = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: bucket,
Key: filename
};
console.log('about to get the data'); //Code stops here some times
return await getresult(params);
};
async function getresult(params){
var result = await s3.getObject(params).promise();
var recordList = result.Body.toString('utf8').split(/\r?\n/).filter(element=>{
return element.length> 5;
})
recordList.shift()
var jsonValues = [];
var jsonKeys = result.Body.toString('utf8').split(/\r?\n/)[0]
recordList.forEach((element) => {
element = element.replace(/"{2,}/g,'"').replace(/, /g,"--").replace(/"{/, "{").replace(/}"/, "}").replace(/,/g, ';').replace(/--/g,', ').split(';');
jsonValues.push(element)
});
var lresult = await query(jsonKeys, jsonValues);
return lresult;
}
async function query(jsonKeys, jsonValues){
var qresult = await conn.getConnection(function(err, connection) {
if (err){
console.log(err,'------------------------------------');// Sometimes i get Sql Connection timeout error here
} else {
console.log("Connected!");
var sql = "INSERT INTO reports ("+jsonKeys+") VALUES ?";
connection.query(sql, [jsonValues], function (err, result) {
if (err){
console.log(err);
connection.release()
return err;
} else {
console.log("1 record inserted");
console.log(result);
connection.release()
return result;
}
});
}
})
}
exports.handler = mainfunc
I have solved the issue by using promise in the "query" function
function query(jsonKeys, jsonValues){
return new Promise(function(resolve, reject) {
conn.getConnection(function (err, connection) {
if (err) {
console.log(err, '------------------------------------');
}
else {
console.log("Connected!");
var sql = "INSERT INTO principal_reports (" + jsonKeys + ") VALUES ?";
connection.query(sql, [jsonValues], function (err, result) {
if (err) {
console.log(err);
connection.release();
reject(err)
}
else {
console.log("1 record inserted");
console.log(result);
connection.release();
resolve(result)
}
});
}
})
})
}
and changed the code
var lresult = await query(jsonKeys, jsonValues);
to
var lresult = await query(jsonKeys, jsonValues).then(data =>{
return data;
}).catch(error =>{
return error;
});
Can't I use promise for nodeJS mysql query?
// My DB settings
const db = require('../util/database');
db.query(sqlQuery, [param1, param2])
.then(result => {
console.log(result);
})
.catch(err => {
throw err;
});
It is returning: TypeError: db.query(...).then is not a function
You mentioned in the comments that you want logic after the query block to be awaited, without placing that logic inside of the callback. By wrapping the method with a Promise, you can do that as such:
try {
const result = await new Promise((resolve, reject) => {
db.query(sqlQuery, (error, results, fields) => {
if (error) return reject(error);
return resolve(results);
});
});
//do stuff with result
} catch (err) {
//query threw an error
}
Something like this should work
function runQuery(sqlQuery){
return new Promise(function (resolve, reject) {
db.query(sqlQuery, function(error, results, fields) {
if (error) reject(error);
else resolve(results);
});
});
}
// test
runQuery(sqlQuery)
.then(function(results) {
console.log(results)
})
.catch(function(error) {
throw error;
});
mysql package does not support promise. We can use then only a function call returns a promise.You can use mysql2 which has inbuilt support for Promise. It will also make your code more readable. From mysql2 docs:
async function main() {
// get the client
const mysql = require('mysql2/promise');
// create the connection
const connection = await mysql.createConnection({host:'localhost',
user: 'root', database: 'test'});
// query database
const [rows, fields] = await connection.execute(query);
// rows hold the result
}
I would aslo recommend you to learn about callbacks, promise and async-await
I tried to get result using mysql database query from called function but do not wait for result in called function. Following is my code for users.js file. I got result in getBankDetail function but do not get result in users function.
var db = require("../db/mysqlconnection");
function users(app){
app.get("/users",async function(req, res, next){
let bankDetail = await getBankDetail();
console.log("bankDetail",bankDetail); //Here I do not got result
return res.send(bankDetail);
});
}
async function getBankDetail(){
db.getConnection(async function(err, connection) {
if (err) throw err; // not connected!
await connection.query('SELECT * FROM bank', function (error, results, fields) {
connection.release();
if (error) throw error;
console.log("bank result",results); //Here I got result
return results;
});
});
}
module.exports = users;
My Question is why do not wait for result in called function? I also used async/await functionality.
function getBankDetail(){
return new Promise((resolve, reject) => {
db.getConnection(function(err, connection) {
if (err) reject(err); // not connected!
connection.query('SELECT * FROM bank', function (error, results, fields) {
connection.release();
if (error) reject(err);
console.log("bank result",results); //Here I got result
resolve(results);
});
});
});
}
And then you can use let bankDetail = await getBankDetail();
If you want to use await on your db.getConnection and connection.query you will have to use mysql2/promises library or promisify those functions yourself
Here is the implementation when you use the promisified version of your database driver:
async function getBankDetail(){
const connection = await db.getConnection();
const data = await connection.query('SELECT * FROM bank');
connection.release();
console.log("bank result", data[0]); //Here I got result
return data[0];
}
I want to return the MySQL result into a variable.
I tried the following but it's not working, as I am getting an empty variable.
const mysql = require('mysql');
const db = require('../config/db');
const connection = mysql.createConnection(db);
module.exports = class Categories {
constructor (res) {
this.res = res;
}
getCategories() {
connection.query("SELECT * FROM `categories`", (error, results, fields) => {
if (error) throw error;
this.pushResult(results);
});
}
pushResult(value) {
this.res = value;
return this.res;
}
};
Just made a callback function first:
var Categories = {
getCategories: function (callback) {
connection.query("SELECT * FROM `categories`", (error, results, fields) => {
if(error) { console.log(err); callback(true); return; }
callback(false, results);
});
}
};
And then used it with route:
app.get('/api/get_categories', (req, res) => {
categories.getCategories(function (error, results) {
if(error) { res.send(500, "Server Error"); return; }
// Respond with results as JSON
res.send(results);
});
});