I am trying to fetch some results and do the further processing based on those results, but I can't proceed to work it sequentially,
var sql = query1;
con.query(sql, function (err, results) {
if (err) throw err;
// ids => 5,2,3,4
for (i = 0; i < results.length; i++) {
target_user = results[i].ID
var sql = "DELETE QUERY";
con.query(sql, function (err) {
if (err) throw err;
console.log(target_user)
var sql = "INSERT QUERY";
console.log(sql)
con.query(sql, function (err) {
if (err) throw err;
})
})
}
})
The above code runs asynchronously, What I expect is an output in a loop like this
// "DELETE QUERY";
//5
// "INSERT QUERY";
// "DELETE QUERY";
//2
// "INSERT QUERY";
and so on..
but what I get is
// "DELETE QUERY";
//5
// "DELETE QUERY";
//5 //not fetching the next array val
// "INSERT QUERY";
// "INSERT QUERY";
Any help is much appriciated.
EDIT
from answers I updated code like this
now the code looks like this
aysnc.forEach(results, function(elem, callback){
target_user = elem.id
console.log('out')
console.log(target_user)
con.query(sql, function (err) {
if (err) throw err;
console.log('in')
console.log(target_user)
})
})
A strange thing happened that output is
out
5
in
5
out
2
in
5 //when it is supposed to be 2
You can still use npm module async in a different way
const mysql = require('mysql');
const async = require('aynsc');
var db; //database variable
async.series([
//creates DB connection and connects
function(callback){
db = mysql.createConnection(DB_INFO); //DB_INFO is an Object with information on the BD to be connected
db.connect(function(err){
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
callback(); //goes to the next function
});
},
//performs the Query 1
function(callback){
db.query('QUERY1', function(){
callback(); //goes to the next function
});
},
//performs the Query 2 only after Query 1 is finished
function(callback){
db.query('QUERY2', function(){
db.end(); //closes connection
callback();
});
}
]);
You could use recursion to solve something like this. Keep calling function until there is no elements left in results
con.query(sql, function (err, results) {
if (err) throw err;
deleteAndInsertResults(results);
})
function deleteAndInsertResult(results)
{
target_user = results[0].ID
var sql = "DELETE QUERY";
con.query(sql, function (err) {
if (err) throw err;
console.log(target_user)
var sql = "INSERT QUERY";
console.log(sql)
con.query(sql, function (err) {
if (err) throw err;
results.shift();
if(results.length){
return deleteAndInsertResult(results);
}
})
})
}
In node.js FOR loop will be executed parallely, so use async module or PROMISE below is an example using async
var async = require('aynsc');
con.query(sql, function (err, results) {
if (err) throw err;
// ids => 5,2,3,4
async.forEach(results, function(elem, callback){
target_user = results[i].ID
var sql = "DELETE QUERY";
con.query(sql, function (err) {
if (err) throw err;
console.log(target_user)
var sql = "INSERT QUERY";
console.log(sql)
con.query(sql, function (err) {
if (err) throw err;
callback()
})
})
}, function(err){
//final callback once loop is done
});
})
Related
I am trying to trigger csv file upload in s3 and insert the data from the file to database using lambda.
Most of the times code executes successfully if i run the code back to back in couple of seconds gap.
But sometimes the problem i face is the code stops execution at console console.log('about to get the data'); and ignore rest of the code and sometimes mysql connection gets time out.
I can find that the problem occurs only when i test the lambda code with more than 20 seconds of gap. So, i guess this is a cold start problem.
I don't want to miss even a single s3 trigger. So, i need help to find flaw in my code that is causing this problem.
const AWS = require('aws-sdk');
const s3 = new AWS.S3({region: 'ap-south-1', apiVersion: '2006-03-01'});
var mysql= require('mysql');
var conn = mysql.createPool({
connectionLimit: 50,
host: 'HOST',
user: 'USER',
password: 'PASSWORD',
database: 'DATABASE'
})
async function mainfunc (event, context, callback) {
console.log("Incoming Event: ", JSON.stringify(event));
const bucket = event.Records[0].s3.bucket.name;
const filename = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: bucket,
Key: filename
};
console.log('about to get the data'); //Code stops here some times
return await getresult(params);
};
async function getresult(params){
var result = await s3.getObject(params).promise();
var recordList = result.Body.toString('utf8').split(/\r?\n/).filter(element=>{
return element.length> 5;
})
recordList.shift()
var jsonValues = [];
var jsonKeys = result.Body.toString('utf8').split(/\r?\n/)[0]
recordList.forEach((element) => {
element = element.replace(/"{2,}/g,'"').replace(/, /g,"--").replace(/"{/, "{").replace(/}"/, "}").replace(/,/g, ';').replace(/--/g,', ').split(';');
jsonValues.push(element)
});
var lresult = await query(jsonKeys, jsonValues);
return lresult;
}
async function query(jsonKeys, jsonValues){
var qresult = await conn.getConnection(function(err, connection) {
if (err){
console.log(err,'------------------------------------');// Sometimes i get Sql Connection timeout error here
} else {
console.log("Connected!");
var sql = "INSERT INTO reports ("+jsonKeys+") VALUES ?";
connection.query(sql, [jsonValues], function (err, result) {
if (err){
console.log(err);
connection.release()
return err;
} else {
console.log("1 record inserted");
console.log(result);
connection.release()
return result;
}
});
}
})
}
exports.handler = mainfunc
I have solved the issue by using promise in the "query" function
function query(jsonKeys, jsonValues){
return new Promise(function(resolve, reject) {
conn.getConnection(function (err, connection) {
if (err) {
console.log(err, '------------------------------------');
}
else {
console.log("Connected!");
var sql = "INSERT INTO principal_reports (" + jsonKeys + ") VALUES ?";
connection.query(sql, [jsonValues], function (err, result) {
if (err) {
console.log(err);
connection.release();
reject(err)
}
else {
console.log("1 record inserted");
console.log(result);
connection.release();
resolve(result)
}
});
}
})
})
}
and changed the code
var lresult = await query(jsonKeys, jsonValues);
to
var lresult = await query(jsonKeys, jsonValues).then(data =>{
return data;
}).catch(error =>{
return error;
});
I tried to get result using mysql database query from called function but do not wait for result in called function. Following is my code for users.js file. I got result in getBankDetail function but do not get result in users function.
var db = require("../db/mysqlconnection");
function users(app){
app.get("/users",async function(req, res, next){
let bankDetail = await getBankDetail();
console.log("bankDetail",bankDetail); //Here I do not got result
return res.send(bankDetail);
});
}
async function getBankDetail(){
db.getConnection(async function(err, connection) {
if (err) throw err; // not connected!
await connection.query('SELECT * FROM bank', function (error, results, fields) {
connection.release();
if (error) throw error;
console.log("bank result",results); //Here I got result
return results;
});
});
}
module.exports = users;
My Question is why do not wait for result in called function? I also used async/await functionality.
function getBankDetail(){
return new Promise((resolve, reject) => {
db.getConnection(function(err, connection) {
if (err) reject(err); // not connected!
connection.query('SELECT * FROM bank', function (error, results, fields) {
connection.release();
if (error) reject(err);
console.log("bank result",results); //Here I got result
resolve(results);
});
});
});
}
And then you can use let bankDetail = await getBankDetail();
If you want to use await on your db.getConnection and connection.query you will have to use mysql2/promises library or promisify those functions yourself
Here is the implementation when you use the promisified version of your database driver:
async function getBankDetail(){
const connection = await db.getConnection();
const data = await connection.query('SELECT * FROM bank');
connection.release();
console.log("bank result", data[0]); //Here I got result
return data[0];
}
I insert a record in the database like this
connection.connect(function(err) {
if (err) throw err;
console.log("Connected!");
var sql = "INSERT INTO xtable (x_date, x_time, x_text) VALUES ('2018-05-26', '23:00:00', 'blablabla')";
connection.query(sql, function (err, result) {
if (err) throw err;
console.log("1 record inserted");
connection.end();
});
});
The above works but lets say I do
var ytime = '19:00:00';
var ydate = '2018-05-29';
var ytext = 'blabla';
connection.connect(function(err) {
if (err) throw err;
console.log("Connected!");
var sql = "INSERT INTO xtable (x_date, x_time, x_text) VALUES (ydate, ytime, ytext)";
connection.query(sql, function (err, result) {
if (err) throw err;
console.log("1 record inserted");
connection.end();
});
});
How do I do that? this gives just errors my node is v8.10.0
The latter didn't work because the sql statement did not interpret your ydate,ytime,ytext as variables but as a part of string. If you want to separate the statement and your data, you should do like this:
var ytime = '19:00:00';
var ydate = '2018-05-29';
var ytext = 'blabla';
var sql = "INSERT INTO xtable (x_date, x_time, x_text) VALUES (?,?,?)";
connection.query(sql, [ydate,ytime, ytext], function(err,result) {
...
});
You just switch your data to use placeholder values, then add the data separately:
connection.query(
"INSERT INTO xtable (x_date, x_time, x_text) VALUES (?, ?, ?)",
[ ydate, ytime, ytext ],
function (err, result) {
// ...
}
}
One thing to note about Node and MySQL is there's tools like Sequelize that make this a lot easier. Anything that supports Promises and async/await is almost always less fuss than a series of nested callbacks.
I use NodeJS to insert data to a table with many to many relationship, and I want to include two foreign keys when I insert data into the table this is how my code looks:
con.query("SELECT * FROM Transaction WHERE TransactionID > 1", function(err, res) {
if (err) {
throw (err);
} else if (res.length > 0) {
console.log("Transaction already exit");
} else {
var transactionID;
var filePK;
con.query("SELECT Filename FROM File WHERE Filename = ?", fileName, function(err, res) {
if (err) throw err;
filePK = JSON.stringify(res);
});
con.query("SELECT TransactionDescriptionPK FROM TransactionDescription WHERE TransactionDescriptionPK > 0", function(err, res) {
if (err) throw err;
//console.log(res);
transactionID = res;
});
var tran = {
TransactionID: data.ID,
TransactionDate: data.Description,
Amount: data.Amount
};
con.query("INSERT INTO Transaction SET ?", tran, function(err, res) {
if (err) throw err;
});
How I return the results from those queries so I can add them to the tran object?
You can use async.parallel
async.parallel([
function(callback) {
// you can directly pass the parallel callback to mysql query
// if you don't need to do anything else
return con.query(
"SELECT * FROM Transaction WHERE TransactionID > 1",
callback
);
},
function(callback) {
// otherwise, just do you what you want (here JSON.stringify)
// then don't forget to return the parallel callback
return con.query(
"SELECT Filename FROM File WHERE Filename = ?",
fileName,
function(err, res) {
if (err)
return callback(err);
const filePK = JSON.stringify(res);
return callback(null, filePK);
}
);
},
function(callback) {
return con.query(
"SELECT TransactionDescriptionPK FROM TransactionDescription WHERE TransactionDescriptionPK > 0",
callback
);
},
], function(err, data) {
// if any mysql queries above encounter an error,
// it calls the parallel final callback and stops other functions
// without errors, data looks like :
// data[0] equals mysql result object of the first query
// data[1] equals filePK const
// data[2] equals mysql result object of the last query
const tran = {
TransactionID: data[0][0].ID,
TransactionDate: data[0][0].Description,
// Amount: data.Amount // don't know where the amount come from,
// but you get the idea
};
con.query(
"INSERT INTO Transaction SET ?",
tran,
function(err, res) {
if (err)
throw err;
// ...
}
);
});
I have the following code. I am relative new to nodejs &js
I want to get values in 1. log but i get undefined.
Only 2. log is outputed to the log.
I read nodeJS return value from callback and
https://github.com/felixge/node-mysql but there is no example about return value.
I donot know how to use return statement with the given example in node-mysql page.
exports.location_internal = function (req, res) {
var r = getExternalLocation(2);
// 1. log
console.log(r);
res.send( r);
}
var getExternalLocation = function (id) {
pool.getConnection(function(err, connection){
if(err) throw err;
var response = {};
connection.query( "select * from external_geo_units where geo_unit_id = "+id, function(err, rows){
if(err) throw err;
response.data= rows;
// 2. log
console.log(response);
return response;
});
connection.release();
});
};
It's asynchronous, so you have to pass in a callback to get the value when it's ready. Example:
exports.location_internal = function(req, res, next) {
getExternalLocation(2, function(err, rows) {
if (err)
return next(err);
console.log(rows);
res.send(rows);
});
};
function getExternalLocation(id, cb) {
pool.getConnection(function(err, conn) {
if (err)
return cb(err);
conn.query("select * from external_geo_units where geo_unit_id = ?",
[id],
function(err, rows) {
conn.release();
cb(err, rows);
});
});
}