I'm trying to use the npm package promise-mysql and return json data (or a string doesn't matter) but I'm having issues following the promise chain with await/async.
With the current code i'm receiving Promise { undefined } in the console.log I have right before the response to the user. The response just sends nothing to the user and closes it. Can anyone point in the right direction of how to debug this?
index.js
app.get("/", async (req, res) => {
console.log( Promise.resolve(await getLogs()) )
res.send(await getLogs());
});
mysql.js
const mysql = require("promise-mysql");
let pool;
async function startDatabasePool() {
pool = await mysql.createPool({
connectionLimit: 10,
host: "xxx",
user: "xxx",
password: "xxx",
database: "xxx"
});
}
async function getDatabasePool() {
if (!pool) await startDatabasePool();
return pool;
}
module.exports = {
getDatabasePool,
startDatabasePool
};
users.js
const { getDatabasePool } = require("./mysql");
async function getLogs() {
let pool = await getDatabasePool();
pool.query("SELECT * from logs order by logdate desc", function(
error,
results,
fields
) {
if (error) throw error;
return JSON.stringify(results);
});
}
module.exports = {
getLogs
};
index.js
app.get("/", async (req, res) => {
const result = await getLogs();
res.send(result);
});
mysql.js
const mysql = require("promise-mysql");
let pool;
module.exports.startDatabasePool = async () => {
pool = await mysql.createPool({
connectionLimit: 10,
host: "xxx",
user: "xxx",
password: "xxx",
database: "xxx"
});
}
module.exports.getDatabasePool = async () => {
if (!pool) await startDatabasePool();
return pool;
}
// convert function as promise
module.exports.executeQuery = async(params) => {
return new Promise((resolve, reject) => {
pool.query(params, function (error, result, fields) {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
};
users.js
const { executeQuery } = require("./mysql");
module.exports.getLogs = async () => {
return await executeQuery("SELECT * from logs order by logdate desc");
}
First I'd try it like:
app.get("/", async (req, res) => {
let logs = await getLogs()
console.log(logs)
res.send(logs);
});
I hope it helps!
db.js
const mysql = require('mysql');
const connection = mysql.createPool({
host: 'localhost',
user : 'xxxxx',
password : 'xxxxx',
database : 'xxxx'
})
module.exports = connection;
customer.js
const db = require("../../db");
in customer.js i need to call a SP of mysql which will take 10 input parameters as it will insert a record in table.
What will be the best way to call SP from customer.js using db.js function
I will create db class like this
let pool;
async function setConnectionPool() {
if (!pool) {
pool = await sql.connect("connectionstring here");
}
sql.on("error", err => {
logger.error("An error has occured while setting connection pool", err);
throw error;
});
}
async function save(entity) {
try {
await pool.request()
.input("SomeId1", sql.VarChar(50), entity.field1)
.execute("SomeProcedureName");
} catch (error) {
console.log("error", error);
}
}
module.exports = save;
From your customer.js you could simply call the method save
Hope this helps.
I have a lot of urls, for every url I call the function load(url), this function parse the html, extracts the needed data and builds a bulk insert query as you can see in my test.js code. The problem is, if I have to many urls (like 100+), I get a Error: ER_LOCK_DEADLOCK from mysql. I tried to use async.queue but this is somehow not working (I don't know why, maybe I am using is wrongly). How can I run many urls + queries one after another, avoiding parallel execution which I think resulted in a deadlock? Even using async.queue results to a DEADLOCK (not always).
test.js
const request = require('request');
const async = require('async');
const pool = require('./database');
const urls = [
'https://www.quora.com/What-is-the-best-way-to-have-delayed-job-queue-with-node-js',
'https://de.wikipedia.org/wiki/Reinhardt-Zimmermann-L%C3%B6sung',
'https://towardsdatascience.com/the-5-clustering-algorithms-data-scientists-need-to-know-a36d136ef68'
]
let load = function(url) {
request({url: url}, function(error, response, html) {
if(!error) {
console.log(html);
/**
* 1. Parse HTML
* 2. Create Array of Values
* 3. Call pool.query(sql, [values], function(error) { ... })
*/
let data = [{}];
let sql = "INSERT IGNORE INTO tbl_test (title, content) VALUES ?";
let values = [];
data.forEach((item) => { values.push(item) });
pool.query(sql, [values], function(error) {
if(error) throw error;
})
} else {
console.log("handle error...");
}
})
}
let jobs = []
/*urls.forEach((url) => {
//jobs.push(load(url)); // --> Works but fails if the urls list is to big -> mysql deadlock error!
jobs.push(function(callback) { callback(load(url)) });
})*/
let q = async.queue(function(task, callback) {
console.log("Task:", task.uri);
callback();
})
q.drain = function() {
console.log('all task completed');
pool.end();
}
urls.forEach((url) => {
q.push({uri: url}, function(err) {
console.log('finished processing ...')
});
});
databse.js
require('dotenv').config();
const mysql = require('mysql');
let pool = mysql.createPool(
{
connectionLimit: 10,
host: process.env.DB_HOST,
port: process.env.DB_PORT,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME
}
);
pool.getConnection((err, connection) => {
if(err) {
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
console.log('Database connection lost.')
}
if(err.code === 'ER_CON_COUNT_ERROR') {
console.log('Database has too many connections.')
}
if(err.code === 'ECONNREFUSED') {
console.log('Database connection refused.')
}
if(err.code === 'POOL_CLOSED') {
console.log('Pool is closed.')
}
}
if(connection) {
connection.release()
}
return;
});
module.exports = pool;
I have changed the code to use async.series instead of async.queue, beacuse the tasks would run in parallel in queue (see: https://caolan.github.io/async/docs.html#queue).
test.js
...
let tasks = [];
context.forEach((ctx) => {
tasks.push(function(callback) { load(ctx, callback) });
});
async.series(tasks, function(err) {
if(err) return next(err);
});
I'm developing some functions with serverless with the nodejs template. I have a service that connects to a mysql database and retrieves some data. Everything is fine when I make the first call, but when I repeat it I receive a "Process exited before completing request" error message.
If I try that same call again, I receive data. So the service is doing right on the odd calls and it's returning the error on the even calls (funny right?). This is the code of the handler function:
module.exports.getAll = (event, context, done) => {
deviceRepository.getAllDevices().then((response) => {
done(null, { response });
}).catch((error) => {
done(error);
});
};
and this is the code of the repository function:
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
module.exports.getAllDevices = () => {
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
As you can see I use promises with the 'when' library. I call the 'done' callback in the handler, and there should be a response from the promise in every possible situation.
I can't see what is wrong with this and why is making the odd requests wrong. Anyone can help?
Thanks in advance!
Solved by myself...
The problem is that I was making the createConnection outside of the handler (when I declared the conn constant).
Moving the createConnection declaration inside the handler function works as expected in every call.
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
module.exports.getAllDevices = () => {
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
Hope this helps. Thanks!
I need to provide the mysql connection for modules. I have a code like this.
var express = require('express'),
app = express(),
server = require('http').createServer(app);
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
});
app.get('/save', function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
var query = connection.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
But how we provide one time mysql connection for all the modules.
You could create a db wrapper then require it. node's require returns the same instance of a module every time, so you can perform your connection and return a handler. From the Node.js docs:
every call to require('foo') will get exactly the same object returned, if it would resolve to the same file.
You could create db.js:
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) throw err;
});
module.exports = connection;
Then in your app.js, you would simply require it.
var express = require('express');
var app = express();
var db = require('./db');
app.get('/save',function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
db.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
This approach allows you to abstract any connection details, wrap anything else you want to expose and require db throughout your application while maintaining one connection to your db thanks to how node require works :)
I took a similar approach as Sean3z but instead I have the connection closed everytime i make a query.
His way works if it's only executed on the entry point of your app, but let's say you have controllers that you want to do a var db = require('./db'). You can't because otherwise everytime you access that controller you will be creating a new connection.
To avoid that, i think it's safer, in my opinion, to open and close the connection everytime.
here is a snippet of my code.
mysq_query.js
// Dependencies
var mysql = require('mysql'),
config = require("../config");
/*
* #sqlConnection
* Creates the connection, makes the query and close it to avoid concurrency conflicts.
*/
var sqlConnection = function sqlConnection(sql, values, next) {
// It means that the values hasnt been passed
if (arguments.length === 2) {
next = values;
values = null;
}
var connection = mysql.createConnection(config.db);
connection.connect(function(err) {
if (err !== null) {
console.log("[MYSQL] Error connecting to mysql:" + err+'\n');
}
});
connection.query(sql, values, function(err) {
connection.end(); // close the connection
if (err) {
throw err;
}
// Execute the callback
next.apply(this, arguments);
});
}
module.exports = sqlConnection;
Than you can use it anywhere just doing like
var mysql_query = require('path/to/your/mysql_query');
mysql_query('SELECT * from your_table where ?', {id: '1'}, function(err, rows) {
console.log(rows);
});
UPDATED:
config.json looks like
{
"db": {
"user" : "USERNAME",
"password" : "PASSWORD",
"database" : "DATABASE_NAME",
"socketPath": "/tmp/mysql.sock"
}
}
Hope this helps.
I think that you should use a connection pool instead of share a single connection. A connection pool would provide a much better performance, as you can check here.
As stated in the library documentation, it occurs because the MySQL protocol is sequential (this means that you need multiple connections to execute queries in parallel).
Connection Pool Docs
From the node.js documentation, "To have a module execute code multiple times, export a function, and call that function", you could use node.js module.export and have a single file to manage the db connections.You can find more at Node.js documentation. Let's say db.js file be like:
const mysql = require('mysql');
var connection;
module.exports = {
dbConnection: function () {
connection = mysql.createConnection({
host: "127.0.0.1",
user: "Your_user",
password: "Your_password",
database: 'Your_bd'
});
connection.connect();
return connection;
}
};
Then, the file where you are going to use the connection could be like useDb.js:
const dbConnection = require('./db');
var connection;
function callDb() {
try {
connection = dbConnectionManager.dbConnection();
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (!error) {
let response = "The solution is: " + results[0].solution;
console.log(response);
} else {
console.log(error);
}
});
connection.end();
} catch (err) {
console.log(err);
}
}
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'yourip',
port : 'yourport',
user : 'dbusername',
password : 'dbpwd',
database : 'database schema name',
dateStrings: true,
multipleStatements: true
});
// TODO - if any pool issues need to try this link for connection management
// https://stackoverflow.com/questions/18496540/node-js-mysql-connection-pooling
module.exports = function(qry, qrytype, msg, callback) {
if(qrytype != 'S') {
console.log(qry);
}
pool.getConnection(function(err, connection) {
if(err) {
if(connection)
connection.release();
throw err;
}
// Use the connection
connection.query(qry, function (err, results, fields) {
connection.release();
if(err) {
callback('E#connection.query-Error occurred.#'+ err.sqlMessage);
return;
}
if(qrytype==='S') {
//for Select statement
// setTimeout(function() {
callback(results);
// }, 500);
} else if(qrytype==='N'){
let resarr = results[results.length-1];
let newid= '';
if(resarr.length)
newid = resarr[0]['#eid'];
callback(msg + newid);
} else if(qrytype==='U'){
//let ret = 'I#' + entity + ' updated#Updated rows count: ' + results[1].changedRows;
callback(msg);
} else if(qrytype==='D'){
//let resarr = results[1].affectedRows;
callback(msg);
}
});
connection.on('error', function (err) {
connection.release();
callback('E#connection.on-Error occurred.#'+ err.sqlMessage);
return;
});
});
}
try this
var express = require('express');
var mysql = require('mysql');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
console.log(app);
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "admin123",
database: "sitepoint"
});
con.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
module.exports = app;
you can create a global variable and then access that variable in other files.
here is my code, I have created a separate file for MySQL database connection called db.js
const mysql = require('mysql');
var conn = mysql.createConnection({
host: "localhost",
user: "root",
password: "xxxxx",
database: "test"
});
conn.connect((err) => {
if (err) throw err;
console.log('Connected to the MySql DB');
});
module.exports = conn;
Then in the app.js file
const express = require('express');
const router = express.Router();
// MySql Db connection and set in globally
global.db = require('../config/db');
Now you can use it in any other file
const express = require('express');
const router = express.Router();
router.post('/signin', (req, res) => {
try {
var param = req.body;
var sql = `select * from user`;
// db is global variable
db.query(sql, (err, data) => {
if (err) throw new SyntaxError(err);
res.status(200).json({ 'auth': true, 'data': data });
});
} catch (err) {
res.status(400).json({ 'auth': false, 'data': err.message });
}
});