We are trying to create backend - RESTFUL api based on mysql DB - in express Nodejs. My boss (self acclaimed mysql guru) insist that we should use more connection pools than 1 - based on the different roles that users are grouped in.
So I need to define/create different connection pools in file B.js - > and make them as global variables so that I could access them later.
In file A.js I have a this route - in which looged in users will be able to call stored procedures with arguments. However, I would like to be able to define from which connection pool the connection will be called - in route itself (based on the session (req.user.role) variable).
So that:
pool1.getConnection(function(err,connection){
if (err) {
connection.release();
res.json({"code" : 100, "status" : "Error in connection database"});
return;
}
console.log('connected as id ' + connection.threadId);
connection.query(query1, val, function (err, rows, fields) {
connection.release();
if(!err) {
var results = JSON.stringify(rows[0]);
console.log(results);
console.log(typeof(results));
console.log(rows[0][0]);
console.log(typeof(rows[0]));
res.send(JSON.stringify(results));
}
});
connection.on('error', function(err) {
connection.release();
res.json({"code" : 100, "status" : "Error in connection database"});
return;
});
});
});
};
refers to pool1, but in case with other user -> pool1 should connect to other pool - defined in other files.
I hope I am clear with my issue.
You don't need to use global variables, you can export an object that holds each pool:
// B.js
module.exports = {
regular : mysql.createPool({ ... }),
admin : mysql.createPool({ ... }),
};
// A.js
const pools = require('./B');
...
let pool = pools[req.user.role];
if (! pool) {
throw Error('there is no pool for user role ' + req.user.role);
}
pool.getConnection(...);
So this declares two pools, one for "regular" users and one for "admin" users. It uses the value of req.user.role (which is assumed to contain one of two values: "regular" or "admin", but that's just for the purpose of this example) to pick the pool that should be used.
I'm not sure what the upside is of having more than one pool, unless they connect to two different databases or use different database credentials.
I don't know why you're doing this or your boss want to do this, more connection pools won't make the application more efficient, it just confuse people. Since the MySQL server has its limit of clients. And by the way, if you want to know a user's role, you have to connect the database and fetch the user data in the first place. Why don't you just use the same connection to do other things and that will be fine.
We have more than one UI developer, each develops one role so we don't want that all UI developers can see all store procedures... (That boss)
Related
I have used one signle connection object of MYSQL in node JS to serve for multiple users.
I mean to say that MySQL connection will be created upon starting the script & it will remain same until the life of the node script/server.
Practically, this is possible to do & i have done the same. Please take a look at below code of NodeJS/MySQL script.
#################################
var http = require('http');
var mysql = require('mysql');
var con = mysql.createConnection({
host: "192.168.1.105",
user: "root",
password: "XXXXXX",
database: "mydb"
});
con.connect(function(err) {
if (err) {
console.error('error: ' + err.message);
process.exit(1);
}
http.createServer(function (req, res) {
continueExecution(req,res);
}).listen(8082);
});
async function continueExecution(req,res){
res.write('calledddd\n');
for (let step = 0; step < 50; step++) {
// Runs 5 times, with values of step 0 through 4.
var bar = `Company Inc ${step}`;
var sql = `INSERT INTO customers (name, address) VALUES ('${bar}', 'Highway 37')`;
res.write(sql + "\n");
con.query(sql, function (err, result) {
if (err) throw err;
res.write("1 record inserted\n");
});
}
res.write('reached\n');
for (let ste = 0; ste < 50; ste++) {
res.write('started Update\n');
var bar = `Company Inc ${ste}`;
var sql = `UPDATE customers SET name = 'UPDATE RECORD' WHERE name = '${bar}'`;
con.query(sql, function (err, result) {
if (err) throw err;
res.write(result.affectedRows + " record(s) updated\n");
if(ste == 50) {
res.writeHead(200, {'Content-Type': 'text/html\n'});
res.write('Databse connected\n');
res.end();
}
});
}
}
#################################
I have several questions in my mind as i am technical expert. But i didn't find any resources over my questions. Please help me on this
Q1. Are there any type of consequences of using one single MySQL connection to provide response to multiple users?
Q2. Let's take an example.
100 users wants to access table name "users_data" at the same time. 25 users are updating their records on the same table with unique primary key. 50 users are selecting their records. another 25 users deleting their records.
All these operations are being done at the same time via parallel Node Script calls from remote device.
To complete all these MySQL transactions, system is using only 1 database connection.
What will happen in this case?
To answer your questions, one of the consequences of using a unique connection is that it can lead to slower request execution.
In fact, even if node will make the requests asynchronously, your database will execute all those requests synchronously, so one after the other in the order they came. As node makes the requests asynchronously, the order in which they are executed by your database is not granted, and the issue you are referencing to might happen.
One easy way to avoid this is to use a connection pool which will create a given number of connection, using the same db user. Here are some links that might help you with this :
using a connection pool with node.js
connect a mysql database with node.js
How do we close the database connection in the below app.post. Do sequelize will automatically taken care of it ?
server.js
const sequelize = new Sequelize(DB_NAME, DB_USERNAME, DB_PASSWORD, {
host: DB_HOST,
dialect: DB_DIALECT,
pool: DB_POOL,
port: DB_PORT
});
const Availability = availabilitySchema(sequelize, DataTypes);
app.post('/service/availability', async (req, res) => {
try {
const userEmail = req.query.email;
const dailyStatus = req.body.dailystatus;
var playerData = {email:userEmail, dailystatus: dailyStatus};
const playerDailyStatus = await Availability.create(playerData);
res.status(200).json({ success: true });
} catch (e) {
res.status(500).json({ message: e.message });
}
});
As i understand it (and I only started looking at sequelize yesterday - comments if I'm wrong, please) Sequelize pools its connections, so there isn't really anything for you to close; it opens and closes connections as necessary much like any other ORM, and mostly those connections are open and live in a pool, are leased from the pool and do some work, then are returned to the pool. You can configure the pool options (looks like you already have) if you want to limit the number of concurrently open connections to your DB but if you're looking in your DB manager and seeing "omg, my sequelize appp has 5 open connections.. now it has 10.. now 15!" that's just how it is; it opens as many connections as necessary (up to the max) to service the workload and leaves them open because it's a huge waste of time to actually open (TCP connect) and close them constantly (TCP disconnect).
When using an ORM you don't micromanage the connections, you just carry out queries using the modeled objects and let the ORM deal with the low level stuff (opening and closing a connection is a level below crafting the SQL to run, and you hand that off to the ORM too). Even in something you've been used to elsewhere, like C# new SqlConnection("connstr").Open() might not actually be opening a TCP cnnection to the DB and closing it; it's probably just leasing and returning to a pool and the underlying framework manages the actual TCP connections and their state
I have two node microservices talking to one common mysql database. Both microservices have this below code to create a connection pool with a connectionlimit of 10 as follows:
// Initializing pool
var pool = mysql.createPool({
connectionLimit: 10,
host: 'localhost',
port: '3306',
user: 'root',
password: 'root'
});
function addConnection(req, res) {
pool.getConnection(function (err, connection) {
if (err) {
connection.release();
res.json({ "code": 500, "status": "Error" });
return;
}
connection.query("select * from user", function (err, rows) {
connection.release();
if (!err) {
res.json(rows);
}
});
connection.on('error', function (err) {
res.json({ "code": 500, "status": "Error" });
return;
});
});
}
For mysql database I have the max_connections set to 200(SHOW VARIABLES LIKE 'max_connections'; returns 200).
With the pool connectionLimit set to 10 for each of the microservice, in which cases or scenarios will the number of connections for any of the microservice will go above 10?
i.e. When and how the node services would be able to maintain more connections then expected?
If I have 3 instances running of same microservice then how does the pool connectionLimit works? What would be the limit of connections for each instance of microservice?
In one of the microservice say I have two apis which does database transactions and both connects to the database(getting connection) through two different functions having
same implementation of mysql.createPool({}) as above. What would happen if both apis are called concurrently and the number of requests made for each of them per second is 100 or more?
Will the number of connections made available be 10 or 20(since there are two mysql pools created with a connectionLimit of 10 each)?
Ideally it would not; but it can go above 10; if suppose some connections become stale i.e. they are closed from client end but are still open on Server end.
If you have multiple instances of same micro-service deployed in multiple VM or docker containers; they are like independent services.. and they have no connection among each other.. Hence, each of them will create its own 10 connection.
Firstly if u set connection pool limit as 10; that does NOT mean that during first moment 10 connections would be created.. While creating a pool; you also specify initial connection parameter suppose 5.. so, when service starts only 5 connections would be created.. and more created only when needed.. with UPPER Limit set as defined by parameter max_connections. Coming back to your question; well if you have NOT implemented synchronization etc. properly then yes it is possible that both pools will initialize their INITIAL_CONNECTIONS..
I am trying to learn node.js so that I can actually get started on working on my personal project. I have been trying to follow the examples in the book "Learning node.js" (by Marc Wandschneider). I, at this point, decided to forgo practicing his example, and go straight into using his code as framework for my practice code.
In my practice code, all I am trying to do is connect to my RDS database (no, I am not using Elastic Beanstalk, btw), and output contents of one of the tables. Seems simple enough, but when I whip up the code for it (based on the book), it seems to attempt connection, but get hung up in the process. This is my code:
var pool = require('generic-pool');
var async = require('async');
var mysql = require('mysql');
var host = "<database-name>.cu8hvhstcity.us-west-2.rds.amazonaws.com",
database = "<database-name>",
user = "<username>",
password = "<someLongPassword>";
var dbClient;
async.waterfall([
// 1. establish connection to database
function (callback) {
console.log("Connecting to database " + database + "...");
dbClient = mysql.createConnection({
host: host,
database: database,
user: user,
password: password,
port: 3306
});
dbClient.connect();
},
// 2. select all from a table (let's go for locations)
function (cb)
{
var query = "SELECT * FROM locations"
console.log("running query \"" + query + "\"...");
dbClient.query(query, cb);
},
function (rows, fields, callback)
{
console.log(fields);
for (var i = 0; i < rows.length; i++)
{
console.log(JSON.stringify(rows, null, '\t'));
}
}
], function (err, results) {
if (err)
{
console.log("An error occurred...");
console.log(err);
}
else
{
console.log("Everything successfully completed!");
}
dbClient.end();
})
This is better than first attempt, when I put a database member to the argument passed to mysql.createConnection(), and it complained that database was unknown. In neither case did either "An error occurred..." nor "Everything successfully completed!" output to the window.
Is there any async stuff going on that is resulting in some kind of non-terminating infinite loop or something? How do I fix this?
The book has an associated GitHub page
NOTE:
Neither my example nor the cited GitHub code make use of that pool variable, so it can simply be commented out. All you need to do to run this yourself is to say npm install async,npm install mysql (as well as creating a dummy RDS database to point to, that contains locations table) before copying, pasting, and running this code.
EDIT:
I fixed the issue with database. I realized that the name of the database used '_', not '-'. Same issue (code hangs) still persists...
I did the following:
In the second function in the array, I needed two parameters, not one.
I fixed thus:function(results, cb)
The third function simply needed to callback(null)
I'm trying to figure out what the best time to actually initialize connections for mysql in node is.
Am I supposed to create a pool of connections and then set them to some global so that all my models have access to the pool? Or am I supposed to initialize connections whenever I'm doing queries?(Seems bad).
I'm sure there's some "proper" way to do it, but I'm not really certain what the best way is.
If you are going to pool connections, then don't initialize connections right when they're needed. When not using a pool, you can just store connection information when your application is starting up, and use it when you need it:
var mysql = require('mysql');
var connection = mysql.createConnection({
host: 'localhost',
user: 'me',
password: 'secret'
});
Then for single use cases:
connection.connect();
connection.query('SELECT 1 + 1 AS solution', function(err, rows, fields) {
// we are done with the connection
connection.end();
if (err) throw err;
console.log('The solution is: ', rows[0].solution);
});
If you're pooling, you should just create a connection pool when your application is starting and fetch connections when you need them. You should not make more than one pool.
var mysql = require('mysql');
var pool = mysql.createPool({
host: 'example.org',
user: 'bob',
password: 'secret'
});
Then when you need a connection, you'd do something like this:
pool.getConnection(function(err, connection) {
connection.query( 'SELECT something FROM sometable', function(err, rows) {
// we are done using the connection, return it to the pool
connection.release();
// the connection is in the pool, don't use it here
});
});
After more research, think I've figured out the right way.
1) Create a connection pool on app start
2) Include that file in your models.
3) Get a connection from the pool.
In the interest of keeping your code cleaner, I think you can also just invoke the pool object directly, according to the manual at https://github.com/felixge/node-mysql. This should abstract the logic for getting and releasing connections from the pool.
EG:
var result = yield pool.query("SELECT * FROM users");
(I'm using co-mysql with support for generators, but syntactically it should be the same w/out the callbacks)