Node.js script does not end when using await on sequelize.athenticate() - mysql

This is the code I'm using to start Sequelize and authenticate. I need to await the authenticate() method to make sure database is ready to be used by other app components:
'use strict';
(async () =>
{
let Sequelize=require('sequelize');
let seq = new Sequelize('admin_apptoolset', 'root', 'root',
{
host: 'localhost',
dialect: 'mysql',
operatorsAliases: false,
pool:
{
max: 5,
min: 0,
acquire: 30000,
idle: 10000
}
});
console.log("Trying to connect to database...");
await seq.authenticate();
console.log("Connected to database...");
return;
})()
I can see how both console logs are dumped to the console but the process doesn't exit after the return. Shouldn't it work?
Thanks in advance.

You need to disconnect from database, because it's prevent event loot from exit.
Add seq.close(); before return;
(async () => {
let Sequelize = require("sequelize");
let seq = new Sequelize("admin_apptoolset", "root", "root", {
host: "localhost",
dialect: "mysql",
operatorsAliases: false,
pool: {
max: 5,
min: 0,
acquire: 30000,
idle: 10000
}
});
console.log("Trying to connect to database...");
await seq.authenticate();
console.log("Connected to database...");
seq.close(); // close connection
return;
})();

Related

How to close mySql query connection in nodejs

I am using raw mySql query for my development.
I want to close the query connection after executing the query.
What can I use as per my code?
My Connection Sample:
const Sequelize = require('sequelize')
const dotenv = require('dotenv');
dotenv.config();
const sequelize =
new Sequelize(process.env.DBNAME, process.env.DBUSER, process.env.DBPASS,
{
host: process.env.HOST,
port: process.env.HOST_PORT,
dialect: 'mysql',
operatorsAliases: 0,
timezone: "+06:00",
pool: {
max: 5,
min: 0,
acquire: 30000,
idle: 10000,
}
}
)
module.exports = sequelize;
My Query sample:
const dbConnect = require('../database/db');
let getEmployeeData =
await dbConnect.query(
`Select * From employee where employeeId = '00001'`,
{type: QueryTypes.SELECT});
return res.json({data: getEmployeeData});
Before return I want to close my query connection. As I am getting error "packets_out_of_order" after idle the connection in nodejs, so I decide to test by closing the connection.
Thanks in advance...
You can try with
dbConnect.end();

Node Mysql Serialize 'ERR_BUFFER_OUT_OF_BOUNDS'

so when i try to run the project i get the above mentioned error. I am simply creating 2 tables and then running serialize.sync(). I am new to both node, mysql, and serielize so i have absolutely no clue why is this happening. Also if you wanna see eveything here is the github project: https://github.com/EmilBabazade/BooksBLog/tree/migrate_to_mysql
I also tried deleting blog_books_dev with mysql cli and running again, it gives the same error regardless of wether the database exist or not.
Here is what error looks like:
[nodemon] restarting due to changes...
[nodemon] starting `node index.js`
server is running on port 3001
tedious deprecated The default value for `config.options.trustServerCertificate` will change from `true` to `false` in the next major version of `tedious`. Set the value to `true` or `false` explicitly to silence this message. node_modules/sequelize/lib/dialects/mssql/connection-manager.js:63:26
tedious deprecated In the next major version of `tedious`, creating a new `Connection` instance will no longer establish a connection to the server automatically. Please use the new `connect` helper function or call the `.connect` method on the newly created `Connection` object to silence this message. internal/process/task_queues.js:79:11
internal/buffer.js:75
throw new ERR_BUFFER_OUT_OF_BOUNDS();
^
RangeError [ERR_BUFFER_OUT_OF_BOUNDS]: Attempt to access memory outside buffer bounds
at boundsError (internal/buffer.js:75:11)
at Buffer.readUInt8 (internal/buffer.js:243:5)
at Packet.type (/home/emil/emil/projects/blogBooks/node_modules/tedious/lib/packet.js:143:24)
at IncomingMessageStream.processBufferedData (/home/emil/emil/projects/blogBooks/node_modules/tedious/lib/incoming-message-stream.js:72:26)
at IncomingMessageStream._transform (/home/emil/emil/projects/blogBooks/node_modules/tedious/lib/incoming-message-stream.js:107:10)
at IncomingMessageStream.Transform._read (/home/emil/emil/projects/blogBooks/node_modules/tedious/node_modules/readable-stream/lib/_stream_transform.js:177:10)
at IncomingMessageStream.Transform._write (/home/emil/emil/projects/blogBooks/node_modules/tedious/node_modules/readable-stream/lib/_stream_transform.js:164:83)
at doWrite (/home/emil/emil/projects/blogBooks/node_modules/tedious/node_modules/readable-stream/lib/_stream_writable.js:409:139)
at writeOrBuffer (/home/emil/emil/projects/blogBooks/node_modules/tedious/node_modules/readable-stream/lib/_stream_writable.js:398:5)
at IncomingMessageStream.Writable.write (/home/emil/emil/projects/blogBooks/node_modules/tedious/node_modules/readable-stream/lib/_stream_writable.js:307:11)
at Socket.ondata (_stream_readable.js:708:22)
at Socket.emit (events.js:315:20)
at addChunk (_stream_readable.js:297:12)
at readableAddChunk (_stream_readable.js:273:9)
at Socket.Readable.push (_stream_readable.js:214:10)
at TCP.onStreamRead (internal/stream_base_commons.js:186:23) {
code: 'ERR_BUFFER_OUT_OF_BOUNDS'
}
[nodemon] app crashed - waiting for file changes before starting...
Here is the code for database:
const Sequelize = require('sequelize')
const User = require('./models/user')
const Blog = require('./models/blog')
const logger = require('../utils/logger')
const dbConfigs = require('./config')
let sequelize
if (process.env.NODE_ENV === 'production') {
sequelize = new Sequelize(dbConfigs.production)
} else if (process.env.NODE_ENV === 'development') {
sequelize = new Sequelize(dbConfigs.development)
} else if (process.env.NODE_ENV === 'test') {
sequelize = new Sequelize(dbConfigs.test)
} else {
logger.error('error when connecting to db, invalid or empty NODE_ENV')
// throw something ..?
}
const userModel = sequelize.define('user', User)
const blogModel = sequelize.define('blog', Blog)
userModel.hasMany(blogModel)
blogModel.belongsTo(userModel)
module.exports = {
sequelize,
User: userModel,
Blog: blogModel,
}
Code for user model:
const Sequelize = require('sequelize')
module.exports = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
username: {
type: Sequelize.STRING,
allowNull: false,
unique: true,
validate: {
len: [5, 20],
},
},
name: Sequelize.STRING,
passwordHash: {
type: Sequelize.STRING,
allowNull: false,
},
isAdmin: {
type: Sequelize.BOOLEAN,
defaultValue: false,
},
}
code for blog model:
const Sequelize = require('sequelize')
module.exports = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
title: {
type: Sequelize.STRING,
AllowNull: false,
validate: {
len: [5, 20],
},
},
content: {
type: Sequelize.STRING,
AllowNull: false,
validate: {
len: [20, 500],
},
},
date: {
type: Sequelize.DATE,
defaultValue: Sequelize.NOW,
},
}
and the options for new Serialize(options) (i deleted password because its my main password for everything irl):
const production = 'link to production db'
const development = {
host: 'localhost',
database: 'blog_books_dev',
username: 'root',
password: '',
dialect: 'mssql',
// open mysql command line and type SHOW GLOBAL VARIABLES LIKE 'PORT';
// to see waht port mysql is running on
port: '3306',
}
const test = {
host: 'localhost',
database: 'blog_books_test',
username: 'root',
password: '',
dialect: 'mssql',
port: '3306',
}
module.exports = { production, development, test }
I wrote mssql instead of mysql:
const development = {
host: 'localhost',
database: 'blog_books_dev',
username: 'root',
password: '',
dialect: 'mssql',
// open mysql command line and type SHOW GLOBAL VARIABLES LIKE 'PORT';
// to see waht port mysql is running on
port: '3306',
}
should be:
const development = {
host: 'localhost',
database: 'blog_books_dev',
username: 'root',
password: '',
dialect: 'mysql', // THIS WAS THE PROBLEM
// open mysql command line and type SHOW GLOBAL VARIABLES LIKE 'PORT';
// to see waht port mysql is running on
port: '3306',
}

How to fix too many in _connectionQueue of Pool?

My very simple Node.js code doesn't seem like its connection pool work as it's supposed to do. _connectionQueue of Pool object just gets longer and longer infinitely, and app dies. I mean it does make a pool and there are pre-made connections already, but they are not reusable or insert requests are too many and fast? I'm not sure..
I've tried to put some more connectionLimit like following :
let state = { pool: null }
export const connect = () => {
state.pool = mysql.createPool({
connectionLimit: 200,
host: process.env.DATABASE_HOST || 'localhost',
user: process.env.DATABASE_USER || 'root',
password: process.env.DATABASE_PASSWORD || 'password',
database: process.env.DATABASE_NAME || 'database'
})
}
export const get = () => state.pool
Mostly given job of this server is subscription and insertion. It subscribes several MQTT topics and just tries to insert messages into RDB. About 100 messages arrives every second, and that code looks like this.
mqttClient.on('message', function (topic, message) {
if(topic.includes('sensor')){
try {
const data = JSON.parse(message.toString())
if(validate(data.uuid)){
const params = [data.a, data.b, data.c, ...]
sensor.setStatus(params)
}
} catch(err){
console.error(err)
}
}
}
export const setStatus = (params) => {
const SQL = `INSERT INTO ...`
db.get().query(SQL, params, (err, result) => {
if (err) console.error(err)
})
}
Then, I see this through chrome-devtools
Object
pool: Pool
config: PoolConfig {acquireTimeout: 10000, connectionConfig: ConnectionConfig, waitForConnections: true, connectionLimit: 200, queueLimit: 0}
domain: null
_acquiringConnections: []
_allConnections: (200) [PoolConnection, PoolConnection, …]
_closed: false
_connectionQueue: (11561) [ƒ, ƒ, ƒ, ƒ, …]
_events: {}
_eventsCount: 0
_freeConnections: []
_maxListeners: undefined
__proto__: EventEmitter
__proto__: Object
I've put console.log into setStatus like following :
export const setStatus = (params) => {
const SQL = `INSERT INTO ...`
console.log(`allConnections=${db.get()._allConnections.length}, connectionQueue=${db.get()._connectionQueue.length}`)
db.get().query(SQL, params, (err, result) => {
if (err) console.error(err)
})
}
, and got these.
allConnections=200, connectionQueue=29
allConnections=200, connectionQueue=30
allConnections=200, connectionQueue=31
allConnections=200, connectionQueue=32
allConnections=200, connectionQueue=33
allConnections=200, connectionQueue=34
...
It seems like server created a connection pool very well, but not using those connections. Instead, trying to create a new connection more and more all the time and those requests just get stuck in _connectionQueue.
It appears you are creating a new pool every time you'd like to make a query. The common model is to create a pool once when the application starts, then use connections from that pool as needed (one pool, many connections).
Also if you're using a simple DB model you can simplify access to the pool by making it global. Below is an alternate to your code you might try:
app.js
const mysql = require('mysql');
const connection = mysql.createPool({
host: process.env.DB_HOST || '127.0.0.1',
user: process.env.DB_USER || 'local_user',
password: process.env.DB_PASSWORD || 'local_password',
database: process.env.DB_NAME || 'local_database'
});
global.db = connection;
modules.js
export const setStatus = (params) => {
let SQL = `INSERT INTO ...`
db.query(SQL, params, (err, result) => {
if (err) console.error(err)
console.log(result)
})
}
Documentation for further reference :: https://github.com/mysqljs/mysql#pooling-connections
Edit 1 - Log pool events
db.on('acquire', function (connection) {
console.log('Connection %d acquired', connection.threadId);
});
db.on('connection', function (connection) {
console.log('Pool id %d connected', connection.threadId);
});
db.on('enqueue', function () {
console.log('Waiting for available connection slot');
});
db.on('release', function (connection) {
console.log('Connection %d released', connection.threadId);
});

Mysql Connection timeout with Sequelize

I have a problem when making the connection to MySql in the vagrant box with Nodejs and Sequelize.
I can connect to MySql DB from Navicat or Python, Nodejs and Mysql package. But I couldn't make the connection with Nodejs and Sequelize.
Guys, Do you have any idea?
const Sequelize = require('sequelize');
const sequelize = new Sequelize('blog', 'root', null, {
host: 'localhost',
dialect: 'mysql',
operatorsAliases: false,
port: 3307,
pool: {
max: 5,
min: 0,
acquire: 40000,
idle: 20000
},
});
sequelize
.authenticate()
.then(() => {
console.log('Connection has been established successfully.');
})
.catch(err => {
console.error('Unable to connect to the database:', err);
});
Usually MySQL work on port 3306 (you are doing the request on port 3307).
For another way you can try to use MySQL package: https://github.com/mysqljs/mysql.
Best regards.

unable to set multipleStatements to true in knex node express

I have inherited an node.js app for RESTFUL api, which uses knex pooling to connect to mysql. I have a nned to perform multiple queries in single function (statement), and as I understand, for that I need multipleStatemnt to be set to truu in knex pooling settings. I have done that:
const connection = (user, password) =>
knex({
client: "mysql",
connection: {
multipleStatements: true,
host: process.env.MYSQL_IP,
port: process.env.MYSQL_PORT,
user,
password,
database: "",
dateStrings: true,
}
});
However, this does not seem to get applied and I cannot execute multiple statements in single query (as per documentation in mysql pooling):
var qString = 'SELECT ?;SELECT ?'
self._client.query(qString, [1,5], function (err, result) {
And if I check my client, I see that multipleStatements are still being set to false:
Pool {
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
config:
PoolConfig {
acquireTimeout: 10000,
connectionConfig:
ConnectionConfig {
host: 'xxx.xxx.xxx.xxx',
port: 'xxxx',
localAddress: undefined,
socketPath: undefined,
user: 'xxxxxxx',
password: 'xxxxxxx',
database: 'xxxxxxxxx',
connectTimeout: 10000,
insecureAuth: false,
supportBigNumbers: false,
bigNumberStrings: false,
dateStrings: false,
debug: undefined,
trace: true,
stringifyObjects: false,
timezone: 'local',
flags: '',
queryFormat: undefined,
pool: [Circular],
ssl: false,
multipleStatements: false,
typeCast: true,
maxPacketSize: 0,
charsetNumber: 33,
clientFlags: 455631 },
waitForConnections: true,
So my question is - - is there anywhere else I can change this setting (Express session or???) or if perhaps someone has some ideas?
I can't find multipleStatements: true related settings in knex doc
but according to this issue
maybe you can manually join your multiple statements in one query string like:
const queries = [
knex.table("game0.stats").update(updateCurrency),
knex.table("game1.stats").update(updateCurrency),
knex.table("game2.stats").update(updateCurrency),
];
const multiQuery = queries.join(";");
console.log(multiQuery);
return knex.raw(multiQuery)
.then((result) => {
})
.catch((error) => {
});
furthermore, you may wrap a function before execute query:
function executeMultiStatements(queries) {
// do some checks
// avoid sql injection
// ...
const multiQuery = queries.join(';')
return knex.raw(multiQuery);
}