Tcp socket server freezes after some time - mysql

Question

I have data coming from different GPS tracker devices. Its a unidirectional data which means I am receiving the data and pushing it into the MySQL DB and firebase. I have a total of 300 Devices connected which are sending data to my server every 10 seconds.
My Server Specs are
AWS t2.xlarge
CPU: 4
Ram: 16GB
what happens is that after 3 days, It stops sending the data into the database. It doesn't stop the server. It just freezes. If I do this
sudo netstat -tulnap | grep :8050
It does show the process and all that but I do not see any data pushing into the DB. It just freezes. I had to reboot the server or I had to stop it using forever and restart it again
forever stop --minUptime 36000000000 server.js
And when I go to my PHPMyAdmin and check the monitor screen what I can see that I have very little free memory left and cache memory is into GBs. It seems like all the memory went into cached memory which left my server freezes. I have no idea where I am doing wrong which is causing it to freeze. For example, at the moment as I am posting a question this is my server current status
As you can see above that in 19 hours cache has been increased and its keep growing. Below is the code
if (cluster.isMaster) {
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('death', function(worker) {
// console.log('worker ' + worker.pid + ' died');
cluster.fork();
});
} else {
net.createServer(function(socket) {
// console.log('received connection...');
socket.on("error", function(err) {
// console.log("socket error: ")
// console.log(err.stack);
socket.destroy();
});
socket.on('data', function(data) {
});
});
}
Recently I have made one change in the code but still it didn't work out which was to close the socket after receiving the data every 5 seconds
socket.on('data', function(data) {
//parse data and push data into db and firebase
socket.end();
});
That's how I am doing MySQL queries
database.js
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 8,
waitForConnections : true,
queueLimit : 300,
host : 'localhost',
user : 'username',
password : '123456',
database : 'dummy'
});
module.exports = pool;
Server.js file (only database code I have pasted as full code has 1400 lines of code)
const db = require('./database');
function getCarDetails(car_id,callback) {
db.getConnection((err, connection) => {
if(err) throw err;
console.log('connected as id ' + connection.threadId);
let selectQuery = 'SELECT * FROM ?? join user_info ON car.user_id = user_info.user_id WHERE ?? = ?';
let query = mysql.format(selectQuery, ["car", "id", car_id]);
// query = SELECT * FROM `todo` where `user` = 'shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}
// rows fetch
if(data.length > 0){
return callback(data[0]);
}else{
return callback(false);
}
});
});
}
function updateIgnitionNotification(car_id,acc_on,acc_off,acc,speed,updated,callback) {
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET notification_acc_on = ?,notification_acc_off = ?,acc = ?,speed = ?,updated = ? Where id = ?';
let query = mysql.format(updateQuery, [acc_on, acc_off,acc,speed,updated, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateLastUpdatedData(car_id,current_datetime,status,acc,monitoring,max_speed,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET updated = ?,status = ?,acc = ?,monitoring = ?, max_speed = ? Where id = ?';
let query = mysql.format(updateQuery, [current_datetime,status,acc.toUpperCase(),monitoring, max_speed, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateCommand(car_id,command,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET command = ? Where id = ?';
let query = mysql.format(updateQuery, [command, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateCarLockNotification(car_id,lock_notification,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET lock_notification = ? Where id = ?';
let query = mysql.format(updateQuery, [lock_notification, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateOverSpeedNotification(car_id,notification_over_speed,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET notification_over_speed = ? Where id = ?';
let query = mysql.format(updateQuery, [notification_over_speed, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateGeoFenceOutsideAttempt(car_id,geofence_attempt,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET geofence_outside_attempt = ? Where id = ?';
let query = mysql.format(updateQuery, [geofence_attempt, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateGeoFenceInsideAttempt(car_id,geofence_attempt,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET geofence_inside_attempt = ? Where id = ?';
let query = mysql.format(updateQuery, [geofence_attempt, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function updateBatteryNotification(car_id,battery,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET notification_battery = ? Where id = ?';
let query = mysql.format(updateQuery, [battery, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
function saveNotificationLog(log,callback){
db.getConnection((err, connection) => {
// let insertQuery = 'INSERT INTO ?? (??,??) VALUES (?,?)';
//let query = mysql.format(insertQuery,["log","user","notes",data.user,data.value]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query('INSERT INTO log SET ?', log, (err, res) => {
connection.release();
if(err) {
return callback(res);
}else{
return callback(res);
}
});
});
}
function saveHistory(history,callback){
db.getConnection((err, connection) => {
// let insertQuery = 'INSERT INTO ?? (??,??) VALUES (?,?)';
//let query = mysql.format(insertQuery,["log","user","notes",data.user,data.value]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query('INSERT INTO car_history SET ?', history, (err, res) => {
connection.release();
if(err) {
return callback(res);
}else{
return callback(res);
}
});
});
}
function updateGeoFenceNotification(car_id,notification_inside,notification_outside,callback){
db.getConnection((err, connection) => {
let updateQuery = 'UPDATE car SET notification_inside = ?,notification_outside = ? Where id = ?';
let query = mysql.format(updateQuery, [notification_inside, notification_outside, car_id]);
// query = UPDATE `todo` SET `notes`='Hello' WHERE `name`='shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}else{
return callback(data);
}
});
});
}
Please help me to resolve this issue

Answer 1

I'll attempt an answer, or at least some observations.
It's probably worth your effort to get your nodejs app working robustly without clustering. Keeping things simple helps with troubleshooting. Once it is solid you can add clustering.
Your server is overprovisioned. The workload you have is 1,800 connections per minute (every six seconds from 300 devices) or 30/sec. A 2-core server with 4-8GiB of RAM should be just fine for this workload if your program is written well. A 1-core server may be enough. And, a program that's expected to stay up for thousands of hours necessarily must be written well. (Disregard this advice if your database server runs on the same virtual machine as your nodejs app.)
You didn't describe how you connect to your database server from your nodejs code. You should use connection pooling to take away the need to close and reopen connections a lot.
Do something like this in your initialization code:
const mysql = require('mysql')
const mySqlPool = mysql.createPool({
connectionLimit : 8, /* make this as small as possible */
waitForConnections : true,
queueLimit : 300, /* enough for 6sec worth of workload */
host : 'dbhost',
user : 'bob',
password : 'secret',
database : 'my_db',
})
/* make the pool available to other code.
* There may be a better way to do this. */
global.mySqlPool = mySqlPool
/* make a little monitor function to
* let you know of database problems.
* this pings the database every 10sec */
let monitorTimeout = setTimeout ( function() {
global.mySqlPool.ping ( function (err) {
if (err) {
clearTimeout(monitorTimeout)
console.error('database connection error', err)
throw err
}
})
}, 10000)
Then when you need to access your database use global.mySqlPool. in place of db. in the code you have. Each query will use a connection from the pool. If all the connections are in use, the query will wait in a queue until a connection is free.
This pooling / queueing strategy puts a much more predictable load on both your nodejs program and your MySQL database.
The same sort of thing should be possible for firebase.

Answer 2

Once you have made database connection pooling work properly, keeping the pool sizes small, your next step for performance improvement is a little harder. But it will make a vast difference.
What you do is this: put your UPDATE operations into a simple queue inside your app. That is, modify your updateWhatever( car_id, whatever ) functions to push() their SQL statements and parameters onto a shared array.
Then, write a function that uses shift() to fetch those items from the array and run them one after the other on the dbms. When the array has multiple items in it, wrap those items in a single database transaction.
This helps performance a lot: most of the MySQL server's work to handle INSERT and UPDATE operations happens when those operations are COMMITed. (If you don't start transactions explicitly, MySQL uses autocommit for every operation.) So if you bundle them together in transaction bundles of a few tens of operations, you put a lot less load on MySQL.
It also reduces your application's potential need to handle many UPDATE operations concurrently on different connections. That in turn reduces contention for access to the tables.
(It's true that the shift() operation on an array has an in-memory performance hit if the array is large. If that turns out to be a problem you can replace the easy-to-program array / push() / shift() queue discipline with a purpose-built queue package.)