nodejs mysql module errors - mysql

Question

I have a nodejs module which get result from a mysql database and insert into another mysql database within a given interval. After few queries it just get stuck and throw "ER_CON_COUNT_ERROR" and says Too many connections. My module is as follows:
function get_data() {
const mysql = require('mysql');
const moment = require('moment');
var items_per_query = 1000;
let to_connection = mysql.createConnection({
host: 'localhost',
user: 'username',
password: 'pass',
database: 'todatabase'
});
let from_connection = mysql.createConnection({
host: 'localhost',
user: 'username',
password: 'pass',
database: 'fromdatabase'
});
from_connection.query("SELECT p.*, d.uniqueid as imei FROM tc_positions p left join tc_devices d on d.id = p.deviceid order by p.id desc limit " + items_per_query, function (err, result, fields) {
if (err) throw err;
var items = [];
var table_columns_list = {};
if (Object.keys(result).length > 0) {
Object.keys(result).forEach(function (key) {
var x = result[key];
table_columns_list = {
'dt_server': moment(x['servertime']).format('YYYY-MM-DD HH:MM:ss'),
'dt_tracker': moment(x['devicetime']).format('YYYY-MM-DD HH:MM:ss'),
'lat': x['latitude'],
'long': x['longitude'],
'altitude': x['altitude'],
'angle': x['course'],
'speed': x['speed'],
'params': x['attributes'],
'fix_time': moment(x['fixtime']).format('YYYY-MM-DD HH:MM:ss'),
'accuracy': x['accuracy'],
'network': x['network']
}
items.push({ 'id': x['id'], 'table_name': 'table_' + x['imei'], 'table_columns': table_columns_list });
});
}
if (items.length >=500) {
var items_to_be_removed = [];
var total_rows_inserted = 0;
for (var x = 0; x < items.length; x++) {
to_connection.query(imei_insert, function (err, results, fields) {
if (err) {
console.log(err.message);
}
else {
let createTable = " create table if not exists .... ";
to_connection.query(createTable, function (err, results, fields) {
if (err) {
console.log(err.message);
}
else {
let insert_data = "INSERT INTO .... ";
to_connection.query(insert_data, function (err, results, fields) {
if (err) {
console.log(err.message);
}
total_rows_inserted++;
items_to_be_removed.push(row_id);
if (total_rows_inserted == items.length) {
//remove data from traccar positions table that were inserted to platform
var ids = items_to_be_removed.join(",");
from_connection.query("DELETE FROM tc_positions where id IN(" + ids + ")", function (err, results, fields) {
if (err) throw err;
console.log('removed ' + total_rows_inserted + ' rows from traccar');
return get_data(); // after finish all task call the same function again
});
}
});
}
});
}
});
}
}
else {
setInterval(get_data, 15000);
}
});
}
and I just call get_data(); function to run. Is there any efficient way to run this module smoothly. I should run it 7 days 24 hours without any interruption.

Answer 1

You're creating a new connection to the servers each time get_data() runs, which is overwhelming them. You need to reuse connections, as for example with connection pooling.