I'm currently trying to insert multiple rows per second into my SQL database. While doing this im coming across that the program is consuming upwards of 1GB ram steadily increasing. The program then crashes due to running out of memory.
I have tried sending the queries in bulk by stringing them together and then running. My current solution does work, however, I'm 90% sure it is the cause of the memory consumption
function insertUsername(id, username, discrim, unix) {
vars = [id, username, discrim, unix];
conn.query("SELECT username, discrim FROM usernames WHERE discordID = ? LIMIT 1", id, function (err, results) {
if (err) throw err;
if (!results[0] == null) {
if (results[0].username != username || results[0].discrim != discrim) {
conn.query(insertUsernameSQL, vars, function (err) {
if (err) throw err;
});
}
}
return;
});
conn.query(insertUsernameSQL, vars, function (err) {
if (err) throw err;
});
}
function insertMessage(unix, id, username, discrim, message, server, sid) {
vars = [unix, id, username, discrim, message, sid, server];
conn.query(insertMessageSQL, vars, function (err) {
if (err) throw err;
});
}
My attempt at a bulk sender:
function transaction_bldr(vars){
sql_transaction.push(vars);
console.log(sql_transaction.length);
if (sql_transaction.length > 1){
conn.query(sql, [sql_transaction], function(err) {
if (err) throw err;
console.log(1);
});
sql_transaction = [];
}
}
After using the first two functions I get this: https://cdn.discordapp.com/attachments/619654531596288000/619963742339203122/unknown.png
Related
My current isolation level for MySQL is tx_transaction = REPEATABLE-READ for each session.
So when I run the below code in different terminals the transactions are serially executed, meaning before the commit of the first transaction, the second would not start.
START TRANSACTION;
SELECT *
FROM test
WHERE id = 4 FOR UPDATE;
UPDATE test
SET parent = 98
WHERE id = 4;
So if I implement this in nodeJS, which of the following would give same result as running two terminals?
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
connection.connect();
let query =
START TRANSACTION;
SELECT *
FROM test
WHERE id = 4 FOR UPDATE;
UPDATE test
SET parent = 98
WHERE id = 4;
connection.query(query, function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
connection.query(query, function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
connection.end();
or using pools
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 10,
host : 'example.org',
user : 'bob',
password : 'secret',
database : 'my_db'
});
let query =
START TRANSACTION;
SELECT *
FROM test
WHERE id = 4 FOR UPDATE;
UPDATE test
SET parent = 98
WHERE id = 4;
pool.query(query, function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
connection.release();
});
pool.query(query, function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
connection.release();
});
My first guess was that the pools would create separate connections and sending queries in same connection would be same as typing in queries in the same terminal. However the documentation says https://github.com/mysqljs/mysql#pooling-connections under introduction section that
Every method you invoke on a connection is queued and executed in
sequence.
and I am not exactly sure what that means.
Also, if I use connection pooling, can I be 100% sure that the concurrently running queries are handled by different sessions? So for example if the pool is not released in the first query, would the second query ALWAYS be executed by another session?
I have done a few tests and realized that Connection Pooling results to the expected outcome.
when I do the following with just connection
let pool = mysql.createConnection({
connectionLimit:10,
host: 'localhost',
user: 'root',
password: 'thflqkek12!',
database: 'donationether'
});
connection.beginTransaction(function (err) {
console.log('first transaction has started');
if (err) {
console.log(err);
return;
}
connection.query(`INSERT INTO users VALUES (null, 0, 'username', 'token')`, function (err, results, fields) {
if (err) {
console.log(err);
return;
}
setTimeout(function () {
connection.commit(function (err) {
if (err) {
console.log(err);
return;
}
console.log('first query done');
connection.release();
})
}, 2000)
});
});
connection.beginTransaction(function (err) {
console.log('second transaction has started');
if(err) {
console.log(err);
return;
}
connection.query(`UPDATE users SET username = 'c_username' WHERE username = 'username'`,function (err, results, fields) {
if(err) {
console.log(err);
return;
}
connection.commit(function (err) {
if(err) {
console.log(err);
return;
}
console.log('second query done');
connection.release();
})
});
});
It leads to following output
first transaction has started
second transaction has started
second query done
first query done
Meaning that the transaction opened by the first connection is ignored and the second transaction finishes before. However, when I use connection pooling for following code,
let pool = mysql.createPool({
connectionLimit:10,
host: 'localhost',
user: 'root',
password: 'thflqkek12!',
database: 'donationether'
});
pool.getConnection(function (err, connection) {
connection.beginTransaction(function (err) {
console.log('first transaction has started');
if (err) {
console.log(err);
return;
}
connection.query(`INSERT INTO users VALUES (null, 0, 'username', 'token')`, function (err, results, fields) {
console.log('first query has started');
if (err) {
console.log(err);
return;
}
setTimeout(function () {
connection.commit(function (err) {
if (err) {
console.log(err);
return;
}
console.log('first query done');
connection.release();
});
}, 2000)
});
});
});
pool.getConnection(function (err, connection) {
connection.beginTransaction(function (err) {
console.log('second transaction has started');
if(err) {
console.log(err);
return;
}
connection.query(`UPDATE users SET username = 'c_username' WHERE username = 'username'`,function (err, results, fields) {
console.log('second query has started');
if(err) {
console.log(err);
return;
}
connection.commit(function (err) {
if(err) {
console.log(err);
return;
}
console.log('second query done');
connection.release();
})
});
});
});
The output is as following
first transaction has started
second transaction has started
first query has started
//2seconds delay
second query has started
first query done
second query done
meaning that the first transaction is blocking the second transaction from executing.
So when the documentation said
Every method you invoke on a connection is queued and executed in sequence
It meant that they are delivered to the database in sequence but it will still be asynchronous and parallel even under transaction. However, connection pooling leads to instantiation of multiple connections and transaction within different pool connection behaves as expected for each transaction.
I'm doing a monitoring system project in which I have Arduino sensors data being sent to a node.js server (thru GET requests) and then stored in a MySQL DB.
Whenvever I successfully send data to the server, it connects to the MySQL DB and queries the last 5 received records to do some processing.
Therefore, I need to store the rows of those 5 records in a variable for later use. Meaning that I have to get rows from a connection.query in a variable.
I read that the fact that I'm not able to do this is because node.js being async. So my questions are:
Is it possible to do the described tasks the way I'm trying?
If not, is there any other way to do so?
I'm not putting the whole code here but I'm running a separated test that also doesn't run properly. Here it is:
var mysql = require('mysql');
var con = mysql.createConnection({
host : "127.0.0.1",
user : "root",
password: "xxxx",
database: "mydb",
port : 3306
});
var queryString = "SELECT id, temp1, temp2, temp3, temp4, level_ice_bank, flow FROM tempdata ORDER BY id DESC LIMIT 5";
con.connect(function(err) {
if (err) throw err;
});
var result_arr = [];
function setValue (value) {
result_arr = value;
}
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
//console.log(rows);
setValue(rows);
}
});
console.log(result_arr);
It logs:
[]
But if I uncomment console.log(rows); it logs what I need to store in the variable result_arr.
Thanks in advance to all.
You're seeing this behaviour because con.query(...) is an asynchronous function. That means that:
console.log(result_arr);
Runs before:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
//console.log(rows);
setValue(rows);
}
});
(Specifically, the setValue(rows) call)
To fix this in your example, you can just do:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
setValue(rows);
console.log(result_arr);
}
});
If you want to do more than just log the data, then you can call a function which depends on result_arr from the con.query callback, like this:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
setValue(rows);
doCleverStuffWithData();
}
});
function doCleverStuffWithData() {
// Do something with result_arr
}
Whats wrong with my query? im having this error:
con.query(
'SELECT nick FROM channels WHERE room=1room',
function(err, rows) {
if (err) throw err;
console.log(rows);
}
);
I tried this, and i have the same error:
var room = "1room";
con.query(
'SELECT nick FROM channels WHERE room=' + room,
function(err, rows) {
if (err) throw err;
console.log(rows);
}
);
It is treating 1room as a variable, not a string. Wrap it in quotes and it should work.
con.query(
'SELECT nick FROM channels WHERE room="1room"',
function(err, rows) {
if (err) throw err;
console.log(rows);
}
);
For your second example, you should get in the habit of escaping the variables you use in queries for security reasons (to prevent SQL injection).
var room = "1room";
con.query(
'SELECT nick FROM channels WHERE room=?',
[room],
function(err, rows) {
if (err) throw err;
console.log(rows);
}
);
Am using node-mysql to add records to a database but am facing a challenge when the records to be inserted are an array of objects and I need the operation to be a transaction. I have simplified my problem by creating a test project to better explain my problem.
Lets say I have to tables users and orders and the data to be inserted looks like this
var user = {
name: "Dennis Wanyonyi",
email: "example#email.com"
};
var orders = [{
order_date: new Date(),
price: 14.99
}, {
order_date: new Date(),
price: 39.99
}];
I want to first insert a user to the database and use the insertId to add the each of the orders for that user. Am using a transaction since in case of an error, I want to rollback the whole process. Here is how I try to insert all the records using node-mysql transactions.
connection.beginTransaction(function(err) {
if (err) { throw err; }
connection.query('INSERT INTO users SET ?', user, function(err, result) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
for (var i = 0; i < orders.length; i++) {
orders[i].user_id = result.insertId;
connection.query('INSERT INTO orders SET ?', orders[i], function(err, result2) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
console.log('success!');
});
});
}
});
});
However I have a problem iterating over the array of orders without having to call connection.commit multiple times within the for loop
I would suggest to construct a simple string for multiple row insert query for orders table in the for loop first and then execute it outside the for loop. Use the for loop to only construct the string. So you can rollback the query whenever you want or on error. By multiple insert query string i mean as follows:
INSERT INTO your_table_name
(column1,column2,column3)
VALUES
(1,2,3),
(4,5,6),
(7,8,9);
You can use Promise.all functionality of Bluebird for this.
var promiseArray = dataArray.map(function(data){
return new BluebirdPromise(function(resolve, reject){
connection.insertData(function(error, response){
if(error) reject(error);
else resolve(response);
}); //This is obviously a mock
});
});
And after this:
BluebirdPromise.all(promiseArray).then(function(result){
//result will be the array of "response"s from resolve(response);
database.commit();
});
This way, you can work all the inserts asyncronously and then use database.commit() only once.
Some kind of task in Node.js are Asynchronous( like I/O , DB and etc..), and there is a lots of LIBS that help to handle it.
but if you want don't use any lib,for iterating an array in JS and use it in an asynchronous functionality its better to implement it as recursive function.
connection.beginTransaction(function(err) {
if (err) {
throw err;
}
connection.query('INSERT INTO users SET ?', user, function(err, result) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
// console.log(result.insertId) --> do any thing if need with inserted ID
var insertOrder = function(nextId) {
console.log(nextId);
if ((orders.length - 1) < nextId) {
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
})
}
console.log(" ok");
});
} else {
console.log(orders[nextId]);
connection.query('INSERT INTO orders SET ?', orders[nextId], function(err, result2) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
insertOrder(nextId + 1);
});
}
}
insertOrder(0);
});
});
as you can see I rewrite your for loop as a recursive function inside.
I would use the async.each to do the iteration and to fire all the queries in parallel. If some of the queries will fail, the asyncCallback will be called with an error and the program will stop processing the queries. This will indicate that we should stop executing queries and rollback. If there is no error we can call the commit.
I' ve decoupled the code a bit more and split it into functions:
function rollback(connection, err) {
connection.rollback(function () {
throw err;
});
}
function commit(connection) {
connection.commit(function (err) {
if (err) {
rollback(connection, err);
}
console.log('success!');
});
}
function insertUser(user, callback) {
connection.query('INSERT INTO users SET ?', user, function (err, result) {
return callback(err, result);
});
}
function insertOrders(orders, userId, callback) {
async.each(orders, function (order, asyncCallback) {
order.user_id = userId;
connection.query('INSERT INTO orders SET ?', order, function (err, data) {
return asyncCallback(err, data);
});
}, function (err) {
if (err) {
// One of the iterations above produced an error.
// All processing will stop and we have to rollback.
return callback(err);
}
// Return without errors
return callback();
});
}
connection.beginTransaction(function (err) {
if (err) {
throw err;
}
insertUser(user, function (err, result) {
if (err) {
rollback(connection, err);
}
insertOrders(orders, result.insertId, function (err, data) {
if (err) {
rollback(connection, err);
} else {
commit(connection);
}
});
});
});
you need to use async library for these kind of operation.
connection.beginTransaction(function(err) {
if (err) { throw err; }
async.waterfall([
function(cb){
createUser(userDetail, function(err, data){
if(err) return cb(err);
cb(null, data.userId);
});
},
function(userid,cb){
createOrderForUser(userid,orders, function() {
if(err) return cb(err);
cb(null);
});
}
], function(err){
if (err)
retrun connection.rollback(function() {
throw err;
});
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
console.log('success!');
});
});
});
var createUser = function(userdetail, cb){
//-- Creation of Orders
};
var createOrderForUser = function (userId, orders, cb) {
async.each(orders, function(order, callback){
//-- create orders for users
},function(err){
// doing err checking.
cb();
});
};
See if you can write a Stored Procedure to encapsulate the queries, and have START TRANSACTION ... COMMIT in the SP.
The tricky part comes with needing to pass a list of things into the SP, since there is no "array" mechanism. One way to achieve this is to have a commalist (or use some other delimiter), then use a loop to pick apart the list.
currentLogs = [
{ socket_id: 'Server', message: 'Socketio online', data: 'Port 3333', logged: '2014-05-14 14:41:11' },
{ socket_id: 'Server', message: 'Waiting for Pi to connect...', data: 'Port: 8082', logged: '2014-05-14 14:41:11' }
];
console.warn(currentLogs.map(logs=>[ logs.socket_id , logs.message , logs.data , logs.logged ]));
I am using https://github.com/felixge/node-mysql
and everytime a mysql query throw an error, for example if a row does not exist. The node server crashes.
connection.connect();
connection.query('SELECT * from table1 where id = 2', function(err, rows, fields) {
if (err) console.log(err);
if (rows[0]) {
console.log('The result is ', rows[0].user);
}
});
connection.end();
How do I simply print the errors to the page rather than crash the server.
If an error occurs, your code console.log's it but tries to access rows[0] anyway. In case of errors rows will be undefined so rows[0] will trigger a new error.
Easily fixed with an else in combination with a length check:
if (err) {
console.log(err);
} else if (rows.length) {
console.log('The result is ', rows[0].user);
} else {
console.log("Query didn't return any results.");
}
I prefer to use the return statement:
connection.connect();
connection.query('SELECT * from table1 where id = 2', function(err, rows, fields) {
if (err) return console.log(err);
if (rows[0]) {
console.log('The result is ', rows[0].user);
}
});
connection.end();
This is cleaner IMO and guarantees that I wont leave anything out of an if statement block where it shouldn't.