I have a script in Node that makes about 4 or 5 requests to MYSQL, but it seems that there is some overload, because after inserting 4 or 5 records I get: "Too many connections"
Below I leave my code
const puppeteer = require('puppeteer');
var fs = require('fs');
const mysql = require('mysql2');
const mysqlConOptions = {
connectionLimit : 10, // default = 10
host: "127.0.0.1",
user: "root",
password: "root123!",
database: "scraperpeli"
};
var con = mysql.createPool(mysqlConOptions);
con.getConnection(function(err) {
if (err) throw err;
console.log("Conectado!");
});
function insertarpost()
{
return new Promise(function(resolve, reject) {
var sql = "INSERT INTO `wp_posts` ( `post_author`, `post_date`, `post_date_gmt`, `post_content`, `post_title`,`post_excerpt`, `post_status`,`comment_status`, `ping_status`,`post_password`, `post_name`,`to_ping`,`pinged`, `post_modified`, `post_modified_gmt`,`post_content_filtered`,`post_parent`, `guid`,`menu_order`, `post_type`, `post_mime_type`,`comment_count`) VALUES (1, '"+fhoy+"', '"+fhoy+"', '"+description+"', '"+title+"','', 'publish', 'open', 'closed','','"+slug+"','','', '"+fhoy+"', '"+fhoy+"','', '0','','0', 'movies','' ,0);";
con.query(sql, function (err, result) {
if (err) throw err;
resolve(result);
console.log("1 registro insertado");
});
});
}
insertarpost().then(obtenerultimopostid);
function obtenerultimopostid() {
return new Promise(function(resolve, reject) {
con.query("SELECT ID FROM wp_posts ORDER BY ID DESC LIMIT 0,1;", function (err, result, fields) {
console.log(result);
var sql = "INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES ("+result[0].ID+",'runtime', '"+runtime+"'),("+result[0].ID+",'original_title','"+titleoriginal+"'),("+result[0].ID+",'Rated','"+cpgrated+"'),("+result[0].ID+",'Country', '"+country+"'),("+result[0].ID+",'date', '"+datemovie+"'),("+result[0].ID+",'imdbRating', '"+repimdb+"'),("+result[0].ID+",'vote_average', '"+reptmdb+"'),("+result[0].ID+",'imdbVotes', '"+quantimdb+"'),("+result[0].ID+",'vote_count', '"+quanttmdb+"'),("+result[0].ID+",'tagline', '"+tagline+"'),("+result[0].ID+",'dt_poster', '"+poster+"'),("+result[0].ID+",'dt_backdrop', '"+backdrop+"'),("+result[0].ID+",'imagenes', '"+backdrops+"'),("+result[0].ID+",'dt_cast', '"+textimgreparto+"'),("+result[0].ID+",'dt_dir', '"+textimgreparto2+"');";
con.query(sql, function (err, result) {
if (err) throw err;
resolve(result);
console.log("1 registro wpmeta insertado");
});
});
}).then(function() {
let c1='';
con.query("SELECT ID FROM wp_posts ORDER BY ID DESC LIMIT 0,1;", function(err, result, fields) {
console.log(result);
for (var i = 0; i < getData.length; i++) {
let source = getData[i]["source"];
let text = getData[i]["text"];
let quality = getData[i]["quality"];
let lang = getData[i]["language"];
new Promise(function(resolve, reject) {
let sql = "INSERT INTO `wp_posts` ( `post_author`, `post_date`, `post_date_gmt`, `post_content`, `post_title`,`post_excerpt`, `post_status`,`comment_status`, `ping_status`,`post_password`, `post_name`,`to_ping`,`pinged`, `post_modified`, `post_modified_gmt`,`post_content_filtered`,`post_parent`, `guid`,`menu_order`, `post_type`, `post_mime_type`,`comment_count`) VALUES (1, '" + fhoy + "', '" + fhoy + "', '', '" + make + "','', 'publish', 'closed', 'closed','','" + make + "','','', '" + fhoy + "', '" + fhoy + "','', '" + result[0].ID + "','','0', 'dt_links','' ,0);";
con.query(sql, function(err, result) {
if (err) throw err;
c1=result.insertId
resolve(result);
console.log("1 registro link insertado");
});
}).then(function() {
console.log(c1);
let sql = "INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES (" + c1 + ",'_dool_url', '" + source + "'),(" + c1 + ",'_dool_type','" + text + "'),(" + c1 + ",'_dool_quality','" + quality + "'),(" + c1 + ",'_dool_lang','" + lang + "');";
con.query(sql, function(err, result) {
if (err) throw err;
console.log("1 registro link meta insertado");
});
// });
});
}
});
}
I don't know why that error comes out, too many queries, am I doing something wrong? thanks.
I have tried increasing the connection limit but nothing, I hope you can guide me to the resolution of the problem
I am trying to make a non-relational DB into a relational DB. So I am starting from data with no unique IDs.
I need to get the result from one SQL call loop through those rows, and for each one, do a SQL SELECT using part of the first result, then another SQL select using the next result, and then a write using IDs from the first and last queries.
I am using Node.js and ES6 promises to keep everything in order, but I seem to be missing something. I was actually trying to do an extra SQL call, and also use that result in the third query, but I am simplifying it to just get one call to feed into another.
Maybe some code will help show what I am trying to do.
Here is my query class that returns promises:
var mysql = require('mysql');
class Database {
constructor() {
this.connection = mysql.createConnection({
host: "localhost",
user: "root",
password: "root",
database: "pressfile"
});
}
query(sql, args) {
return new Promise((resolve, reject) => {
this.connection.query(sql, args, (err, result, fields) => {
if (err) return reject(err);
resolve (result);
});
});
}
close() {
return new Promise((resolve, reject) => {
this.connection.end(err => {
if (err) return reject (err);
resolve();
});
});
}
}
This was stolen pretty much as is from a tutorial site, and this part seems to work pretty well. Then here comes the loop, and the multiple queries:
var contactId;
var address1;
var orgName;
var database = new Database();
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
contactId = result[i].contactId;
orgName = result[i].org;
var sql2 = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
console.log(sql2);
database.query(sql2)
.then(result2 => {
console.log(result2);
var orgId = result2[0].organizationId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return ""; //database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
I know it is kind of unraveling at the end, but I'm not wanting to do the INSERT query until I know I can get it right. Right now in the console, I get a valid organization object, followed by:
`INSERT INTO contact_organization (contactId, organizationId) VALUES (17848, 29)'
17848 is the final contactId that is returned in the for loop. How can I get the contactId that is assigned before the second query. I know I am not doing this asynchronous stuff right.
Try something like this. Just a quick solution. (not tested).
const selectOrg = (result) => {
contactId = result[i].contactId;
orgName = result[i].org;
var sql = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
return database.query(sql);
};
const insertOrg = (result) => {
var orgId = result[0].organizationId;
var sql = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
return database.query(sql);
};
database.query("SELECT * FROM contact")
.then(result => {
const promises = [];
for (var i = 0; i < result.length; i++) {
promises << selectOrg(result)
.then(insertOrg);
}
return Promise.all(promises);
})
.then(allResults => {
console.log(allResults);
})
.catch((err) => {
databse.close();
});
I found a way to do this, but it is kind of cheesy. I included the contactId as a constant in the SQL query to get the organization, so I could then pass the value to the .then, keeping everything in order.
My sql2 statement becomes:
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
Then when that query returns, I can just pull the correct contactId out as result[0].contactId, from the same result I get the organizationId from.
Here is the final code:
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
var contactId = result[i].contactId;
var orgName = result[i].org;
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
database.query(sql2)
.then(result2 => {
var orgId = result2[0].organizationId;
var contactId = result2[0].contactId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
The console.log(result3) returns a bunch of these:
OkPacket {
fieldCount: 0,
affectedRows: 1,
insertId: 0,
serverStatus: 2,
warningCount: 0,
message: '',
protocol41: true,
changedRows: 0 }
And I got one contact_organization inserted for every contact row returned from the first query.
The problem is that when I use ? parameter for passing my values with JSON name-pair values, the mysql row does not insert anything but blank values in the row (using INSERT INTO statement).
Following is a function in my node:
function registerdone(req, res) {
var username = req.body.username;
var password = req.body.password;
var firstname = req.body.firstname;
var lastname = req.body.lastname;
var encryptedPassword = bcrypt.hashSync(password, salt);
console.log("encryptedPassword: " + encryptedPassword);
var getUser = "INSERT INTO users (username, password, firstname, lastname) VALUES ('" + req.param("username") + "','" + encryptedPassword + "','" + req.param("firstname") + "','" + req.param("lastname") + "')";
console.log("Query from registerdone is :" + getUser);
mysql.fetchData(function(err, results) {
if (err) {
throw err;
ejs.renderFile('./views/failRegister.ejs', function(err, result) {
console.log('User with same Username already exists...');
});
} else {
console.log(req.body.username + " Registered !!!");
ejs.renderFile('./views/successRegister.ejs', function(err, result) {
// render on success
if (!err) {
res.end(result);
}
// render or error
else {
res.end('An error occurred');
console.log(err);
}
});
}
}, getUser, queryParams);
}
This works perfectly well when I use:
var getUser = "INSERT INTO users (username, password, firstname, lastname) VALUES ('" + req.param("username") + "','" + encryptedPassword + "','" + req.param("firstname") + "','" + req.param("lastname") + "')";
But when I use :
var getUser = "INSERT INTO users (username, password, firstname, lastname) VALUES ( ? ) ";
var queryParams = {
'username': username,
'password': encryptedPassword,
'firstname': firstname,
'lastname': lastname
};
and send queryParams with the callback function, I get all the values as null in mysql row.
The mysql DAO is:
function getConnection(){
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'root',
database : 'ebay_main',
port : 3306
});
return connection;
}
function fetchData(callback, sqlQuery, queryParams){
console.log("\nSQL Query ::"+sqlQuery);
var connection =getConnection();
connection.query(sqlQuery, queryParams, function(err, rows, fields) {
if(err){
console.log("ERROR : " + err.message);
}
else
{ // return err or result
console.log("DB Results:"+JSON.stringify(rows));
callback(err, rows);
}
});
console.log("\nConnection closed..");
connection.end();
}
You can use a more concise syntax: INSERT ?? SET ?
var table = 'users';
var queryParams = {
'username': username,
'password': encryptedPassword,
'firstname': firstname,
'lastname': lastname
};
connection.query('INSERT ?? SET ?', [table, queryParams ], function(err, rows, fields) {
//...
});
I am trying to Insert Data in Database through Node JS. Code working good showing "Record Inserted" msgs but no rows getting updated in MySQL.
This is the code where i am performing insert operation
connection.query('SELECT * FROM menu WHERE item_name=\'' + userResponces[2].toLowerCase() + '\'', function(err, rows){
if (err) throw err;
else{
i_id = rows[0].item_id;
console.log('i_id ' + i_id);
connection.query('INSERT INTO customer VALUES(default,' + c_name + ',' + c_addr + ',' + c_mob + ')', function(err, res){
if(err.fatal){
console.log(''+err.message);
}
else{
console.log("Record Inserted");
connection.query('SELECT MAX(customer_id) AS c_id FROM customer', function(err, res){
if(err) throw err;
else{
c_id = parseInt(res[0].c_id) + 1;
console.log('c_id ' + c_id);
console.log(i_id + ' ' + c_id + ' ' + qty);
connection.query('INSERT INTO order1() VALUES(default,' + i_id + ',' + c_id + ',' + qty + ',1)', function(err, res){
if(err) throw err;
else
console.log("Record Inserted");
});
}
});
}
});
}
});
In above code SELECT statement working perfectly, so undoubtedly no error in connection. Still this is for connection.
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'nodeuser',
password : 'password',
database : 'foodorder'
});
connection.connect(function(err){
if(!err) {
console.log("Database is connected ...");
} else {
console.log("Error connecting database ...");
}
});
You first condition tests err.fatal.
But if the query returns a SQL error like ER_NO_SUCH_TABLE, err object hasn't a fatal property.
{ [Error: ER_NO_SUCH_TABLE: Table 'bad_table_name' doesn't exist]
code: 'ER_NO_SUCH_TABLE',
errno: 1146,
sqlState: '42S02',
index: 0 }
So, here, you should test on err rather than err.fatal
connection.query('INSERT INTO customer VALUES(default,' + c_name + ',' + c_addr + ',' + c_mob + ')', function(err, res){
if (err){
return console.log(err);
}
else{
console.log("Record Inserted");
// ...
});
Btw, think about escaping values :
connection.query(
'INSERT INTO customer VALUES(default, ?, ?, ?)',
[c_name, c_addr, c_mob],
function(err, res) {
//...
}
);