I'm going to check id of my data if exist in mysql update it and if not create it, but in select result it dosen't work and go to esle section and create action most of the times and got Duplicate entry error. at the end I run this file by infinite loop.
I should tell this when I have a lot data in redis this problem will happen.
I'll be glad if help me, thanks :)
Here is my code :
var
mysql = require('mysql'),
redis = require('redis'),
infiniteLoop = require('infinite-loop'),
client = redis.createClient(),
lop = new infiniteLoop,
config = require( __dirname + '/app/Config').config,
con = function() {
return mysql.createConnection({
host : config.db_host,
user : config.db_user,
password : config.db_pass,
database : config.db_name
});
},
insertDB = function( data ) {
var connection = con();
connection.connect( function( err ) {
connection.query( 'SELECT * FROM real_user WHERE id =' + data.id, function( err, res ) {
if ( err ) {
data = JSON.stringify(data);
client.lpush('aipi', data, function(err,reply) {
if ( err ) throw err;
});
}
console.log( "out", data.id, res.length, res );
if ( typeof res != 'undefined' && res.length > 0 ) {
console.log( "update", data.id );
connection.query( 'UPDATE real_user SET ? WHERE id =' + res[0].id, { request_count: data.request_count, updated_at: data.created_at },
function( err, res ) {
if ( err ) throw err;
});
} else {
console.log( "create", data.id );
connection.query( 'INSERT INTO real_user SET ?', data, function( err, res ) {
if ( err ) throw err;
});
}
connection.end();
});
});
},
run = function() {
var data;
client.brpop('aipi', 0, function(err, reply) {
if ( reply[1] ) {
data = JSON.parse(reply[1]);
insertDB(data);
}
});
};
lop.add(run, []).run();
Fixed it by :
ON DUPLICATE KEY UPDATE
connection.connect( function( err ) {
connection.query( "INSERT INTO real_user SET ? ON DUPLICATE KEY UPDATE updated_at = '" + data.created_at
+ "', request_count = '" + data.request_count + "'",
data, function( err, res ) {
connection.end();
});
});
Related
i am beginner in node.js , i hava an array of json data and i wanted to update my table squad by all the rows of my json array , so i used a loop , and then after i execute , i can only see the json input with console.log(data.players) but there is no response or anything , like the function is dead , no errors , the update is not done , just displaying the entry data
Here the input data:
{"id":1,"email":"d","password":"d","name":"d","price":5,"points":5,"fixture":"d","userid":1,
"players":
[{"id":2724,"firstname":"Lucas Digne","lastname":"Lucas Digne","position":"D","price":0,"rating":"7.3","appearences":1,"goals":0,"assists":0,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/2724.png","teamid":45,"clubid":1,"fixtureid":592848,"points":2},{"id":19599,"firstname":"Emiliano Martínez","lastname":"Emiliano Martínez","position":"G","price":0,"rating":"6.6","appearences":1,"goals":0,"assists":0,"cleansheets":5,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/19599.png","teamid":66,"clubid":1,"fixtureid":592855,"points":11},{"id":2741,"firstname":"Mathew Ryan","lastname":"Mathew Ryan","position":"G","price":0,"rating":"5.0","appearences":1,"goals":0,"assists":0,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/2741.png","teamid":42,"clubid":1,"fixtureid":592851,"points":-1},{"id":289,"firstname":"Andrew Robertson","lastname":"Andrew Robertson","position":"D","price":0,"rating":"7","appearences":1,"goals":0,"assists":0,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/289.png","teamid":40,"clubid":1,"fixtureid":592850,"points":2},{"id":289,"firstname":"Andrew Robertson","lastname":"Andrew Robertson","position":"D","price":0,"rating":"7","appearences":1,"goals":0,"assists":0,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/289.png","teamid":40,"clubid":1,"fixtureid":592850,"points":2},{"id":2726,"firstname":"Kurt Zouma","lastname":"Kurt Zouma","position":"D","price":0,"rating":"7.2","appearences":1,"goals":0,"assists":0,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/2726.png","teamid":49,"clubid":1,"fixtureid":592851,"points":2},{"id":633,"firstname":"İlkay Gündoğan","lastname":"İlkay Gündoğan","position":"M","price":0,"rating":"8.3","appearences":1,"goals":0,"assists":1,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/633.png","teamid":50,"clubid":1,"fixtureid":592852,"points":5},{"id":633,"firstname":"İlkay Gündoğan","lastname":"İlkay Gündoğan","position":"M","price":0,"rating":"8.3","appearences":1,"goals":0,"assists":1,"cleansheets":0,"redcards":0,"yellowcards":0,"image":"https://media.api-sports.io/football/players/633.png","teamid":50,"clubid":1,"fixtureid":592852,"points":5}]}
Here the node.js:
updatefixture:(data,callback)=>{
console.log(JSON.stringify(data));
for(var item of data.players){
pool.query(
'UPDATE squad SET appearences = 1, rating = ? goals = ? assists = ? , cleensheets = ?, redcards = ?, yellowcards = ? , points = ? WHERE id = ? AND fixtureid = ? ',
[
item.id,
item.rating,
item.goals,
item.assists,
item.cleansheets,
item.redcards,
item.yellowcards,
item.points,
item.id,
item.fixtureid
],
);
}
(error,result) => {
if(error){
console.log(error);
return callback(error);
}
return callback(null,result);
}
},
I just want a way to update all rows with the condition that made with where clause and make sure that the rows will be updated, any help for a beginner like me will be so appreciated
It seems you just moved callback of pool.query function outside of the loop.
Try the following:
updatefixture:(data,callback)=>{
function addPlayer(item) {
return new Promise((accept, reject) => {
pool.query('UPDATE squad SET appearences = 1, rating = ? goals = ? assists = ? , cleensheets = ?, redcards = ?, yellowcards = ? , points = ? WHERE id = ? AND fixtureid = ? ',
[
item.id,
item.rating,
item.goals,
item.assists,
item.cleansheets,
item.redcards,
item.yellowcards,
item.points,
item.id,
item.fixtureid
], (error, result) => {
if(error){
reject(error);
} else {
accept(result);
}
});
});
}
console.log(JSON.stringify(data));
Promise.all(data.players.map(addPlayer)).then((result)=>{
console.log(result)
callback(null, result)
}).catch((error)=>{
console.log('Error:', error)
callback(error)
})
},
How to run a loop query inside an async query?
let someRows1, someRows2, someRows3, otherRows1;
.then( rows => {
someRows3 = rows;
someRows3.forEach(function (element) {
database.query( 'UPDATE `orders` SET `amount` = "' + element.value + '" WHERE `id` = "' + element.id + '"' );
});
})
.then( () => {
// do something with someRows and otherRows
});
you could use for await...of link to documentation
I am trying to make a non-relational DB into a relational DB. So I am starting from data with no unique IDs.
I need to get the result from one SQL call loop through those rows, and for each one, do a SQL SELECT using part of the first result, then another SQL select using the next result, and then a write using IDs from the first and last queries.
I am using Node.js and ES6 promises to keep everything in order, but I seem to be missing something. I was actually trying to do an extra SQL call, and also use that result in the third query, but I am simplifying it to just get one call to feed into another.
Maybe some code will help show what I am trying to do.
Here is my query class that returns promises:
var mysql = require('mysql');
class Database {
constructor() {
this.connection = mysql.createConnection({
host: "localhost",
user: "root",
password: "root",
database: "pressfile"
});
}
query(sql, args) {
return new Promise((resolve, reject) => {
this.connection.query(sql, args, (err, result, fields) => {
if (err) return reject(err);
resolve (result);
});
});
}
close() {
return new Promise((resolve, reject) => {
this.connection.end(err => {
if (err) return reject (err);
resolve();
});
});
}
}
This was stolen pretty much as is from a tutorial site, and this part seems to work pretty well. Then here comes the loop, and the multiple queries:
var contactId;
var address1;
var orgName;
var database = new Database();
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
contactId = result[i].contactId;
orgName = result[i].org;
var sql2 = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
console.log(sql2);
database.query(sql2)
.then(result2 => {
console.log(result2);
var orgId = result2[0].organizationId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return ""; //database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
I know it is kind of unraveling at the end, but I'm not wanting to do the INSERT query until I know I can get it right. Right now in the console, I get a valid organization object, followed by:
`INSERT INTO contact_organization (contactId, organizationId) VALUES (17848, 29)'
17848 is the final contactId that is returned in the for loop. How can I get the contactId that is assigned before the second query. I know I am not doing this asynchronous stuff right.
Try something like this. Just a quick solution. (not tested).
const selectOrg = (result) => {
contactId = result[i].contactId;
orgName = result[i].org;
var sql = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
return database.query(sql);
};
const insertOrg = (result) => {
var orgId = result[0].organizationId;
var sql = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
return database.query(sql);
};
database.query("SELECT * FROM contact")
.then(result => {
const promises = [];
for (var i = 0; i < result.length; i++) {
promises << selectOrg(result)
.then(insertOrg);
}
return Promise.all(promises);
})
.then(allResults => {
console.log(allResults);
})
.catch((err) => {
databse.close();
});
I found a way to do this, but it is kind of cheesy. I included the contactId as a constant in the SQL query to get the organization, so I could then pass the value to the .then, keeping everything in order.
My sql2 statement becomes:
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
Then when that query returns, I can just pull the correct contactId out as result[0].contactId, from the same result I get the organizationId from.
Here is the final code:
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
var contactId = result[i].contactId;
var orgName = result[i].org;
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
database.query(sql2)
.then(result2 => {
var orgId = result2[0].organizationId;
var contactId = result2[0].contactId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
The console.log(result3) returns a bunch of these:
OkPacket {
fieldCount: 0,
affectedRows: 1,
insertId: 0,
serverStatus: 2,
warningCount: 0,
message: '',
protocol41: true,
changedRows: 0 }
And I got one contact_organization inserted for every contact row returned from the first query.
I'm using node with Mysql and here's my problem.
I'm trying to add new photos on my database and return it as an array
here is my function :
function addNewPhotos(_id, files) {
var deferred = Q.defer();
var new_photos = []
_.each(files, function (one) {
var data = [
one.path,
_id,
0
]
var sql = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)';
db.connection.query(sql, data, function (err, result) {
if (err)
deferred.reject(err.name + ': ' + err.message);
var sql = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
if (result){
db.connection.query(sql, [result.insertId], function(err, photo) {
if (err) deferred.reject(err.name + ': ' + err.message);
if (photo) {
new_photos.push(photo[0]);
}
});
}
})
})
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise}
The Insert works well but i can't retrieve the results to send them back to the client. (my array is empty)
Thanks.
Always promisify at the lowest level, in this case db.connection.query().
if(!db.connection.queryAsync) {
db.connection.queryAsync = function(sql, data) {
return Q.Promise(function(resolve, reject) { // or possibly Q.promise (with lower case p), depending on version
db.connection.query(sql, data, function(err, result) {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
};
}
Now the higher level code becomes very simple :
function addNewPhotos(_id, files) {
var sql_1 = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)',
sql_2 = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
return Q.all(files.map(function(one) {
return db.connection.queryAsync(sql_1, [one.path, _id, 0]).then(function(result) {
return db.connection.queryAsync(sql_2, [result.insertId]);
});
}));
};
To prevent a single failure scuppering the whole thing, you might choose to catch individual errors and inject some kind of default ;
function addNewPhotos(_id, files) {
var sql_1 = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)',
sql_2 = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?',
defaultPhoto = /* whatever you want as a default string/object in case of error */;
return Q.all(files.map(function(one) {
return db.connection.queryAsync(sql_1, [one.path, _id, 0]).then(function(result) {
return db.connection.queryAsync(sql_2, [result.insertId]);
}).catch(function() {
return defaultPhoto;
});
}));
};
Do the return in your async loop function when all has been done
function addNewPhotos(_id, files) {
var deferred = Q.defer();
var new_photos = [];
var todo = files.length;
var done = 0;
_.each(files, function (one) {
var data = [
one.path,
_id,
0
]
var sql = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)';
db.connection.query(sql, data, function (err, result) {
if (err)
deferred.reject(err.name + ': ' + err.message);
var sql = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
if (result){
db.connection.query(sql, [result.insertId], function(err, photo) {
if (err) deferred.reject(err.name + ': ' + err.message);
if (photo) {
new_photos.push(photo[0]);
}
if(++done >= todo){
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise
}
});
}
else
{
if(++done >= todo){
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise;
}
}
})
})
}
So, I need to change the Mysql Database, which will be used by some asynchronous calls to read from a database.
There is one databases, having a table containing multiple database names, where I could read database names. And change my database accordingly.
But if the old read operation is in process database connection gets modified.
How can I create a new Database connection, retaining the old database connection?
Even when both have 2 different variable names.
Below is index.js routing file.
var mysqldb = require('mysql2-promise')();
var config = require('../config.js');
function scheduledSync( req, res, next ){
var companyIds = [];
var getCompanies = function (){
var deferred = Q.defer();
schema.getModel('Company').find([] , function( err, data ){
if(err){
deferred.reject();
} else{
deferred.resolve( data );
}
});
return deferred.promise;
}
getCompanies().then(function(companyData){
promise.each( companyData, function( company ){
mysqldb.configure( config.sql );
var companyQuery = "SELECT * FROM companies WHERE id = "+company.company_id+"";
return mysqldb.query( companyQuery ).spread(function( rows ){
var value = rows[0];
var lastUpdated = company.lastUpdated.toISOString().replace(/T/, ' ').replace(/\..+/, '');
if( value != null ){
return callAPIs( value, value.id, lastUpdated ).then(function(){
console.log( 'Company '+company.company_id + ' Migrated' );
}, function(error) {
console.log('rejection');
});
}
});
}). then(function(){
response = 'success';
return response;
});
});
}
var callAPIs = promise.method( function( companyData, companyId, lastUpdated ){
config.companysql.database = companyData.dbname;
url = companyData.domainname;
var promises = [
areas.getAreaData( config.companysql, companyId, lastUpdated ),
];
Q.all(promises).then( function( values ){
var lastPromise = [
company.getCompanyData( config.companysql, url, companyId, lastUpdated )
];
Q.all(lastPromise).then(function(){
complete();
complete();
return true;
})
});
});
Below is the config.js file.
var _ENVIRONMENT = 'Development';
//process.env.NODE_ENV = env;
/**
We will configure the database credentials as well as API Layer End point based on the environment selected.
*/
switch (_ENVIRONMENT) {
case 'Development':
var _DBHOST = 'dummyhost'
, _DBUSER = 'xxxxxxx'
, _DBPASS = 'xxxxxxx'
, _DBNAME = 'xxxxxxxx'
, _MONGOURL = 'mongodb://dummyhost:27017/xxxxx'
, _DEBUG = false;
break;
case 'Staging':
var _DBHOST = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
, _DBUSER = 'xxxxxxxxxx'
, _DBPASS = 'xxxxxxxxxxxx'
, _DBNAME = 'xxxxxxxxxxxxxxxx'
, _MONGOURL = 'mongodb://xxxxxxxxxxx:xxxx/xxxxx'
, _DEBUG = false;
break;
// case 'Production':
// break;
default:
}
module.exports = {
sql: {
host : _DBHOST
, user : _DBUSER
, password : _DBPASS
, database : _DBNAME
, debug : _DEBUG
},
companysql: {
host : _DBHOST
, user : _DBUSER
, password : _DBPASS
, database : ''
, debug : _DEBUG
},
mongourl : _MONGOURL
}
in area.js
var companyDB = require('mysql2-promise')();
var getAreaData = function( config, companyId, lastUpdated ){
companyDB.configure( config );
/*
Manipulate Area Table
*/
}
I solved the promise issue with mysql poolcluster.
Posting it if anyone else would need any help
var poolCluster = mysql.createPoolCluster();
poolCluster.add( 'MASTER', config.sql );
schema = promise.promisifyAll(schema);
var getsignupdata = function( compIdUpdate, companyIds, mysqldb ){
var deferred = Q.defer();
var companyQuery = "SELECT * FROM companies WHERE id IN ("+companyIds+") ";
poolCluster.getConnection( 'MASTER', function ( err2, mysqlcon ) {
mysqlcon.query( companyQuery , function( err, rows ){
if( !err ){
for( i in rows ){
var companyId = rows[i].id;
config.companysql.database = rows[i].dbname;
poolCluster.add( 'SLAVE'+companyId, config.companysql );
if( compIdUpdate != null )
rows[i]['lastUpdated'] = compIdUpdate[""+companyId];
}
deferred.resolve( rows );
}
});
});
return deferred.promise;
};
function scheduledSync( req, res, next ){
getCompanies().then(function( data ){
getsignupdata( data.compIdUpdate, data.companyIds, mysqldb ).then(function( companyData ){
console.log("scheduledSync " + companyData);
promise.map( companyData, function( company ){
var lastUpdated;
if(company.lastUpdated != null)
lastUpdated = company.lastUpdated.toISOString().replace(/T/, ' ').replace(/\..+/, '');
else
lastUpdated = null;
if( company != null ){
return callAPIs( company, company.id, lastUpdated ).then(function(){
console.log( 'Company '+ company.id + ' Migrated' );
});
}
}). then(function( err ){
response = 'success';
return response;
});
});
});
}
var callAPIs = promise.method( function( companyData, companyId, lastUpdated ){
poolCluster.getConnection( 'SLAVE'+companyId, function ( err, companyConfig ) {
entity.getEntityData( companyConfig, companyId, lastUpdated ).then(function(){
areas.getAreaData( companyConfig, companyId, lastUpdated );
discount.getDiscountData( companyConfig, companyId, lastUpdated );
holderscheme.getHolderSchemeData( companyConfig, companyId, lastUpdated );
scheme.getSchemeData( companyConfig, companyId, lastUpdated );
skunits.getSkunitData( companyConfig, companyId, lastUpdated );
stockAtWarehouse.getStockAtWareMap( companyConfig, companyId, lastUpdated );
company.getCompanyData( config.companysql, companyData, companyId, lastUpdated );
});
});
});