Knex creating database and inserting data to it - mysql

I am using Knex with node.js to create a table and insert some data to it. First I was first creating table and then inserting data but it ended up so that sometimes table was not created yet when data was going to be inserted. Then I ended up using callbacks like below. Now I'm mixing callbacks and promises and I'm not sure if it's very good thing. What could I do to make following work without callback and still take care that table is created before inserting data?
function executeCallback(next, tableName) {
knex.schema.hasTable(tableName)
.then((exists) => {
if (!exists) {
debug('not exists');
// Table creation for mysql
knex.raw(`CREATE TABLE ${tableName} ( id INT(6) UNSIGNED AUTO_INCREMENT PRIMARY KEY, timestamp BIGINT NOT NULL, deviceId VARCHAR(255) NOT NULL, data JSON )`)
.then((rows) => {
debug(rows);
next('Table created (mysql)');
})
.catch((err) => {
debug(`Error: ${err}`);
next(`Error: ${err}`);
});
} else {
debug('Table exists');
next('Table exists');
}
});
}
.
executeCallback((response) => {
debug('back from callback', response);
debug('insert');
knex(req.body.tableName).insert({
timestamp: req.body.timestamp,
deviceId: req.body.deviceId,
data: req.body.data,
})
.catch((err) => {
debug(`Error: ${err}`);
res.status(500).json({ success: false, message: `Error: ${err}` });
})
.then((dataid) => {
debug(`Inserted with id: ${dataid}`);
res.status(201).json({ success: true });
});
}, req.body.tableName);

In general mixing callbacks and Promises is discouraged. I would suggest looking into the async/await pattern for using Promises, as that is often easier to read in code. It works well with knex js too.
One trick with Node callbacks is the convention of the function parameters, where the first parameter is the error, and the second is the success result. Like this: function (error, results) {...} This makes the result easy to check, like
if(error) {
// do error stuff
return
}
// do success stuff with `results`
One could call that function like next(new Error('bad')) for an error, or next(null, 'success object') for a success.
Your callback next is only taking one parameter, and you are not checking its value. It matters whether the result was 'Table Exists' 'Table Created' or 'Error' to what you do next.
You might try something like this:
async function handleInsert(tableName, res) {
try {
let hasTable = await knex.schema.hasTable(tableName)
if(!exists) {
let createResult = await knex.raw(`CREATE TABLE...`)
// check create results, throw if something went wrong
}
//table guaranteed to exist at this point
let insertResult = await knex(req.body.tableName).insert({
timestamp: req.body.timestamp,
deviceId: req.body.deviceId,
data: req.body.data,
})
debug(`Inserted with id: ${insertResult}`) //might need insertResult[0]
res.status(201).json({ success: true })
} catch(err) {
// any error thrown comes here
console.log('Server error: ' + err)
res.error('Bad thing happened, but do not tell client about your DB')
}
}
One more thing. In general, you can either assume the tables you need exist already. Or use a migration to build your DB on server start/update.

Related

How to send Post request for each iteration of an array in Node/Express?

I have some raw json that I'm trying to send to my back end server in mysql. I'm currently trying to loop through the specific array in the json that I need and sending data from each of the children in the array via a POST request but I am getting "Cannot set headers after they are sent to the client".
app.post('/reddit-import', function (req, res) {
console.log("Route /reddit-import POST");
let data = req.body.data.children
data.forEach(child => {
let sql1 = `CALL insert_user('${child.data.author}',
'${child.data.author_fullname}');`
connection.query(sql1,
data,
function (errQuery, result) {
if (errQuery) {
console.log(errQuery);
res.json({status: "Error", err: errQuery});
res.end();
} else {
console.log("Insert ID: ", result.insertId);
res.json({status: result.insertId, err: ""});
res.end();
}
}
);
When I send the POST request, my backend gets 2 rows of data before it hits me with the error message...any ideas?
You seem to be ending your outer response in the data.forEach with a res.end(), which I’m assuming is used to indicate the end of the outer HTTP request to the client. Did you perhaps mean to use “result” there instead?
Try this if you need to keep track insert IDs:
app.post('/reddit-import', function(req, res) {
console.log("Route /reddit-import POST");
let data = req.body.data.children
const insertIds = data.map(child => {
return new Promise((resolve, reject) => {
const sql = `CALL insert_user('${child.data.author}', '${child.data.author_fullname}')`;
connection.query(sql, (err, result) => {
if (err) {
console.log(err);
return reject(err);
}
console.log("Insert ID: ", result.insertId);
return resolve(result.insertId);
});
});
});
return Promise.all(insertIds)
.then(ids => {
return res.json({
insertIds: ids
});
})
.catch(err => {
return res.status(500).json({
message: 'got query error'
});
});
});
What this basically does is that on each query, you keep track of the insert IDs. We need to use Promises because the query() function is asynchronous, meaning it runs independently and there's no other way to keep track of the data outside of its function(err, result) callback. Now we have an array of Promises which contains the insert IDs, and what's left is to send a response that this is successful. And in order to do that, we can't simply do res.json(insertIds) because insertIds is an array of Promises and we still need to extract the values. We can easily extract all data at once from an array of Promises by using Promise.all(insertIds).then(ids => ...). If you wish to send a response informing that the request is successful, do so in this then callback. Lastly and most importantly, we handle errors in a Promise chain's .catch() block. This is where you want to send a response informing the client that there are errors.
Some things that we can improve from this solution is to implement rollbacks in case we have errors, and of course validations of parameters. Unfortunately I have to leave this to the OP to implement.
Also, keep in mind you should only send a response once and only once each request.

Node.js - Use asynchronous function to get a return value in a synchronous way without callbacks

I have a function to retrieve a list of UserID's from a mysql database.
function GetUsers(callback) {
UpdateLogFile('Function Call: GetUsers()')
var users = []
Database.execute( connectionStr,
database => database.query('select UserID from Users')
.then( rows => {
for (let i = 0; i < rows.length; i++){
users.push(rows[i].UserID)
}
return callback(users)
})
).catch( err => {
console.log(err)
})
}
For Reference:
Database class which came from here
const mysql = require( 'mysql' )
class Database {
constructor( config ) {
this.connection = mysql.createConnection( config )
}
query( sql, args ) {
return new Promise( ( resolve, reject ) => {
this.connection.query( sql, args, ( err, rows ) => {
if ( err )
return reject( err )
resolve( rows )
})
})
}
close() {
return new Promise( ( resolve, reject ) => {
this.connection.end( err => {
if ( err )
return reject( err )
resolve()
})
})
}
}
Database.execute = function( config, callback ) {
const database = new Database( config )
return callback( database ).then(
result => database.close().then( () => result ),
err => database.close().then( () => { throw err } )
)
}
After hours of learning about promises and callbacks, I was finally able to get GetUsers() to at least work and return what I'm looking for. However, I seem to only be able to use it as such:
GetUsers(function(result){
// Do something with result
})
But I would really like to be able to have a traditional return statement in the function so that I could use it like this: var users = GetUsers(). I have seen posts saying that this is impossible due to the nature of asynchronous functions but I am still hopeful since I would really like to be able to avoid callback hell. I tried the code below but "users" just results as undefined after execution. So, my main goal is to be able to get the return value from GetUsers() without chaining callbacks together since I have other functions that behave similarly. Is this possible?
var users
GetUsers(function(result){
users = result
})
console.log(users)
This is a very confusing topic, and it took me a while to really understand why what you are asking simply is not possible (at least, in the exact way you are asking). For the examples I will using python Django and Node.js to compare.
Sync
def synchronous():
print('foo') //this will always print first
print('bar')
def getUsers():
with connection.cursor() as cursor:
cursor.execute('SELECT * FROM USERS') //this query is executed
users = cursor.fetchall()
print('foo') //this doesn't trigger until your server gets a response from the db, and users is defined
print(users)
Async
function asynchronous() {
console.log('foo'); //this will also always print first
console.log('bar');
}
function getUsers() {
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) { //this is a "callback"
console.log(users); //this will print
//everything inside of here will be postponed until your server gets a response from the db
});
console.log('foo') //this will print before the console.log above
console.log(users); //this will print undefined
//this is executed before the query results are in and will be undefined since the "users" object doesn't exist yet.
}
A callback is simply the function that your server is supposed to run once you get a response. We typically use the actual word "callback" like this:
function getUsers(callback) {
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) {
if (error) throw error; //always do your error handling on the same page as your query. Its much cleaner that way
callback(users) //server asks what to do with the "users" object you requested
});
}
Now on somewhere else on your server:
getUsers(function(users) {// the callback gets called here
console.log(users); //do what you want with users here
});
The getUsers function takes some other function (ie a callback) as its argument and executes that function after you perform your query. If you want to do the same thing without using the word "callback", you can use an await/async function like fsociety, or you explicitly write out your code and not make functions that take other functions as their arguments.
This is functionality identical to the code from above:
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) {
if (error) throw error;
console.log(users);
});
Callback hell is inevitable, but it really Isn't too bad once you get the hang of it.
Use an async-await function instead.
async function GetUsers(callback) {
try {
UpdateLogFile('Function Call: GetUsers()')
var users = []
let rows = await Database.execute( connectionStr,
database => database.query('select UserID from Users')
for (let i = 0; i < rows.length; i++){
users.push(rows[i].UserID)
}
return callback(users)
} catch(err) {
console.log(err)
}
}
Hope this helps!

Why is my AWS Lambda node.js mysql query not returning?

I'm trying to write some external data into some local tables. We'll be looping through an array, writing most of the data in each array element to the main table and the rest to related tables, replacing all the data each time.
I've stripped the code down to the bare bones to show the problem I'm having. The DELETE runs fine, but the INSERT runs only once, and doesn't even return.
I have a screenshot of the output at https://imgur.com/a/zA6Hz8g .
In it, you can see that the code for the DELETE runs fine (ComQueryPacket sent, OkPacket returned) but when it gets to the INSERT, the ComQueryPacket is sent but nothing is returned. And then the code just falls through.
This results in the first row writing successfully, but no subsequent rows get written.
I've tried changing the connection to use pools, but that didn't help either.
Any ideas?
var mysql = require('mysql');
var promise = require('promise');
const con = mysql.createConnection({
<connectionInfo>,
debug: true
});
function connectToDB() {
return new promise((resolve, reject) => {
console.log("IN connectToDB");
con.connect(function(err) {
if (err) {
console.log("ERROR: Could not connect -- " + err);
reject;
}
console.log("Connected!");
resolve();
});
});
}
function deleteExistingMainRow() {
return new promise((resolve, reject) => {
var query = "DELETE FROM my_table";
con.query(query, [],
function(err, result) {
if (err) {
console.log("ERROR in deleteExistingMainRow: " + err);
reject;
}
else {
console.log("DEBUG: Successful delete of main row");
resolve();
}
});
});
}
function writeMainRow(data_row) {
return new promise((resolve, reject) => {
console.log("IN writeMainRow");
var query = 'INSERT INTO my_table SET id = ?';
con.query(query, [data_row.id],
function(err, result) {
console.log("YES we tried to query");
if (err) {
console.log("ERROR in writeMainRow: " + err);
reject(err);
}
else {
console.log("DEBUG: Successful write of main row");
resolve();
}
});
});
}
exports.handler = function(event, context) {
connectToDB().then(function(script) {
deleteExistingMainRow().then(function(script) {
var data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
data.forEach(data_row => {
writeMainRow(data_row).then(function(script) {
console.log("DEBUG: Main row written in forEach");
},
function(err) {
if (err) { console.log("ERR"); } process.exit(0);
}());
});
console.log("DEBUG: Hey we're exiting now");
con.commit;
con.end(function(err) {
console.log("Error on con end: " + err);
});
context.done(null, "DONE");
process.exit(0);
});
});
};
Just a few moths ago AWS made Node.js v 8.10 runtime available in lambda.
Which means, you can use async/await and Promises. So, we can rearrange code to something like this:
exports.handler = async (event, context) => {
const dbConnection = await connectToDB();
await deleteExistingMainRow();
const data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
// use here for...of loop to keep working with async/await behaviour
for(const data_row of data){
await writeMainRow(data_row);
}
}
Also, you can rewrite your code to use native Promises or async/await functions.
And of course, cover logic on try/catch block, I've skipped them for simplicity.
The reason why your code is not behaving as you expect is because of the asynchronous nature of NodeJS.
Your for_each loop spawns several threads that are going to INSERT the data in your database.
As soon as these threads are started, the rest of the code will execute, starting with console.log("DEBUG: Hey we're exiting now");
So the commit happens before all the INSERT calls are done and, more importantly, you're calling Process.exit() in your code. This terminates the runtime, even before the INSERT can finish.
Call callback() instead as per https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
Handling your multiple asynchronous writes can be done differently. First, as grynets commented before me, I would strongly suggest to rewrite your code using async/await to make the call easier to read.
Then, you have to understand that each call to writeMainRow will return its own Promise and your code must wait for ALL promises to complete before to commit() and to callback()
Promise.all(...) will do that for you. See the doc at https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
try using
INSERT INTO table_name(id) VALUES (?);
I know both your query and the above query works the same. Just give it a Try.
And just make sure your for loop is working properly sending values to the writeMainRow(function). It wouldnt show an error you if pass an empty value and make sure you are not passing the same values in the for loop. And i think you have to pass writeMainRow(data_row.id) rather than writeMainRow(data_row).
Hope this helps.
And one more suggestion if you are updating multiple rows there are options in mysql node library like transactions. Using those functions will be more effective and you can roll back the result if you face error. Other option is to write procedures, in which case you mysql server will bear the computation.

How to validate a field based on another model data

I'm trying to validate a date field based on information stored in the database through another model.
When I test the api, validation works correctly throwing the exception, however, the insertion occurs before this exception. That is, it does not prevent the insertion in thedatabase. Where I went wrong?
This is my validate function:
module.exports = (sequelize, DataTypes) => {
const Step = sequelize.define('Step', {
...
resultDate: {
type: DataTypes.DATE,
validate: {
isEven(value){
sequelize.models.Call
.findById(this.call_id)
.then(call => {
if(value >= call.endingDate) throw new Error('Error message here!');
});
...
And this is the result:
Executing (default): SELECT [...] `Call`.`id` = '19c7e81e-5c23-4fd5-8623-0170deee6cd4');
Executing (default): INSERT INTO `Steps` [...];
Unhandled rejection Error message here!
Clearly, the initial SELECT is to perform validation, however, before the validation quit and throw the exception, the API inserts into the database and returns success already!
How do I ask the model to wait for all validations before inserting?
By changing custom validator arity (the second argument is a callback) you can change it to an asynchronous handler. So your code should look like this:
module.exports = (sequelize, DataTypes) => {
const Step = sequelize.define('Step', {
...
resultDate: {
type: DataTypes.DATE,
validate: {
isEven(value, next){
sequelize.models.Call
.findById(this.call_id)
.then(call => {
next(value >= call.endingDate ? 'Error message here!' : null)
})
.catch(next);
...

JSON result is showing command of SUM MySQL

I'm getting an effective return calculation of all values column my MYSQL table but when i went to check my api result; the column value sum command is appearing in the result of the JSON.
[
{
"SUM(producFinalPrice)": 6000
}
]
This my router get data from my data base.
router.get('/sell/home-card-last-sell-compared-to-the-previous-day', function (req, res) {
connection.query('SELECT SUM(producFinalPrice) FROM new_sell',
function (err, lastSellComparedToThePreviousDayCardHome, fields) {
if (err) {
throw err;
}
res.send(lastSellComparedToThePreviousDayCardHome);
}
);
});
`
Making my mistake clearer, i'm making the final request from the previous query in AJAX; Through this way.
$(function () {
$.ajax({
type: "GET",
url: "/sell/home-card-last-sell-compared-to-the-previous-day",
success: function (lastSellComparedToThePreviousDayCardHome) {
var $lastSellComparedPreviousDay = $('#last-sell-compared-to-the-previous-day');
$.each(lastSellComparedToThePreviousDayCardHome, function (i, SellPreviousDay) {
$lastSellComparedPreviousDay.append('<li>R$ ' + SellPreviousDay.producFinalPrice + '.</li>');
});
console.log('Sucess return to Last Sell Previous Day!', lastSellComparedToThePreviousDayCardHome);
}
});
});
Basically that's it I could not find anything that would help me..
Thanks for helping me out ;]
Use an alias:
connection.query('SELECT SUM(producFinalPrice) AS productFinalSum FROM new_sell', ...
^^^^^^^^
Then when you parse your JSON in Node check for the productFinalSum key. Actually, I think you could even use SUM(productFinalSum) to access your JSON, but it looks awkward.