Pagination in nodejs with mysql - mysql

In my project I need to query the db with pagination and provide user the functionality to query based on current search result. Something like limit, I am not able to find anything to use with nodejs. My backend is mysql and I am writing a rest api.

You could try something like that (assuming you use Express 4.x).
Use GET parameters (here page is the number of page results you want, and npp is the number of results per page).
In this example, query results are set in the results field of the response payload, while pagination metadata is set in the pagination field.
As for the possibility to query based on current search result, you would have to expand a little, because your question is a bit unclear.
var express = require('express');
var mysql = require('mysql');
var Promise = require('bluebird');
var bodyParser = require('body-parser');
var app = express();
var connection = mysql.createConnection({
host : 'localhost',
user : 'myuser',
password : 'mypassword',
database : 'wordpress_test'
});
var queryAsync = Promise.promisify(connection.query.bind(connection));
connection.connect();
// do something when app is closing
// see http://stackoverflow.com/questions/14031763/doing-a-cleanup-action-just-before-node-js-exits
process.stdin.resume()
process.on('exit', exitHandler.bind(null, { shutdownDb: true } ));
app.use(bodyParser.urlencoded({ extended: true }));
app.get('/', function (req, res) {
var numRows;
var queryPagination;
var numPerPage = parseInt(req.query.npp, 10) || 1;
var page = parseInt(req.query.page, 10) || 0;
var numPages;
var skip = page * numPerPage;
// Here we compute the LIMIT parameter for MySQL query
var limit = skip + ',' + numPerPage;
queryAsync('SELECT count(*) as numRows FROM wp_posts')
.then(function(results) {
numRows = results[0].numRows;
numPages = Math.ceil(numRows / numPerPage);
console.log('number of pages:', numPages);
})
.then(() => queryAsync('SELECT * FROM wp_posts ORDER BY ID DESC LIMIT ' + limit))
.then(function(results) {
var responsePayload = {
results: results
};
if (page < numPages) {
responsePayload.pagination = {
current: page,
perPage: numPerPage,
previous: page > 0 ? page - 1 : undefined,
next: page < numPages - 1 ? page + 1 : undefined
}
}
else responsePayload.pagination = {
err: 'queried page ' + page + ' is >= to maximum page number ' + numPages
}
res.json(responsePayload);
})
.catch(function(err) {
console.error(err);
res.json({ err: err });
});
});
app.listen(3000, function () {
console.log('Example app listening on port 3000!');
});
function exitHandler(options, err) {
if (options.shutdownDb) {
console.log('shutdown mysql connection');
connection.end();
}
if (err) console.log(err.stack);
if (options.exit) process.exit();
}
Here is the package.json file for this example:
{
"name": "stackoverflow-pagination",
"dependencies": {
"bluebird": "^3.3.3",
"body-parser": "^1.15.0",
"express": "^4.13.4",
"mysql": "^2.10.2"
}
}

I taked the solution of #Benito and I tried to make it more clear
var numPerPage = 20;
var skip = (page-1) * numPerPage;
var limit = skip + ',' + numPerPage; // Here we compute the LIMIT parameter for MySQL query
sql.query('SELECT count(*) as numRows FROM users',function (err, rows, fields) {
if(err) {
console.log("error: ", err);
result(err, null);
}else{
var numRows = rows[0].numRows;
var numPages = Math.ceil(numRows / numPerPage);
sql.query('SELECT * FROM users LIMIT ' + limit,function (err, rows, fields) {
if(err) {
console.log("error: ", err);
result(err, null);
}else{
console.log(rows)
result(null, rows,numPages);
}
});
}
});

Was looking for a quick solution. maybe would be useful for someone.
SELECT id FROM complexCoding LIMIT ? OFFSET ?
",req.query.perpage,((req.query.page-1) * req.query.perpage)
Do not forget to paginate according to the total count of id divided by perpage

I wrote a pagination class in order to use it on different pages, I used bootstrap to style the links, you can change it if you're not using bootstrap.
Items route
router.get('/items/:page',(req,res) => {
const db = require('mysql'),
Pagination = require('./pagination'),
// Get current page from url (request parameter)
page_id = parseInt(req.params.page),
currentPage = page_id > 0 ? page_id : currentPage,
//Change pageUri to your page url without the 'page' query string
pageUri = '/items/';
/*Get total items*/
db.query('SELECT COUNT(id) as totalCount FROM items',(err,result)=>{
// Display 10 items per page
const perPage = 10,
totalCount = result[0].totalCount;
// Instantiate Pagination class
const Paginate = new Pagination(totalCount,currentPage,pageUri,perPage);
/*Query items*/
db.query('SELECT * FROM items LIMIT '+Paginate.perPage+' OFFSET '+Paginate.start,(err,result)=>{
data = {
items : result,
pages : Paginate.links()
}
// Send data to view
res.render('items',data);
});
});
});
On items view, just print "pages" to generate pagination links
{{ pages }}
pagination.js >> Add this code to pagination.js and import it to any page you want to use pagination
class Pagination{
constructor(totalCount,currentPage,pageUri,perPage=2){
this.perPage = perPage;
this.totalCount =parseInt(totalCount);
this.currentPage = parseInt(currentPage);
this.previousPage = this.currentPage - 1;
this.nextPage = this.currentPage + 1;
this.pageCount = Math.ceil(this.totalCount / this.perPage);
this.pageUri = pageUri;
this.offset = this.currentPage > 1 ? this.previousPage * this.perPage : 0;
this.sidePages = 4;
this.pages = false;
}
links(){
this.pages='<ul class="pagination pagination-md">';
if(this.previousPage > 0)
this.pages+='<li class="page-item"><a class="page-link" href="'+this.pageUri + this.previousPage+'">Previous</a></li>';
/*Add back links*/
if(this.currentPage > 1){
for (var x = this.currentPage - this.sidePages; x < this.currentPage; x++) {
if(x > 0)
this.pages+='<li class="page-item"><a class="page-link" href="'+this.pageUri+x+'">'+x+'</a></li>';
}
}
/*Show current page*/
this.pages+='<li class="page-item active"><a class="page-link" href="'+this.pageUri+this.currentPage+'">'+this.currentPage+'</a></li>';
/*Add more links*/
for(x = this.nextPage; x <= this.pageCount; x++){
this.pages+='<li class="page-item"><a class="page-link" href="'+this.pageUri+x+'">'+x+' </a></li>';
if(x >= this.currentPage + this.sidePages)
break;
}
/*Display next buttton navigation*/
if(this.currentPage + 1 <= this.pageCount)
this.pages+='<li class="page-item"><a class="page-link" href="'+this.pageUri+this.nextPage+'">Next</a></li>';
this.pages+='</ul>';
return this.pages;
}
}
module.exports = Pagination;

Related

nodejs mysql pool connection just idle

I am creating a nodejs module which retrieve some data from a mysql database and insert into another mysql database after some data processing. My main requirement is to make the module alive 24 hours even there is no data in the first database.. it will just keep checking for any new data. But unfortunately the module just doing nothing after few minutes of running. My function is as follows:
var to_pool = mysql.createPool({
connectionLimit: 100,
host: 'localhost',
user: 'username',
password: 'password',
database: 'toDatabase',
multipleStatements: true
});
var from_pool = mysql.createPool({
connectionLimit: 100,
host: 'localhost',
user: 'username',
password: 'password',
database: 'fromDatabase'
});
get_data(to_pool, from_pool);
var items_per_query = 100;
function get_data(to_pool, from_pool) {
from_pool.getConnection(function (err, from_connection) {
if (err) throw err; // not connected!
//main database query
from_connection.query("SELECT p.*, d.uniqueid as imei FROM tc_positions p left join tc_devices d on d.id = p.deviceid order by p.id desc limit " + items_per_query, function (err, result, fields) {
if (err) throw err;
var items = [];
if (Object.keys(result).length > 0) {
Object.keys(result).forEach(function (key) {
var x = result[key];
items.push({ 'id': x['id'], 'table_name': x['imei'], 'table_columns': table_columns_list });
});
}
if (items.length >= items_per_query) {
var items_to_be_removed = [];
let imei_insert = "";
let insert_data = "";
for (var x = 0; x < items.length; x++) {
let all_values = "";
let i = 0;
for (let v of Object.values(items[x]['table_columns'])) {
i++;
all_values += "'" + v + "'";
if (i < Object.keys(items[x]['table_columns']).length) {
all_values += ",";
}
}
insert_data += "INSERT INTO " + items[x]['table_name'] + "(dt_server,dt_tracker,lat,lng,altitude,angle,speed,params,fix_time,accuracy,network) VALUES(" + all_values + "); ";
items_to_be_removed.push(items[x]['id']);
if (items_to_be_removed.length == items_per_query) {
var final_query = imei_insert + ' ' + createTable + ' ' + insert_data;
to_pool.getConnection(function (err, platform_connection) {
if (err) throw err;
platform_connection.query(final_query, function (err, results, fields) {
if (err) throw err;
var ids = items_to_be_removed.join(",");
from_connection.query("DELETE FROM tc_positions where id IN(" + ids + ")", function (err, results, fields) {
if (err) throw err;
console.log('removed ' + items_to_be_removed.length + ' rows from traccar');
items_to_be_removed = [];
insert_data = "";
from_connection.destroy();
platform_connection.destroy();
// after finish all task call the same function again
return get_data(to_pool, from_pool);
});
});
});
}
}
}
else {
setInterval(function () { get_data(to_pool, from_pool); }, 10000);
}
});
});
}
the get_data() function is being called every 10 secs but the "main database query" portion never execute after sometimes. Is there any way to execute the database query again and again as the get_data() function call?
it is better to use a package manager like PM2 and start your script like this
pm2 start app.js
no need to setup intervals in your code, let the code run and exit, PM2 will restart it automatically when it stops running, this will make your code alive 24 hours as per your requirement, you can also setup delays or setup restart strategies

Using multiple queries and render to HTML

I'm trying to complete 2 select with 2 different SQL Server queries in the same HTML.
I'm using this code to complete 1
router.js
app.get('/bajatel', function(req, res, next) {
var data = []
request.query('select NLineas from tb_lineas', function(err, fields){
if(err) {
console.log('error en la busqueda');
return;
}
console.log(fields)
// Pass the DB result to the templates
for (var i = 0; i < fields.recordset.length ; i++){
data.push({NLineas: fields.recordset[i]})
}
res.render('bajatel.html', {Valuelist: data});
console.log(data)
});
});
db.js
var sql = require('mssql');
var config = {
user : 'sa',
password : 'xxxxxxxx',
server : 'xxxxxxxx',
database : 'db_test'
};
var connection = sql.connect(config, function(err) {
if (err){
throw err ;
} else{
console.log('connected');
}
});
module.exports.connection = connection;
I want to add a second query in the app.get to have a second array:
var motivo = []
request.query('Select DescMotivo from tb_Motivos', function(err, rows){
if(err){
console.log('error en la busqueda');
return;
}
for (var i = 0; i < rows.recordset.length; i++){
motivo.push({DescMotivo: rows.recordset[i]})
}
The final render should be:
res.render('bajatel.html',{Valuelist: data, Motivolist: motivo})
but I don't know how to export the data to complete the array outside of the query.
the idea is to have 2 boxes with the queries options and they must select the default values
Could somebody help me?

Node.js Checkbox Iteration

How do I iterate through the checked checkboxes, without hard coding and checking each box one at a time. Kinda like this link, but with Node.js and not JavaScript
Iterate all checkbox in JavaScript:
So I'm attempting to display the contents of my database (using MySQL), but I don't want to hard code 'req.body.checkboxName' for every checkbox in case a new column is added onto a table (the checkboxes are each of the columns in a given table). So I need to find a way that will iterate through every checkbox in the webpage and see if it is checked. If it is then we'll add them together in a string and query for those columns. My code below is for the Node.js POST method (after hitting submit once the checkboxes are checked) and the other is my Pug/Jade code (the JavaScript Templating Engine that I use, sorry it isn't plain html).
In case you're confused how the Pug/Jade file runs initially, another GET method renders that Pug/Jade file with the column names; that GET method is basically the POST method below up until the '//Need help' line
Node.js
app.post('/GetTables', function(req, res){
var columnsRequest = 'DESCRIBE ' + tableName;
var columnsList = [];
connection.query(columnsRequest, function(err, results, fields) {
if(err){
throw err;
}
for (var index in results) {
console.log(results[index].Field);
columnsList.push(results[index].Field);
}
});
//Need help, the next 5 or so lines are non-working ~psuedocode
var checkedList = '';
req.body.CHECKBOXES.each(function(index, element){
if(CHECKBOX.ischecked(){
checkedList += CHECKBOX.name() + ', '
}
});
var mysqlRequest = 'SELECT ' + checkedList + ' FROM ' + tableName;
connection.query(mysqlRequest, function(err, results, fields) {
if(err){
throw err;
}
res.render('webPage', {'columnstodisplay': results});
});
});
Pug/Jade
form(method = 'POST', action = '/GetTables', id = 'tableform')
fieldset
each item in columns
//this loop sets up the checkboxes for each of the columns
p
input(type="checkbox", name=item, value=item)
span #{item}
br
input(type ='submit', value ='Submit')
Thanks in advance!
Sorry for waiting a month to answer my own question. Below is the code that I have. Essentially what I did was when the submit button was clicked, it called a JavaScript function that sees which checkboxes are checked and creates two hidden elements; an array of checked checkboxes and the string to place in the MySQL query (the query asks for the columns/the checkboxes that were checked).
app.post('/GetTables', function(req, res){
var columnsRequest = 'DESCRIBE ' + tableName;
var columnsList = [];
connection.query(columnsRequest, function(err, results, fields) {
if(err){
throw err;
}
for (var index in results) {
console.log(results[index].Field);
columnsList.push(results[index].Field);
}
});
//changed the line below****
var mysqlRequest = 'SELECT ' + req.body.checkboxNames + ' FROM ' + tableName;
connection.query(mysqlRequest, function(err, results, fields) {
if(err){
throw err;
}
//changed the line below********
res.render('webPageName', {'columns': columnsList, 'rows': results, 'fields': fields, 'recheckboxes':req.body.checkboxArray});
});
});
Pug/Jade
form(method = 'POST', action = '/GetTables', id = 'tableform')
fieldset
each item in columns
//this loop sets up the checkboxes for each of the columns
p
input(type="checkbox", name=item, value=item)
span #{item}
br
input(type ='submit', value ='Submit', onclick="sendCBs(document.thisForm, '/gettables')")
script.
function sendCBs(form, path, method) {
var cbNames = '';
var cbArray = [];
var count = 0;
for (var i = 0; i < form.elements.length; i++) {
if (form.elements[i].type == 'checkbox') {
if (form.elements[i].checked == true) {
cbNames += form.elements[i].value + ', ';
cbArray.push(form.elements[i].value);
count++;
}
}
}
if(count > 0){
cbNames = cbNames.replace(/,\s*$/, ""); //remove the last comma if 1 or more checkboxes selected
}
else{
return;
}
method = method || "POST"; //if not specified, method will be post
var inputform = document.createElement("form");
inputform.setAttribute("method", method);
inputform.setAttribute("action", path);
var hiddenField1 = createHiddenInput("checkboxNames", cbNames);
var hiddenField2 = createHiddenInput("checkboxArray", cbArray);
inputform.appendChild(hiddenField1);
inputform.appendChild(hiddenField2);
document.body.appendChild(inputform);
inputform.submit();
}

Express REST API response methods are not recognized

I have this really simple get request that returns json that I am trying to implement. I have followed the tutorials for Express Web Framework REST API, but for some reason I keep getting the same error
ERROR:
TypeError: res.status is not a function
or
TypeError: res.json is not a function
index.js:
var express = require('express');
var router = express.Router();
var pg = require('pg');
var connectionString = 'pg://postgres:postgres#postgres/feed';
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index', { title: 'Express' });
});
router.get('/api/leaderboard', function(resp, req){
var results = [];
pg.connect(connectionString, function(err, client, done){
if(err){
done();
console.log(err);
return res.status(500).json({ success: false, data: err});
}
var query = client.query("SELECT * FROM log WHERE (logged >= date_trunc('week', CURRENT_TIMESTAMP - interval '1 week') AND logged <= date_trunc('week', CURRENT_TIMESTAMP));");
var counter = 0;
var b1 = {};
var b2 = {};
var b3 = {};
var b4 = {};
b1.energy_sum_week = 0;
b2.energy_sum_week = 0;
b3.energy_sum_week = 0;
b4.energy_sum_week = 0;
b1.zne_sum_week = 30000;
b2.zne_sum_week = 30000;
b3.zne_sum_week = 30000;
b4.zne_sum_week = 30000;
query.on('row', function(row){
//results.push(row);
if(row['address'] == 215){
b1.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
}
else if (row['address'] == 1590) {
b2.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
} else if (row['address'] == 1605) {
console.log(row);
b3.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
} else if (row['address'] == 1715) {
b4.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
}
});
query.on('end', function(){
done();
//make zne lower than everything
results.push(b1);
results.push(b2);
results.push(b3);
results.push(b4);
resp.json(results);
});
});
});
module.exports = router;
It seems like it can't recognize the response object. Tried a bunch of different things like passing in the request and response's to the query callbacks, and using promises.
Getting kinda desperate here :/
The res variable doesn't exist in the current context, you probably expect that the line
router.get('/api/leaderboard', function(resp, req){
had this form
router.get('/api/leaderboard', function(req, res){
You are passing resp as the req object and the req as the resp object.
Try changing the order.
router.get('/api/leaderboard', function(req, resp){...}

Node js + postgresql : Generate JSON with Node.js with multiple postgresql

I have just started coding with node.js, I understand that node.js is asynchronous but not sure how to deal with this problem.
I'm querying postgresql and building a JSON as follows,
I added my code :
var express = require('express');
var router = express.Router();
var pg = require('pg');
var client = require('../routes/database.js');
var cookieParser = require('cookie-parser');
var dateFormat = require('date-format');
var async = require('async');
var today = dateFormat(new Date());
router.post('/conversation/my', function(req, res) {
var userId = 59;
var limit = 10;
var offset = 0;
var postgresql = "select id from conversation where party_id = '" + userId + "' and reply_id = 0 order by created_on desc limit " + limit + " offset " + offset + "";
var postsJSON = { };
var arr = new Array();
client.query(postgresql, function(err, data) {
if (err) {
console.log(err);
return rollback(client);
}
var rows = data.rows;
for ( i = 0; i < rows.length; i++) {
var post = rows[i];
var post_obj = {};
post_obj.id = post.id;
getConversationResponse(post.id, function(err, res) {
if (!err) {
post_obj.actor = res;
arr.push(post_obj);
console.log(JSON.stringify(arr));
res.send({
data : arr
});
}
});
}
});
});
function getConversationResponse(conversation_id, cb) {
client.query('SELECT * FROM people WHERE id ='+conversation_id+';', function(err, actor) {
client.query('SELECT * FROM users WHERE id ='+conversation_id+';', function(err, user) {
var actor_obj = {};
actor_obj.id = user.id;
actor_obj.name = user.name;
actor_obj.email = user.email;
client.end();
cb (null,actor_obj);
});
});
}
module.exports = router;
But i am getting error :
{ [Error: write EPIPE] code: 'EPIPE', errno: 'EPIPE', syscall: 'write' }
events.js:72
throw er; // Unhandled 'error' event
^
Error: write EPIPE
at errnoException (net.js:901:11)
at Object.afterWrite (net.js:718:19)
npm ERR! weird error 8
npm WARN This failure might be due to the use of legacy binary "node"
npm WARN For further explanations, please read
/usr/share/doc/nodejs/README.Debian
Your client.end(); should not be in the function. It should be after the first client.query for conversations has completed. Currently what happens is that for each conversation, you call the function to get the people/users, where you end the client. So the next conversation row cannot get the people/users.
Also, ideally you should join the three tables and get all the data in one shot.
var postgresql = "select id, people.*, users.* from conversation, people, users where party_id = '" + userId + "' and people.id=conversation.id and users.id=conversations.id and reply_id = 0 order by created_on desc limit " + limit + " offset " + offset + "";