Store JSON data in MySQL table - mysql

I have a problem with storing JSON data in MySQL table using NodeJS.
JSON data looks like this:
{
"header":
{
"file1":0,
"file2":1,
"subfiles":{
"subfile1":"true",
"subfile2":"true",
}
},
"response":
{
"number":678,
"start":0,
"docs":[
{
"id":"d3f3d",
"code":"l876s",
"country_name":"United States",
"city":"LA"
},
{
"id":"d2f2d",
"code":"2343g",
"country_name":"UK",
"city":"London"
}
]
}
}
and I want to store only fields in docs array (or response object).
I'm trying to get data and store in mysql in this way:
var express = require('express');
var mysql = require('mysql');
var request = require("request");
var app = express();
app.use('/', express.static('../client/app'));
app.use('/bower_components', express.static('../client/bower_components/'));
var server = require('http').createServer(app);
var bodyParser = require('body-parser');
app.jsonParser = bodyParser.json();
app.urlencodedParser = bodyParser.urlencoded({ extended: true });
//mysql connection setup
var connection = mysql.createConnection({
host : "localhost",
port: "3306",
user : "root",
password : "root",
database : "db",
multipleStatements: true
});
request('http://url.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
//console.log(body) //
}
var data = body.toString();
console.log(string);
var query = connection.query('INSERT INTO table SET ?', data, function(err, result) {
// Finish
});
console.log(query.sql);
});
server.listen(3000, function () {
'use strict';
});
In log I got
INSERT INTO table SET '{\n \"header\":{\n \"file1\":0,\n \"file2\":1,\n \"subfiles\":{\n \"subfile1\":\"true\",\n \"subfile2\":\"true\"}},\n \"response\":{\"number\":678,\"start\":0,\"docs\":[\n {\n \"id\":\"d3f3d\",\n \"code\":\"l876s\",\n....
output message, but I don't have data in MySQL table.
Do I need to specify every column in query?

at your //Finish comment you should have added some console.log(err) to see why there was no data inserted.
Here the solution:
var data = JSON.parse(body);
var responseJson = JSON.stringify(data.response);
var query = connection.query('INSERT INTO table SET column=?', [responseJson], function(err, result) {
if(err) throw err;
console.log('data inserted');
});

Related

Node.js, Mysql TypeError: Cannot read property 'apikey' of undefined

I am working on a basic auth middleware for a API it uses Node.js Mysql but if someone puts a incorrect key in auth header and sends the request the entire API crashes heres my code the issue is with the callback but I don't know how to fix that.
var express = require('express');
var app = express();
app.get('/', (request, response) => {
response.sendStatus(200);
});
let listener = app.listen(3000, () => {
console.log('Your app is currently listening on port: ' + listener.address().port);
});
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
database : 'systemdata'
});
connection.connect();
function systemAuth(apikey, callback)
{
connection.query('SELECT apikey FROM systemdata.systemkeys WHERE apikey = ?', [apikey], function(err, result)
{
if (err)
callback(err,null);
else
callback(null,result[0].apikey);
});
}
var auth = function (req, res, next) {
systemAuth(req.headers.apikey, function(err,data){
if (err) {
console.log("ERROR : ",err);
} else {
console.log("result from db is : ",data);
}
if(data == req.headers.apikey) {
next()
}else{
res.status(401).send({"error": "Missing or Invalid API-Key", "apikey": req.headers.apikey, "valid": "false"})
}
})
}
app.use(auth)
You will also have to check whether your result actually contains any rows.
A query not returning any rows is not an error, so err won't be set, if result is an empty array. And accessing an element by an index which does not exist leads to undefined, thus the error you are seeing.
function systemAuth(apikey, callback)
{
connection.query('SELECT apikey FROM systemdata.systemkeys WHERE apikey = ?', [apikey], function(err, result)
{
if (err) // some error with the query
callback(err,null);
else if (!result || result.length == 0) // no matching rows found
callback(new Error("invalid apikey"), null);
else // a matching row is found
callback(null,result[0].apikey);
});
}

Write JSON to mysql database with node.js

I'm trying to write a JSON object (or string, unsure) to my mysql database using node.js. I first retrieved the JSON via an xml url using xml2js. I am able to log the json string result in my console via JSON.stringify, but I am unsure how to proceed from here.
Here is the url I took the xml from: https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml
I would like to write each instance from the JSON string to a row, with the columns as the name of the data. It would look something like this:
Here is my code in index.js, which I enact with node index.js on the console:
var parseString = require('xml2js').parseString;
var http = require('http');
var https = require('https');
var mysql = require('mysql');
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "password",
database: "mydb"
});
function xmlToJson(url, callback) {
var req = https.get(url, function(res) {
var xml = '';
res.on('data', function(chunk) {
xml += chunk;
});
res.on('error', function(e) {
callback(e, null);
});
res.on('timeout', function(e) {
callback(e, null);
});
res.on('end', function() {
parseString(xml, function(err, result) {
callback(null, result);
});
});
});
}
var url = "https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml"
xmlToJson(url, function(err, data) {
if (err) {
return console.err(err);
}
strungout = JSON.stringify(data, null, 1);
console.log(strungout);
//strungout contains my json string
})
con.connect(function(err) {
if (err) throw err;
//below is where I might make an insert statement to insert my values into a mysql table
var sql = someinsertstatement
con.query(sql, function (err, result) {
if (err) throw err;
console.log("records inserted");
res.end();
});
});
As mentioned, when I run the above code in my console, the console returns the JSON, though I am unsure how to assign this to a variable that I can then write into my mysql database.
Alternatively, if there is an easier way to write xml from a website directly to my mysql database, I would certainly appreciate any pointers. I feel like it should be easier than this, but I am new to pretty much all of it.
EDIT:
Adding the JSON. I removed the line breaks to consolidate it. Trying to assign the result '4.68' to a variable.
data = {"site": {"observed": [{"datum": [{"valid": [{"_": "2019-02-21T19:42:00-00:00","$": {"timezone": "UTC"}}],"primary": [{"_": "4.68","$": {"name": "Stage","units": "ft"}}]}]}]}};
Thank you.
This worked on my end. Found that the main data you seek is site.observed.datum
const parser = require('xml2json');
const request = require("request");
var mysql = require('mysql');
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "password",
database: "mydb"
});
var api_url = 'https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml';
function xmlToJson(url, callback){
return request({
method: 'GET',
url: api_url,
}, function (error, response, body) {
if (error) {
return callback({
errorResponse: error,
rowsToInsert: false
});
}else{
let jsonRes = JSON.parse(parser.toJson(body));
let datumResult = jsonRes.site.observed.datum;//I had to log Object.keys multple time to get the
const readyForDB = datumResult.map(x => {
let timeOfReading = x.valid.$t;
let stage = x.primary.$t;
let flow = x.secondary.$t;
return [
timeOfReading, stage, flow
]
});
return callback({
errorResponse: false,
rowsToInsert: readyForDB
});
}
})
}
return xmlToJson(api_url, ({errorResponse, rowsToInsert}) => {
if(errorResponse){
throw callback.errorResponse;
}
return con.connect(function(err) {
if (err) throw err;
//below is where I might make an insert statement to insert my values into a mysql table
var sql = "INSERT INTO forecast (timeOfReading, stage, flow) VALUES ?"
con.query(sql, [rowsToInsert], function (err, result) {
if (err) throw err;
console.log(result.affectedRows + " rows inserted");
});
});
});
Sounds like you have the JSON you want but are unsure how to access data within it. Correct me if I'm wrong.
Lets say you have this JSON object called "test":
{
a:1
b:{
x:2
}
}
You can access the value of 1 by calling test.a, and similarly access the value of 2 by calling test.b.x

Extracting information from reddit json and placing it in mysql database using node.js

I'm trying to extract JSON data from /r/askreddit and put it in a mysql database table called "post". The columns in the table are information such as the title of the post, url of the post, and the username of the poster.
I'm at a complete loss at this point on how to bring the data from the raw JSON into my table from the raw JSON, as I thought it should now be working.
Here is my .js server file, any help is appreciated. Thanks.
/*jshint esversion: 6 */
let mysql = require('mysql2');
let dbInfo = require('./dbInfo.js');
let express = require('express');
let bodyParser = require("body-parser");
let app = express();
// Add static route for non-Node.js pages
app.use(express.static('public'));
// Configure body parser for handling post operations
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/reddit-import', function (req, res) {
console.log("Route for the /r/askreddit POST");
let sql = for (let i=0; i < x.data.children.length; i++) {
"insert into post (post_title, post_date, post_url, user_name) values (?,?,?,?)"
};
let data = [req.body.post_title, req.body.post_date, req.body.post_url, req.body.user_name];
connection.query(sql,
data,
function (errQuery, result) {
if (errQuery) {
console.log(errQuery);
res.json({status: "Error", err: errQuery});
} else {
console.log("Insert ID: ", result.insertId);
res.json({status: result.insertId, err: ""});
}
}
);
});
// Create database connection
console.log('Creating connection...\n');
let connection = mysql.createConnection({
host: dbInfo.dbHost,
port: dbInfo.dbPort,
user: dbInfo.dbUser,
password: dbInfo.dbPassword,
database: dbInfo.dbDatabase
});
// Connect to database
connection.connect(function(err) {
console.log('Connecting to database...\n');
// Handle any errors
if (err) {
console.log(err);
console.log('Exiting application...\n');
} else {
console.log('Connected to database...\n');
// Listen for connections
// Note: Will terminate with an error if database connection
// is closed
const ip = 'localhost';
const port = 8080;
app.listen(port, ip, function () {
try {
console.log('Alumni server app listening on port ' + port);
} catch (err) {
console.log(err);
}
});
}
});

How to provide a mysql database connection in single file in nodejs

I need to provide the mysql connection for modules. I have a code like this.
var express = require('express'),
app = express(),
server = require('http').createServer(app);
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
});
app.get('/save', function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
var query = connection.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
But how we provide one time mysql connection for all the modules.
You could create a db wrapper then require it. node's require returns the same instance of a module every time, so you can perform your connection and return a handler. From the Node.js docs:
every call to require('foo') will get exactly the same object returned, if it would resolve to the same file.
You could create db.js:
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) throw err;
});
module.exports = connection;
Then in your app.js, you would simply require it.
var express = require('express');
var app = express();
var db = require('./db');
app.get('/save',function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
db.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
This approach allows you to abstract any connection details, wrap anything else you want to expose and require db throughout your application while maintaining one connection to your db thanks to how node require works :)
I took a similar approach as Sean3z but instead I have the connection closed everytime i make a query.
His way works if it's only executed on the entry point of your app, but let's say you have controllers that you want to do a var db = require('./db'). You can't because otherwise everytime you access that controller you will be creating a new connection.
To avoid that, i think it's safer, in my opinion, to open and close the connection everytime.
here is a snippet of my code.
mysq_query.js
// Dependencies
var mysql = require('mysql'),
config = require("../config");
/*
* #sqlConnection
* Creates the connection, makes the query and close it to avoid concurrency conflicts.
*/
var sqlConnection = function sqlConnection(sql, values, next) {
// It means that the values hasnt been passed
if (arguments.length === 2) {
next = values;
values = null;
}
var connection = mysql.createConnection(config.db);
connection.connect(function(err) {
if (err !== null) {
console.log("[MYSQL] Error connecting to mysql:" + err+'\n');
}
});
connection.query(sql, values, function(err) {
connection.end(); // close the connection
if (err) {
throw err;
}
// Execute the callback
next.apply(this, arguments);
});
}
module.exports = sqlConnection;
Than you can use it anywhere just doing like
var mysql_query = require('path/to/your/mysql_query');
mysql_query('SELECT * from your_table where ?', {id: '1'}, function(err, rows) {
console.log(rows);
});
UPDATED:
config.json looks like
{
"db": {
"user" : "USERNAME",
"password" : "PASSWORD",
"database" : "DATABASE_NAME",
"socketPath": "/tmp/mysql.sock"
}
}
Hope this helps.
I think that you should use a connection pool instead of share a single connection. A connection pool would provide a much better performance, as you can check here.
As stated in the library documentation, it occurs because the MySQL protocol is sequential (this means that you need multiple connections to execute queries in parallel).
Connection Pool Docs
From the node.js documentation, "To have a module execute code multiple times, export a function, and call that function", you could use node.js module.export and have a single file to manage the db connections.You can find more at Node.js documentation. Let's say db.js file be like:
const mysql = require('mysql');
var connection;
module.exports = {
dbConnection: function () {
connection = mysql.createConnection({
host: "127.0.0.1",
user: "Your_user",
password: "Your_password",
database: 'Your_bd'
});
connection.connect();
return connection;
}
};
Then, the file where you are going to use the connection could be like useDb.js:
const dbConnection = require('./db');
var connection;
function callDb() {
try {
connection = dbConnectionManager.dbConnection();
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (!error) {
let response = "The solution is: " + results[0].solution;
console.log(response);
} else {
console.log(error);
}
});
connection.end();
} catch (err) {
console.log(err);
}
}
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'yourip',
port : 'yourport',
user : 'dbusername',
password : 'dbpwd',
database : 'database schema name',
dateStrings: true,
multipleStatements: true
});
// TODO - if any pool issues need to try this link for connection management
// https://stackoverflow.com/questions/18496540/node-js-mysql-connection-pooling
module.exports = function(qry, qrytype, msg, callback) {
if(qrytype != 'S') {
console.log(qry);
}
pool.getConnection(function(err, connection) {
if(err) {
if(connection)
connection.release();
throw err;
}
// Use the connection
connection.query(qry, function (err, results, fields) {
connection.release();
if(err) {
callback('E#connection.query-Error occurred.#'+ err.sqlMessage);
return;
}
if(qrytype==='S') {
//for Select statement
// setTimeout(function() {
callback(results);
// }, 500);
} else if(qrytype==='N'){
let resarr = results[results.length-1];
let newid= '';
if(resarr.length)
newid = resarr[0]['#eid'];
callback(msg + newid);
} else if(qrytype==='U'){
//let ret = 'I#' + entity + ' updated#Updated rows count: ' + results[1].changedRows;
callback(msg);
} else if(qrytype==='D'){
//let resarr = results[1].affectedRows;
callback(msg);
}
});
connection.on('error', function (err) {
connection.release();
callback('E#connection.on-Error occurred.#'+ err.sqlMessage);
return;
});
});
}
try this
var express = require('express');
var mysql = require('mysql');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
console.log(app);
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "admin123",
database: "sitepoint"
});
con.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
module.exports = app;
you can create a global variable and then access that variable in other files.
here is my code, I have created a separate file for MySQL database connection called db.js
const mysql = require('mysql');
var conn = mysql.createConnection({
host: "localhost",
user: "root",
password: "xxxxx",
database: "test"
});
conn.connect((err) => {
if (err) throw err;
console.log('Connected to the MySql DB');
});
module.exports = conn;
Then in the app.js file
const express = require('express');
const router = express.Router();
// MySql Db connection and set in globally
global.db = require('../config/db');
Now you can use it in any other file
const express = require('express');
const router = express.Router();
router.post('/signin', (req, res) => {
try {
var param = req.body;
var sql = `select * from user`;
// db is global variable
db.query(sql, (err, data) => {
if (err) throw new SyntaxError(err);
res.status(200).json({ 'auth': true, 'data': data });
});
} catch (err) {
res.status(400).json({ 'auth': false, 'data': err.message });
}
});

Node js call function, that access mysql database and returns json result, multiple times

I'm new to Node.js. I have a function 'getFromDb' that accesses a mysql database and returns a json file with some data. What if I have an array of query data and I want to call the same function through a for loop to get a json file for each element of the array?
var http = require('http');
http.createServer(function(req, res) {
console.log('Receving request...');
var callback = function(err, result) {
res.setHeader('Content-disposition', 'attachment; filename=' + queryData+ '.json');
res.writeHead(200, {
'Content-Type' : 'x-application/json'
});
console.log('json:', result);
res.end(result);
};
getFromDb(callback, queryData);}
).listen(9999);
function getFromDb(callback, queryData){
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'xxxx',
password : 'xxxx',
database : 'xxxx',
port: 3306
});
connection.connect();
var json = '';
var data = queryData + '%';
var query = 'SELECT * FROM TABLE WHERE POSTCODE LIKE "' + data + '"';
connection.query(query, function(err, results, fields) {
if (err)
return callback(err, null);
console.log('The query-result is: ', results);
// wrap result-set as json
json = JSON.stringify(results);
/***************
* Correction 2: Nest the callback correctly!
***************/
connection.end();
console.log('JSON-result:', json);
callback(null, json);
});
}
You could use the async library for node for this. That library has many functions that make asynchronous programming in NodeJS much easier. The "each" or "eachSeries" functions would work. "each" would make all the calls to mysql at once time, while "eachSeries" would wait for the previous call to finish. You could use that inside your getFromDB method for your array.
See:
https://github.com/caolan/async#each
var http = require('http'),
async = require('async');
http.createServer(function(req, res) {
console.log('Receving request...');
var callback = function(err, result) {
res.setHeader('Content-disposition', 'attachment; filename=' + queryData+ '.json');
res.writeHead(200, {
'Content-Type' : 'x-application/json'
});
console.log('json:', result);
res.end(result);
};
getFromDb(callback, queryData);}
).listen(9999);
function getFromDb(callback, queryData){
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'xxxx',
password : 'xxxx',
database : 'xxxx',
port: 3306
});
connection.connect();
var arrayOfQueryData = ["query1", "query2", "query3", "query4", "query5"];
var jsonResults = [];
async.each(arrayOfQueryData, function (queryData, cb) {
var data = queryData + '%';
var query = 'SELECT * FROM TABLE WHERE POSTCODE LIKE "' + data + '"';
connection.query(query, function(err, results, fields) {
if (err)
return cb(err);
console.log('The query-result is: ', results);
// wrap result-set as json
var json = JSON.stringify(results);
console.log('JSON-result:', json);
jsonResults.push(json);
cb();
});
}, function (err) {
connection.end();
// callbacks from getFromDb
if (err) {
callback(err);
}
else {
callback(null,jsonResults);
}
});
}
use async module. it is the best one. If u dont want to add new module try following;
var count = 0;
array.forEach(function(element) { //array of the data that is to be used to call mysql
++count; //increase counter for each service call
async.db.call(element, callback); //the async task
}
var data = [];
function callback(err, resp) {
--count;//subtract for each completion
data.push(resp)
if(count == 0) { //return data when all is complete
return data;
}
}
I would recommend the async module though. it is very good practice and useful.