Auto-create mysql table with StrongLoop - mysql

I am trying to use Strongloop with MySql but cannot figure out how to migrate or automatically create tables into a MySql database.
Is there at least a way to export the models into MySql schemas or do I have to manually create the tables?
I've been trying with the mysql demo app, and going over the docs for a while but no luck - http://docs.strongloop.com/display/DOC/MySQL+connector
Thanks!

I created /server/boot/autoupdate.js. It runs when the app boots. It loads "model-config" and "datasources" JSON and migrates or updates all models to the datasources defined for them.
# /server/boot/autoupdate.js
module.exports = function(app) {
var path = require('path');
var models = require(path.resolve(__dirname, '../model-config.json'));
var datasources = require(path.resolve(__dirname, '../datasources.json'));
function autoUpdateAll(){
Object.keys(models).forEach(function(key) {
if (typeof models[key].dataSource != 'undefined') {
if (typeof datasources[models[key].dataSource] != 'undefined') {
app.dataSources[models[key].dataSource].autoupdate(key, function (err) {
if (err) throw err;
console.log('Model ' + key + ' updated');
});
}
}
});
}
function autoMigrateAll(){
Object.keys(models).forEach(function(key) {
if (typeof models[key].dataSource != 'undefined') {
if (typeof datasources[models[key].dataSource] != 'undefined') {
app.dataSources[models[key].dataSource].automigrate(key, function (err) {
if (err) throw err;
console.log('Model ' + key + ' migrated');
});
}
}
});
}
//TODO: change to autoUpdateAll when ready for CI deployment to production
autoMigrateAll();
//autoUpdateAll();
};

You can simply migrate models by adding following lines to your server.js file before app.start method:
app.datasources['mySqlConnection'].automigrate(['orders','customers', 'User', 'ACL'], function(err) {
console.log(err);
});
Add models to the array as per your need.
Run the application by slc run.
Note: mySqlConnection is the connection name, replace it by your own connection name.

To update and/or create all mysql tables for your models:
var dataSource = app.dataSources.mysql;
dataSource.autoupdate(null, function (err) {
if(err) return cb(err);
return cb();
});

LoopBack calls it auto-migration. Check these links and search for that term:
Recipes for LoopBack Models, part 5 of 5: Model Synchronization with Relational Databases
Data sources and connectors

In my case, I manually created MySQL tables and then created the models. For existing MySQL tables, I create the models where property names are the same as MySQL field's names.
So here are my steps in using StrongLoop LoopBack with MySQL Database:
Create MySQL Database and Tables (or use existing database).
Install MySQL connector using npm install loopback-connector-mysql --save
Add your MySQL Database details on datasources.json file.
Create a model for each table using slc lb model tablename -i OR edit models.json file and add the properties manually. (document: http://docs.strongloop.com/display/DOC/Creating+a+LoopBack+application#CreatingaLoopBackapplication-Creatingmodels)
Properties' names should be the same as MySQL field's names (more information on mapping MySQL to JSON data types: http://docs.strongloop.com/display/DOC/MySQL+connector#MySQLconnector-MySQLtoJSONtypes)

In the same kind of issue, if you need to automatically create a database, you can use the createDatabase option in your dataSource JSON file.
"mysql": {
"host": "localhost",
"port": 0,
"database": "db",
"username": "root",
"password": "",
"name": "mysql",
"connector": "mysql",
"debug": false,
"createDatabase": true
}
So you don't need to write yourself the queries to create the base.
Hope it helps.

jduhls answer is beautiful, but I needed to tweak it slightly to add some static data into tables. Here's my tweaked version, along with an example of loading data into a simple SystemSettings table (id, settingName, settingValue):
var async = require('async');
var SYSTEM_SETTINGS = [
{
"settingName": "mustPayInAdvance",
"settingValue": "false",
}
];
module.exports = function(app) {
var path = require('path');
var models = require(path.resolve(__dirname, '../model-config.json'));
var datasources = require(path.resolve(__dirname, '../datasources.json'));
var modelUpdates = [];
function buildModelListForOperation(){
Object.keys(models).forEach(function(key) {
if (typeof models[key].dataSource != 'undefined') {
if (typeof datasources[models[key].dataSource] != 'undefined') {
modelUpdates.push({operation: app.dataSources[models[key].dataSource], key: key});
}
}
});
}
function createStaticData() {
app.models.SystemSettings.create(SYSTEM_SETTINGS, function(err, created) {
if (err)
throw err;
else
console.log('Sample data was imported.');
});
}
function processModelsAndData(operationType) {
buildModelListForOperation();
// Create all models
async.each(modelUpdates, function(item, callback) {
item.operation[operationType](item.key, function (err) {
if (err) throw err;
console.log('Model ' + item.key + ' migrated');
callback();
});
}, function (err) {
if (err) throw err;
createStaticData();
});
}
//TODO: change to 'autoupdate' when ready for CI deployment to production
processModelsAndData('automigrate');
};

i discovered an easy way to accomplish this task. The reference link is: Clique Here
You can use prototype or not, in my case, i do nott used.
For the documentation, you should use:
ds.autoupdate (models, function (error) {
if (!error) {
   console.log( "Updated models.");
   }else{
   console.log( "An error has occurred:" + error);
   }
   ds.disconnect();
});
Where:
var path = require ( 'path');
var app = require (path.resolve (__ dirname, '../server/server'));
var ds = app.datasources.x;
and x is datasource attribute name, example of /server/datasources.json:
{
  "x": {
    "Host": "localhost"
    "Port": 3306,
    "Database", "loopapp"
    "Password": "",
    "Name": "x"
    "User", "root"
    "Connector": "mysql"
  }
}
Note (1): Models can be the string model name or the array of string (models names).
Note (2): If you prefer not to put models, all models of the file whose base attribute equals "PersistedModel", will be updated.
With that, i used like this:
autoupdate function () {
ds.autoupdate (function (error) {
     if (!error) {
         console.log( "Updated all models");
      }else {
         console.log( "An error has occurred:" + error);
      }
      ds.disconnect();
   });
}
and i called the: autoupdate();
You can put this code in a file.js and call the command line: node file.js.
If you want this file to be called every time you start the program, put it on /server/boot/file.js path.
Obviously, if you want to use automigrate, only replace the autoupdate word in the code above, by automigrate.

Related

Write JSON to mysql database with node.js

I'm trying to write a JSON object (or string, unsure) to my mysql database using node.js. I first retrieved the JSON via an xml url using xml2js. I am able to log the json string result in my console via JSON.stringify, but I am unsure how to proceed from here.
Here is the url I took the xml from: https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml
I would like to write each instance from the JSON string to a row, with the columns as the name of the data. It would look something like this:
Here is my code in index.js, which I enact with node index.js on the console:
var parseString = require('xml2js').parseString;
var http = require('http');
var https = require('https');
var mysql = require('mysql');
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "password",
database: "mydb"
});
function xmlToJson(url, callback) {
var req = https.get(url, function(res) {
var xml = '';
res.on('data', function(chunk) {
xml += chunk;
});
res.on('error', function(e) {
callback(e, null);
});
res.on('timeout', function(e) {
callback(e, null);
});
res.on('end', function() {
parseString(xml, function(err, result) {
callback(null, result);
});
});
});
}
var url = "https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml"
xmlToJson(url, function(err, data) {
if (err) {
return console.err(err);
}
strungout = JSON.stringify(data, null, 1);
console.log(strungout);
//strungout contains my json string
})
con.connect(function(err) {
if (err) throw err;
//below is where I might make an insert statement to insert my values into a mysql table
var sql = someinsertstatement
con.query(sql, function (err, result) {
if (err) throw err;
console.log("records inserted");
res.end();
});
});
As mentioned, when I run the above code in my console, the console returns the JSON, though I am unsure how to assign this to a variable that I can then write into my mysql database.
Alternatively, if there is an easier way to write xml from a website directly to my mysql database, I would certainly appreciate any pointers. I feel like it should be easier than this, but I am new to pretty much all of it.
EDIT:
Adding the JSON. I removed the line breaks to consolidate it. Trying to assign the result '4.68' to a variable.
data = {"site": {"observed": [{"datum": [{"valid": [{"_": "2019-02-21T19:42:00-00:00","$": {"timezone": "UTC"}}],"primary": [{"_": "4.68","$": {"name": "Stage","units": "ft"}}]}]}]}};
Thank you.
This worked on my end. Found that the main data you seek is site.observed.datum
const parser = require('xml2json');
const request = require("request");
var mysql = require('mysql');
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "password",
database: "mydb"
});
var api_url = 'https://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=deld1&output=xml';
function xmlToJson(url, callback){
return request({
method: 'GET',
url: api_url,
}, function (error, response, body) {
if (error) {
return callback({
errorResponse: error,
rowsToInsert: false
});
}else{
let jsonRes = JSON.parse(parser.toJson(body));
let datumResult = jsonRes.site.observed.datum;//I had to log Object.keys multple time to get the
const readyForDB = datumResult.map(x => {
let timeOfReading = x.valid.$t;
let stage = x.primary.$t;
let flow = x.secondary.$t;
return [
timeOfReading, stage, flow
]
});
return callback({
errorResponse: false,
rowsToInsert: readyForDB
});
}
})
}
return xmlToJson(api_url, ({errorResponse, rowsToInsert}) => {
if(errorResponse){
throw callback.errorResponse;
}
return con.connect(function(err) {
if (err) throw err;
//below is where I might make an insert statement to insert my values into a mysql table
var sql = "INSERT INTO forecast (timeOfReading, stage, flow) VALUES ?"
con.query(sql, [rowsToInsert], function (err, result) {
if (err) throw err;
console.log(result.affectedRows + " rows inserted");
});
});
});
Sounds like you have the JSON you want but are unsure how to access data within it. Correct me if I'm wrong.
Lets say you have this JSON object called "test":
{
a:1
b:{
x:2
}
}
You can access the value of 1 by calling test.a, and similarly access the value of 2 by calling test.b.x

NodeJs - How to share MySQL pool accross my models to avoid 'ER_CON_COUNT_ERROR'

I'm currently testing my node app using ApacheBench. I run into an issue with my database which is ER_CON_COUNT_ERROR: Too many connections.
I'm using a short library on the top of MySQL node module that you can see just below
var mysql = require('mysql');
var config = require('path/to/config');
var message = require('./myMessageLib.js');
var pool = mysql.createPool({
connectionLimit : 100,
host: config.db.mysql.host,
user: config.db.mysql.user,
password: config.db.mysql.password,
database: config.db.mysql.database
});
var query = function(query_str, values, next) {
pool.getConnection((err, connection) => {
if (err) {
console.error("MySQL Fail to get a connection in pool : " + err);
if (typeof connection !== "undefined")
connection.release();
next(error, null);
return ;
}
connection.query(query_str, values, function(error, data, fields) {
connection.release();
if (error)
if (config.app.env.dev)
throw (error);
else {
next(error, null);
return (message.error("MySQL query failed : " + query_str + " / err : " + error));
}
if (data.length == 0)
next(null);
else
next(data);
})
})
}
exports.query = query;
I use this library in my model by doing something like this
var mysql = require('path/to/mysqllib');
/**
* Class PlayerModel
*/
function PlayerModel() { };
PlayerModel.prototype.get = function(id, next) {
mysql.query("SELECT ....", [id], function(player) {
// stuff
})
}
module.exports = PlayerModel;
The things is on my homepage I use different models like the one presented above and each one launch a query to get some database information. When I launch an ApacheBench with only 50 concurrency levels I got the ER_CON_COUNT_ERROR: Too many connections. So I've got the feeling that the pool isn't well made because it seems that it didn't respect the connections limit of 100 written in the short MySQL lib.
I was thinking about creating and storing the pool in the global nodejs variable to be able to share it correctly accros my modules but I'm not sure it's a good way and maybe also I'm doing something wrong on my pool implentation.
Do you have any idea or improvements to suggest ?
Thanks mates!
I figured out the issue.
My app was deploying in cluster mode. Two process were running at the same time. Because of that, two pools of 100 connections could have been created which is resulting on a total of 200 connections which is higher than the MySQL default connection limit.
Great that found a solution and here's another one with less code.
create a js file, dbconnection.js for example
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit: 10,
host: '...',
user: '...',
password: '...',
database: '...',
dateStrings: true
});
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
In the other file where you want to use the connection
var db = require('./dbconnection.js');
And instead of
connection.query
Use
db.connection.query

NodeJs Store Json from multiple requests to MySQL DB

I am working on a crawler that stores the retrieved and parsed data into my MySQL DB.
Storing the results works pretty well, but I am stuck when it comes to end the connection.
I am using request, cheerio, async and mysql:
// DB params
var con = mysql.createConnection({
host: "localhost",
user: "user",
password: "password",
database: "mydatabase",
});
// Open connection to DB
con.connect(function(err) {
if (err) {
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
// Array of Cities to crawl
var cities = ["olten", "zurich"];
// Todays Date
var today = new Date();
...
today = dd + '.' + mm + '.' + yyyy;
// Search every city
async.each(cities, function(item, callback){
// The Page to crawl
var page = "https://www.thepageto.com/search_xhr?fn=" + /*cities[i]*/ item + "&tn=&db=" + today + "&sort=trip_date&order";
console.log("Visiting page " + page);
request(page, function(error, response, body) {
if (error) {console.log("Error: " + error);}
// Check status code (200 is HTTP OK)
console.log("Status code: " + response.statusCode);
if (response.statusCode === 200) {
// Parse JSON
var data = JSON.parse(body);
// Use only result > html code
var content = data.html.results;
// Parse html
var $ = cheerio.load(content);
// Iterate through offers & Extract information & Store in a Object
$('.trip.relative').each(function() {
// Empty Object, has the same structure as the Db table
var json = {id: "", driver: "", rating: "", date: "", time: "", start: "", stops: "", end: "", price: "", url: "", query: ""};
// Assigning values
json.driver = $(this).find('.username').text();
...
// Save filled object in Db
save(json, callback);
});
}
});
}, function(err){
if (err) {
console.log('could not save');
} else {
console.log('lets end connection');
con.end(function(err) {if (err) throw err;});
}
}
);
function save(item, callback) {
con.query('INSERT INTO offers SET ?', item, callback);
}
The problem is, that the DB connection is ended before anything could be stored into the DB. The con.end() is always fired before the actual query or before the query completed. I have the intension that the save function has to return something when its done. If i run the script without the con.end(), the data is stored into the DB perfectly.
I am new to Node so I am still learning and hopefully you can help me with this.
Edit:
I now get an:
ReferenceError: callback is not defined
at save (/home/juki/crawler/crawler.js:143:46)
As well as an:
Error: Callback was already called.
at Query._callback (/home/juki/crawler/node_modules/async/dist/async.js:839: 36)
It looks like you do not call callback from your crawler.
You also do not seem to wait for save to finish. Try to rework save function, so it takes callback too. Something like:
function save(item, callback) {
con.query('INSERT INTO offers SET ?', item, callback);
}
And then call it like:
save(json, callback);
where callback is the one you get from async.each.

Create databases connection on startup - sails.js

I know you can use Waterline and the models approach, but the issue is that I have to access more than one database to generate a response. Besides the models approach makes it difficult to think on a solution due to the fact that the data is joined and infered from different tables under the different databases.
Therefore, I would like to know how can I open a DB connection using the mysql or mysql2 native driver and reuse it all over the app. Where is the most suitable place, a hook, etc.? How can I close them when the app goes down?
TA
A custom hook will be the best for you. For Redis, I created one like this:
api/hooks/redis/index.js
var redisModule = require('redis');
module.exports = function connectToRedis(sails) {
return {
connectToRedis: function (callback) {
var hook = this;
var config = sails.config.connections.redis;
var redisClient = redisModule.createClient(config.port, config.host);
hook.initAdapters();
redisClient.on('connect', function () {
sails.log.verbose('Connection to redis was succesfull!');
// select db
redisClient.select(config.db, function (err) {
if (err) {
return callback(err);
}
sails.adapters.redis = redisClient;
callback();
});
});
redisClient.on('error', function (error) {
sails.log.error(error);
callback();
});
},
initAdapters: function () {
if (sails.adapters === undefined) {
sails.adapters = {};
}
},
// Run automatically when the hook initializes
initialize: function (cb) {
var hook = this;
hook.connectToRedis(function() {
cb();
});
},
};
};

How to make multiple mysql queries in Node with promises

G'day all,
I'm trying to convert some old php code over to Node, and part of the journey has been trying to figure out the best way to perform sql queries against my database (I'm using SQL so I can port the existing database over).
I've got them working, but have encountered the "Pyramid of Doom" problem, and it's subsequent scope issues (i.e. the returned values not baing available to subsequent "then"s).
An example of the sort of code I have here is: (dbPool.queryOPromise returns a query wrapped in a promise)
dbPool.queryOPromise(query)
.then(function(result){
console.log(result);
var query = {
sql:"INSERT INTO newusers (newuserid, ipaddress, email) VALUES (?,?,?)",
values: [newuserid, ipAddress, email]
};
dbPool.queryOPromise(query)
.then(function(value){
console.log(value);
if(value.code==200) {
res.status(200).json({code:200, status:"New User Created"});
} else {
res.status(400).json({code:value.code, status:"Error creating new user: ".value.status});
}
})
})
Does anyone have a view on the best way to attack this situation?
Thanks!
You're supposed to return the subsequent promises instead of calling .then on them
dbPool.queryOPromise(query)
.then(function(result) {
console.log(result);
var query = {
sql: "INSERT INTO newusers (newuserid, ipaddress, email) VALUES (?,?,?)",
values: [newuserid, ipAddress, email]
};
// RETURN the second promise,
return dbPool.queryOPromise(query);
})
.then(function(value) {
console.log(value);
if (value.code == 200) {
res.status(200).json({code: 200, status: "New User Created"});
} else {
res.status(400).json({code: value.code, status: "Error creating new user: ".value.status });
}
})
.catch(console.error); // and always catch the errors at the end.
It's a #1 rookie mistake in using promises. Checkout this wonderfully written article addressing issues exactly like this
For node -v > 8.x only,
Share my working example:
I use this Promisified MySQL middleware for Node.js
read this article Create a MySQL Database Middleware with Node.js 8 and Async/Await
here is my database.js
var mysql = require('mysql');
// node -v must > 8.x
var util = require('util');
// !!!!! for node version < 8.x only !!!!!
// npm install util.promisify
//require('util.promisify').shim();
// -v < 8.x has problem with async await so upgrade -v to v9.6.1 for this to work.
// connection pool https://github.com/mysqljs/mysql [1]
var pool = mysql.createPool({
connectionLimit : process.env.mysql_connection_pool_Limit, // default:10
host : process.env.mysql_host,
user : process.env.mysql_user,
password : process.env.mysql_password,
database : process.env.mysql_database
})
// Ping database to check for common exception errors.
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
// Promisify for Node.js async/await.
pool.query = util.promisify(pool.query)
module.exports = pool
You must upgrade node -v > 8.x
you must use async function to be able to use await.
example:
var pool = require('./database')
// node -v must > 8.x, --> async / await
router.get('/:template', async function(req, res, next)
{
...
try {
var _sql_rest_url = 'SELECT * FROM arcgis_viewer.rest_url WHERE id='+ _url_id;
var rows = await pool.query(_sql_rest_url)
_url = rows[0].rest_url // first record, property name is 'rest_url'
if (_center_lat == null) {_center_lat = rows[0].center_lat }
if (_center_long == null) {_center_long= rows[0].center_long }
if (_center_zoom == null) {_center_zoom= rows[0].center_zoom }
_place = rows[0].place
} catch(err) {
throw new Error(err)
}