The Task
I am trying read all the records from one table, and insert them into another table. I will re-use these functions later on in the app.
The Problem
The data doesn't appear to be being passed correctly. I am not getting any errors, yet the records are not inserted.
I debugged the data variable, and all the records are consoled correctly. For some reason, I am losing them when I call my insertData function.
Note: My table name is results, which may be a little confusing. I may change that in the future.
Entire Source Code
var mysql = require('mysql');
var connection = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: "marketplace"
});
connection.connect(function (err) {
if (err) throw err;
console.log("Connected!");
})
const getData = function (tblName) {
var data = {};
switch (tblName) {
case "results":
sql = "SELECT * FROM results";
break;
case "items":
sql = "SELECT * FROM items";
break;
default:
sql = "SELECT * FROM results";
break;
}
return new Promise(resolve => {
connection.query(sql, function (err, results) {
if (err) {
console.log(err);
} else {
data.numResults = results.length;
data.data = results;
resolve(data);
}
})
})
}
const insertData = function (tbl, entity) {
return new Promise(resolve => {
var sql = `INSERT INTO ${tbl} (title,price,location,miles,imgUrl,itemURL) VALUES ?`;
var insertedIds = [];
for (var i = 0; i < entity.length; i++) {
connection.query(sql, entity[i], function (err, res) {
if (err) throw err;
insertedIds.push(res.insertId);
});
}
resolve(insertedIds);
})
}
const init = async function () {
var data = await getData("items");
console.log(data); // Works. Display all data
var insertSuccess = await insertData("results", data);
}
init();
Data Structure
{
id: 251,
title: '2008 Jeep Wrangler Unlimited Sahara Sport Utility 4D',
price: '$10,500',
location: 'Lake Sarasota, Florida',
miles: '123K miles',
itemURL: '/marketplace/item/174406817245706/',
imgUrl: 'https://scontent-mia3-1.xx.fbcdn.net/v/t1.0-0/c43.0.260.260a/p261x260/98434536_3577078698974198_8432375958719168512_n.jpg?_nc_cat=111&_nc_sid=843cd7&_nc_oc=AQlLn3CVPZmD4dKSsfXd-rV0WxXBo98zneuGAEgz2JP2yWrkt5rxI-fa1ShTtMGYbrw&_nc_ht=scontent-mia3-1.xx&oh=89e552882baeb14c642a1cd28b8ba683&oe=5EEA3991',
seen: 0,
created_date: 2020-05-20T20:45:24.000Z
}
Schema
DROP TABLE IF EXISTS `results`;
CREATE TABLE IF NOT EXISTS `results` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`title` varchar(255) DEFAULT NULL,
`price` varchar(255) DEFAULT NULL,
`location` varchar(255) DEFAULT NULL,
`miles` varchar(22) DEFAULT 'unavailable',
`itemURL` text,
`imgUrl` text,
`created_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=612 DEFAULT CHARSET=latin1;
COMMIT;
Output
When I try to console.log the insertSuccess variable, which should show me a list of inserted id's:
.
.
.
console.log(insertSuccess);
Related
I've decided to start using mysql2 library lately and created the following example to illustrate what currently works.
A MySQL tempt table is created, a series of queries are run in parallel and then it terminates.
I then found this article MySQL Limits on Digital Ocean.
Specifically the bullet point referring to:
Connection pooling is not supported for MySQL.
My question is, should I use mysql.createConnection() instead of mysql.createPool()? I ask because, both of them work just fine.
// deno run --allow-env --allow-net todo.ts
import * as mysql2 from "https://deno.land/x/mysql2/mod.ts";
const pool = mysql2.createPool({
host: "db.ondigitalocean.com",
port: 12345,
user: "user1",
password: "letmein",
database: "devdb",
connectionLimit: 10,
timezone: "+00:00" // Development
// timezone: "-06:00" // Production
});
/* CREATE temp table tblTodos in database memory */
const sqlCREATE1: string =
`CREATE TEMPORARY TABLE IF NOT EXISTS tblTodos (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
userId BIGINT UNSIGNED DEFAULT NULL,
CHECK (userId >= 0),
todoCreated TIMESTAMP DEFAULT NULL,
todoModified TIMESTAMP DEFAULT NULL,
todoStatus VARCHAR(16) DEFAULT NULL,
todoTitle VARCHAR(128) DEFAULT NULL,
todoBody VARCHAR(1024) DEFAULT NULL,
PRIMARY KEY (id)
) ENGINE = INNODB
AUTO_INCREMENT=2001
DEFAULT CHARSET = utf8mb4
COLLATE = utf8mb4_0900_ai_ci;`;
/* SELECT 1 shows an empty table */
const sqlSELECT1: string =
`SELECT
*
FROM
tblTodos;`;
/* INSERT 1 adds a Pending todo record to the table */
const sqlINSERT1: string =
`INSERT INTO tblTodos (
id, userId,
todoCreated, todoModified, todoStatus,
todoTitle, todoBody
)
VALUES
(
NULL, 1001,
NOW(), NOW(), 'Pending',
'Shopping List #1', 'Apples & Oranges'
);`;
/* SELECT 2 shows the Pending todo record */
const sqlSELECT2: string =
`SELECT
*
FROM
tblTodos
WHERE
todoStatus = 'Pending';`;
/* UPDATE 1 changes todo status from Pending to Complete */
const sqlUPDATE1: string =
`UPDATE
tblTodos
SET
todoModified = NOW(),
todoStatus = 'Complete'
WHERE
id = 2001
AND userId = 1001;`;
/* SELECT 3 shows the Complete todo record */
const sqlSELECT3: string =
`SELECT
*
FROM
tblTodos
WHERE
todoStatus = 'Complete'
AND userId = 1001;`;
/* DELETE 1 deletes todo from table */
const sqlDELETE1: string =
`DELETE FROM
tblTodos
WHERE
id = 2001
AND userId = 1001;`;
/* SELECT 4 once again shows an empty table */
const sqlSELECT4: string =
`SELECT
*
FROM
tblTodos;`;
/* DROP 1 deletes table tblTodos from database */
const sqlDROP1: string =
`DROP
TEMPORARY TABLE IF EXISTS tblTodos;`;
const connection = await pool.getConnection();
let create1Result, select1Result, insert1Result, select2Result,
update1Result, select3Result, delete1Result, select4Result,
drop1Result;
try {
create1Result = await connection.execute(sqlCREATE1);
if (create1Result) console.log("Table tblToDos created.");
select1Result = await connection.execute(sqlSELECT1);
if (select1Result) console.log("Table tblToDos contains", select1Result[0].length, "records.");
insert1Result = await connection.execute(sqlINSERT1);
if (insert1Result) console.log(insert1Result[0].affectedRows, "record(s) inserted.", "id:", insert1Result[0].insertId);
select2Result = await connection.execute(sqlSELECT2);
if (select2Result) console.log(select2Result[0]);
update1Result = await connection.execute(sqlUPDATE1);
if (update1Result) console.log(update1Result[0].affectedRows, "record(s) updated.");
select3Result = await connection.execute(sqlSELECT3);
if (select3Result) console.log(select3Result[0]);
delete1Result = await connection.execute(sqlDELETE1);
if (delete1Result) console.log(delete1Result[0].affectedRows, "record(s) deleted.");
select4Result = await connection.execute(sqlSELECT4);
if (select4Result) console.log("Table tblToDos contains", select1Result[0].length, "records.");
drop1Result = await connection.execute(sqlDROP1);
if (drop1Result) console.log("Table tblToDos droped.");
} catch(error) {
// Complete MySQL error message.
// console.log(`%cError: ${error.message}`, "color: #e53935");
// Build a smaller MySQL error message.
const errorPart1 = error.message.split(";")[0];
const errorPart2 = error.message.split(" use ")[1];
console.log(`%cError: ${errorPart1} ${errorPart2}`, "color: #e53935");
} finally {
connection.release();
}
// Debug created datetime
console.log(select3Result[0][0].todoCreated.toLocaleString());
// End pool when web server shuts down.
await pool.end();
I have three tables with data schema, like:
TABLE user (
user_id BINARY(16) PRIMARY KEY NOT NULL,
created DATETIME NOT NULL,
last_updated DATETIME,
coordinator BINARY(16),
num_updates INT NOT NULL
);
TABLE summary (
user_id BINARY(16) PRIMARY KEY NOT NULL,
calculation_time DATETIME NOT NULL,
calculation_method VARCHAR(25) NOT NULL,
label VARCHAR(50) NOT NULL,
critical_count INT NOT NULL,
median_risk FLOAT(10)
);
TABLE actions(
user_id BINARY(16) PRIMARY KEY NOT NULL,
label VARCHAR(50) NOT NULL,
access_count INT NOT NULL,
median FLOAT(10)
);
The data for all the users (user table) is simply fetched using the lambda handler function in the following manner:
const AWS = require('aws-sdk');
const rdsDataService = new AWS.RDSDataService();
module.exports.hello = async (event, context, callback) => {
const req_id = "5a9dbfca-74d6-471a-af27-31beb4b53bb2";
const sql = 'SELECT * FROM user WHERE user_id=:id';
try {
const params = {
resourceArn: 'arn:aws:rds:us-west-********************',
secretArn: 'arn:aws:secretsmanager:us-west**************',
sql,
database: 'dev_db1',
continueAfterTimeout: true,
includeResultMetadata: true,
parameters: [{ 'name': 'id', 'value': { 'stringValue': `${req_id}` } }]
}
const db_res = await rdsDataService.executeStatement(params).promise();
const convertToJson = (dbresponse) => {
const columns = dbresponse.columnMetadata.map(col => col.name);
const row_data = dbresponse.records.map(row => {
const json_obj = {};
row.map((val, i) => {
json_obj[columns[i]] = Object.values(val)[0];
});
return json_obj;
});
return row_data;
};
const modified_data = convertToJson(db_res);
const response = {
body: {
statusCode: 200,
message: 'Data fetched successfully',
data: modified_data,
}
};
callback(null, response);
} catch (error) {
console.log('Error Received', error);
const error_res = {
body: {
statusCode: error.statusCode,
message: error.message,
data: null
}
}
callback(null, error_res);
}
};
If the same is followed for another table summary or actions, it also works. Now, I need to combine all the columns of these three tables and then return the data (returned rows should match on the basis of req_id).
My working snippet: https://dbfiddle.uk/?rdbms=mysql_5.7&fiddle=016ecc94c792611fbaca810605e81a6a
But the final result obtained contains the column user_id in duplicated form i.e. three times inclusion. I don't need the same column to be repeated thrice.
I am a bit new to handling MySQL queries, so unable to figure out the exact reason for the error even when the table exists. The MYSQL version used in Aurora is 5.7.
Any help to resolve the same is appreciated!
Plan A: Explicitly specify the columns you want. Extra benefit: You can get rid of the ids, which tend to be useless to others reading the output.
Plan B: (This option is not always possible.) Instead of JOIN .. ON t1.a = t2.a, say JOIN .. USING(a)
I like to use short aliases. Here's doing all things together:
SELECT u.last_name, u.first_name,
s.risk_score,
t.likes
FROM user AS u
JOIN summary AS s USING(user_id)
LEFT JOIN test AS t USING(user_id)
In general, it is not wise to have a 1:1 relationship (as you have via user_id); you may as well have all the columns in a single table.
try this
SELECT users.*, summary.* from users, summary WHERE users.user_id = summary.user_id
OR
SELECT * from users, summary WHERE users.user_id = summary.user_id
I want to insert into my table values which I get from my front.
so. I have
const Workers = function (workers) {
this.id = workers.id,
this.workers = workers.workers,
this.room = workers.room,
this.team = workers.team,
this.city = workers.city,
this.hotel_name = workers.hotel_name,
this.address = workers.address
};
Workers.create = (newWorkers, result) => {
sql.query(`INSERT INTO rooms_split (workers, room, hotel_name, address, createdAt, updatedAt) VALUES( ? , ? , ? , ?, DEFAULT, DEFAULT )`,
[newWorkers.workers, newWorkers.room, newWorkers.hotel_name, newWorkers.address], (err, res) => {
if (err) {
console.log("error: ", err);
result(err, null);
return;
}
console.log("created splitted room: ", {
id: res.insertId,
...newWorkers
});
result(null, {
id: res.insertId,
...newWorkers
});
});
};
And there is my controller
exports.create = (req, res) => {
console.log("body " + JSON.stringify(req.body));
if (!req.body) {
res.status(400).send({
message: "Content can not be empty!"
});
}
const workers = new Workers({
workers: req.body.workers,
room: req.body.room,
hotel_name: req.body.hotel_name,
address: req.body.address
});
Workers.create(workers, (err, data) => {
if (err)
res.status(500).send({
message: err.message || "Some error occurred while creating the Alias."
});
else res.send(data);
});
}
Output from
console.log("body " + JSON.stringify(req.body));
is
body {"workers":["John Snow","Juri Boyka"],"room":"45","hotel_name":"Test Hamburg","address":"Hamburg 5, test Strase"}
and it looks fine but when is time to insert it into table I got error
code: 'ER_PARSE_ERROR',
errno: 1064,
sqlMessage: "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ''Juri Boyka', `room` = '45', `team` = NULL, `city` = NULL, `hotel_na' at line 1",
sqlState: '42000',
index: 0,
sql: "INSERT INTO rooms_split SET `id` = NULL, `workers` = 'John Snow', 'Juri Boyka', `room` = '45', `team` = NULL, `city` = NULL, `hotel_name` = 'Test Hamburg', `address` = 'Hamburg 5, test Strase'"
}
I kniw what this error means but I have no idea why when I want to make query nodejs(?) separates my value so instead ['something','something2'] I got 'something', 'something2' and he is right that there are not enough columns
Change
[newWorkers.workers, newWorkers.room, newWorkers.hotel_name, newWorkers.address]
to
[newWorkers.workers.join(), newWorkers.room, newWorkers.hotel_name, newWorkers.address]
since the workers data type in MySQL is varchar. Therefore, you'll need to stringify your incoming workers array
Firstly, if anyone can edit my question title or question to make more sense, please do.
I have a node/express app making mysql queries with mysql.js. I have a query that looks up a table of questions and then runs a map function on the results. Within that map function, I need to query another table, of answers, corresponding to each record in the questions table. The value I need is the number of answers to that question, ie the number of records in each answers table. I've tried all kinds of different examples, but nothing quite fits my case in a way that makes sense to me. New at Node and Express, and even MySQL so having a hard time picking out quite what to.
I understand that the problem is the async nature of node. getAnswersCount() returns "count" before the query finishes. Below is my code. Need some advice on how to achieve this.
The value 123 is assigned to count just to clarify the trace results.
app.get('/', (req, res) => {
db.query('SELECT * FROM questions LIMIT 0, 100',
(error, results) => {
if (error) throw error;
questions = results.map(q => ({
id: q.id,
title: q.title,
description: q.description,
answers: getAnswersCount( q.id )
}));
res.send( questions );
});
});
const getAnswersCount = ( id ) =>
{
const tableName = 'answers_' + id;
var count = 123;
var sql = `CREATE TABLE IF NOT EXISTS ${tableName}(
id int primary key not null,
answer varchar(250) not null
)`;
db.query( sql,
(error, results) => {
if (error) throw error;
//console.log( 'answers table created!' );
});
sql = `SELECT COUNT(*) AS answersCount FROM ${tableName}`;
db.query( sql,
(error, results) => {
if (error) throw error;
//console.log( count ); // will=123
count = results[0].answersCount;
//console.log( count ); // will = results[0].answerCount
});
// I know this code runs before the query finishes, so what to do?
//console.log( count ); //still 123 instead of results[0].answersCount
return count;
}
EDIT: After attempting various versions of Michael Platt's suggestion in his answer without success, I finally worked out a solution using Express callbacks and a promise, adding the answers values to the questions array afterwards:
app.get( '/', (req, res, next ) => {
db.query('SELECT * FROM questions LIMIT 0, 100',
(error, results) => {
if (error) throw error;
questions = results.map(q => ({
id: q.id,
title: q.title,
description: q.description,
}));
next();
});
}, (req, res ) => {
questions.map( currentElem => {
getAnswersCount( currentElem.id ).then( rowData => {
currentElem.answers = rowData[0].answersCount;
if( currentElem.id == questions.length ) res.send( questions );
});
});
});
const getAnswersCount = ( id ) => {
const tableName = 'answers_' + id;
var sql = `CREATE TABLE IF NOT EXISTS ${tableName}(
id int primary key not null,
answer varchar(250) not null
)`;
db.query( sql,
(error, results) => {
if (error) throw error;
//console.log( 'answers table created!' );
});
sql = `SELECT COUNT(*) AS answersCount FROM ${tableName}`;
return new Promise( ( resolve, reject ) => {
db.query( sql, ( error, results ) => {
if ( error ) return reject( err );
resolve( results );
});
});
}
I'm not sure which database module you are using to connect to and query the database but you could make the method async and then await the response from the query like so:
const getAnswersCount = async ( id ) =>
{
const tableName = 'answers_' + id;
var count = 123;
var sql = `CREATE TABLE IF NOT EXISTS ${tableName}(
id int primary key not null,
answer varchar(250) not null
)`;
var results = await db.query(sql);
sql = `SELECT COUNT(*) AS answersCount FROM ${tableName}`;
var count = db.query(sql)[0].answerCount;
// I know this code runs before the query finishes, so what to do?
//console.log( count ); //still 123 instead of results[0].answersCount
return count;
}
app.get('/', async (req, res) => {
db.query('SELECT * FROM questions LIMIT 0, 100',
(error, results) => {
if (error) throw error;
questions = results.map(q => {
const answerCount = await getAnswersCount( q.id )
return {
id: q.id,
title: q.title,
description: q.description,
answers: answerCount
}
}));
res.send( questions );
});
});
I think that will give you what you want and run correctly but it might require a bit of tweaking. You may need to async the function on the actual route itself as well and await the call for getAnswersCount but that should just about do it.
I'm going to check id of my data if exist in mysql update it and if not create it, but in select result it dosen't work and go to esle section and create action most of the times and got Duplicate entry error. at the end I run this file by infinite loop.
I should tell this when I have a lot data in redis this problem will happen.
I'll be glad if help me, thanks :)
Here is my code :
var
mysql = require('mysql'),
redis = require('redis'),
infiniteLoop = require('infinite-loop'),
client = redis.createClient(),
lop = new infiniteLoop,
config = require( __dirname + '/app/Config').config,
con = function() {
return mysql.createConnection({
host : config.db_host,
user : config.db_user,
password : config.db_pass,
database : config.db_name
});
},
insertDB = function( data ) {
var connection = con();
connection.connect( function( err ) {
connection.query( 'SELECT * FROM real_user WHERE id =' + data.id, function( err, res ) {
if ( err ) {
data = JSON.stringify(data);
client.lpush('aipi', data, function(err,reply) {
if ( err ) throw err;
});
}
console.log( "out", data.id, res.length, res );
if ( typeof res != 'undefined' && res.length > 0 ) {
console.log( "update", data.id );
connection.query( 'UPDATE real_user SET ? WHERE id =' + res[0].id, { request_count: data.request_count, updated_at: data.created_at },
function( err, res ) {
if ( err ) throw err;
});
} else {
console.log( "create", data.id );
connection.query( 'INSERT INTO real_user SET ?', data, function( err, res ) {
if ( err ) throw err;
});
}
connection.end();
});
});
},
run = function() {
var data;
client.brpop('aipi', 0, function(err, reply) {
if ( reply[1] ) {
data = JSON.parse(reply[1]);
insertDB(data);
}
});
};
lop.add(run, []).run();
Fixed it by :
ON DUPLICATE KEY UPDATE
connection.connect( function( err ) {
connection.query( "INSERT INTO real_user SET ? ON DUPLICATE KEY UPDATE updated_at = '" + data.created_at
+ "', request_count = '" + data.request_count + "'",
data, function( err, res ) {
connection.end();
});
});