Unexpected Error: write callback called multiple times executing Gulp.js tasks inside a loop - gulp

I'm trying to execute some gulp tasks inside a map loop to generate css files for each sass theme I defined.
Finally, I return all merged tasks with merge-stream package.
Unfortunately, callback seems sent for each iteration / gulp task of my loop and i'm getting the following message :
Error: write callback called multiple times
#Task('build')
build() {
var self = this;
var tasks = this.themes.map((theme) => {
this.foldersXml = getAttributes('jntFolderWithExternalProvider');
this.filesXml = getAttributes('jntFolder');
this.foldersXml[theme] = []; this.foldersXml[theme].push(getAttributes('jntFolderWithExternalProvider'));
this.filesXml[theme] = []; this.filesXml[theme].push(getAttributes('jntFolderWithExternalProvider'));
fs.readFile(this.themesFolder + '/' + theme + '/' + this.fileName, 'utf8', (err: Error, data: string & Buffer) => {
if (err) {
throw new gutil.PluginError({
plugin: 'build',
message: 'Main file doesn\'t exist for the theme: ' + theme
});
} else {
var vars = data.match(/\$(.*?)\:/g);
this.requiredVars.map(requiredVar => {
if(vars !== null){
if(!vars.contains(requiredVar)){
throw new gutil.PluginError({
plugin: 'build',
message: 'Required variable ' + requiredVar + ' is not defined'
});
};
}
});
}
});
return gulp.src(this.templatesFolder + '/' + this.pattern)
.pipe(header('#import \'' + this.themesFolder + '/' + theme + '/' + this.fileName + '\';'))
.pipe(sass.sync().on('error', gutil.log))
.pipe(rename(function (path: any) {
var file = path.basename + path.extname;
var folderXml = getAttributes('jntFolderWithExternalProvider') as any;
folderXml[file] = [ getAttributes('jntCssFile') ];
self.foldersXml[theme][0][file] = []; self.foldersXml[theme][0][file].push(folderXml);
var fileXml = getAttributes('jntFolderWithExternalProvider') as any;
fileXml['jcr:content'] = [ getAttributes('jntRessource') ];
self.filesXml[theme][0][file] = []; self.filesXml[theme][0][file].push(fileXml);
path.dirname += '/' + file;
}))
.pipe(gulp.dest(this.themesFolder + '/' + theme))
});
return merge(tasks);
}
#SequenceTask()
run(){
return ['build'];
}
I just want to get rid of this message and be sure that callback is invoked only at the end. What's the best approach for you ?

Related

Extract data from JSON within Lambda so it's not undefined

Looking for advice / second opinion. I'm trying to pass JSON via HTTP API (api gateway) > Lambda. I'm receiving the data (pic of Cloudwatch), getting undefined when trying to extract values. The file is being written to S3, but undefined.
I included Lambda code, picture of Cloudwatch logs. I'm about there :) . Newbie here...
Logs
Lambda Code
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
var bucketName = process.env.bucketName;
var folder = process.env.folder;
var filename = getFileName();
console.log("Filename:" + filename);
var raw = JSON.stringify(event.body);
console.log("raw after stringify:" + raw);
var results = JSON.parse(raw);
console.log("results:" + results);
let firstname = results.firstName;
console.log("firstName:" + firstname);
let lastname = results.lastName;
console.log("lastName:" + lastname);
let message = results.Message;
console.log("Message:" + message);
var content = message + "," + firstname + "," + lastname;
console.log("content:" + content);
var keyName = getKeyName(folder, filename);
var params = { Bucket: bucketName, Key: keyName, Body: content };
s3.putObject(params, function (err, data) {
if (err)
console.log(err)
else
console.log("Successfully saved object to: " + bucketName + "/" + keyName);
});
function getKeyName(folder, filename) {
return folder + '/' + filename;
}
function getFileName() {
var _uuid = uuidv4();
var _date = Date.now();
return _uuid + "-" + _date;
}
function uuidv4() {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}
var html = '<html><head><title>Prayer received result</title></head>' +
'<body><h1>Your Prayer has been received!</h1></body></html>';
//callback(null, res); - use this when using proxy model
callback(null, html);
};
Made the following changes.
//var raw = JSON.stringify(event.body);
//console.log("raw after stringify:" + raw);
var results = JSON.parse(event.body);
console.log("results:" + results);
Hope this helps others. I'm newer as of this post to Lambda, JSON.

Error: File not found with singular glob gulp 4

I have problems with build with gulp 4.
And I get this error:
Error: File not found with singular glob: /app/build/js/all.min.js (if this was purposeful, use `allowEmpty` option)
at Glob.<anonymous> (/app/node_modules/glob-stream/readable.js:84:17)
at Object.onceWrapper (events.js:315:30)
at emitOne (events.js:116:13)
at Glob.emit (events.js:211:7)
at Glob._finish (app/node_modules/glob/glob.js:197:8)
at done (app/node_modules/glob/glob.js:182:14)
at Glob._processSimple2 (app/node_modules/glob/glob.js:688:12)
at app/node_modules/glob/glob.js:676:10
at Glob._stat2 (app/node_modules/glob/glob.js:772:12)
at lstatcb_ (app/node_modules/glob/glob.js:764:12)
at RES (/app/node_modules/inflight/inflight.js:31:16)
at f (app/node_modules/once/once.js:25:25)
at FSReqWrap.oncomplete (fs.js:152:21)
Here is my gulpfile.js
var gulp = require('gulp');
var environments = require('gulp-environments');
var htmlReplace = require('gulp-html-replace');
var preprocess = require('gulp-preprocess');
var replace = require('gulp-replace');
var concat = require('gulp-concat');
var terser = require('gulp-terser');
var filelist = require('gulp-filelist');
var addsrc = require('gulp-add-src');
var rename = require('gulp-rename');
var gulpSequence = require('gulp-sequence');
var textTransformation = require('gulp-text-simple');
var less = require('gulp-less');
var reworkUrls = require('gulp-rework-urls');
var cleanCss = require('gulp-clean-css');
var gulpif = require('gulp-if');
var hash = require('gulp-hash-filename');
var log = require('fancy-log');
var colors = require('ansi-colors');
var PluginError = require('plugin-error');
var del = require('del');
var argv = require('yargs').argv;
var _ = require('lodash');
var file = require('gulp-file');
var shell = require('gulp-shell');
var fs = require('fs');
var app = argv.app || 'main';
var vendorFree = /^true$/i.test(process.env.VENDOR_FREE) || false;
console.info('Is vendor free: [' + vendorFree + ']');
var appendHash = !argv.nohash;
console.info('Appending hash to resource file names: [' + appendHash + ']');
var config = {
targetDir: 'build',
targetJs: 'js/all.min.js',
targetConfig: 'js/config.json',
targetCss: 'css/all.min.css',
targetLessCss: 'css/all.less.css',
targetFonts: 'fonts',
htmlTemplate: app + '/index.template.html',
targetHtml: 'index.html',
fileNameFormat: '{name}.{hash}{ext}'
};
var sources = {};
var development = environments.development;
var production = environments.production;
var env = environments.current().$name;
console.info('Environment: [' + env + ']');
if ([development.$name, production.$name].indexOf(env) === -1) {
console.info('Unsupported Environment [' + env + ']. Building as production...');
environments.current(production);
}
var params = JSON.stringify(_.merge(loadConfig('default'), loadConfig(env), {app: app, vendorFree: vendorFree}));
function targetPath(subPath) {
return config.targetDir + '/' + subPath;
}
function _loadConfig(fileName, dir) {
var filePath = './' + dir + '/config/' + fileName + '.json';
return fs.existsSync(filePath) ? require(filePath) : null;
}
function loadConfig(fileName) {
return _.merge(_loadConfig(fileName, './'), _loadConfig(fileName, app));
}
var onLessCompileError = function (err) {
log.error(colors.red('[ERROR]'), err);
this.emit('end', new PluginError('less', err, {showStack: true}));
if (!argv.skipError) {
process.exit(1);
}
};
gulp.task('js', gulp.series([
production() && 'js:bundle',
function () {
console.log('config.targetJs: ', config.targetJs);
return gulp.src([
'node_modules/core-js/client/shim.min.js',
'node_modules/zone.js/dist/zone.min.js',
'node_modules/hammerjs/hammer.min.js'
])
.pipe(development(addsrc.append('node_modules/systemjs/dist/system.src.js')))
.pipe(production(addsrc.append(targetPath(config.targetJs))))
.pipe(development(addsrc.append('config.js')))
.pipe(production(terser({
mangle: {
keep_fnames: true
},
compress: {
pure_funcs: [
'console.debug', 'console.info', 'console.log'
]
}
})))
.pipe(production(concat(config.targetJs)))
.pipe(production(replace(/^/, "document.addEventListener('DOMContentLoaded', function() {")))
.pipe(production(replace(/$/, '})')))
.pipe(production(gulpif(appendHash, hash({format: config.fileNameFormat}))))
.pipe(production(gulp.dest(config.targetDir)))
.pipe(filelist())
.pipe(textTransformation(function (jsFiles) {
sources.js = eval(jsFiles);
})());
}
].filter(Boolean)));
gulp.task('js:config', function () {
return gulp.src([], {allowEmpty: true})
.pipe(file(config.targetConfig, params))
.pipe(gulp.dest(config.targetDir));
});
gulp.task('js:bundle', shell.task('npm run pack -- --env.appName=' + app + (appendHash ? '' : ' --env.nohash')));
gulp.task('less', function () {
return gulp.src('less/styles.less')
.pipe(less().on('error', onLessCompileError))
.pipe(rename(config.targetLessCss))
.pipe(gulp.dest(config.targetDir));
});
gulp.task('css', gulp.series('less', function() {
return gulp.src([
'node_modules/#angular/material/prebuilt-themes/indigo-pink.css',
'node_modules/open-sans-fontface/open-sans.css',
'node_modules/typeface-rubik/index.css'
]).pipe(production(reworkUrls(function (url) {
return url.replace(/^(.\/)?((images)|(fonts)\/)/, '$1../$2')
.replace(/^.\/files\//, '../fonts/');
})))
.pipe(addsrc.append(targetPath(config.targetLessCss)))
.pipe(production(concat(config.targetCss)))
.pipe(production(cleanCss()))
.pipe(production(gulpif(appendHash, hash({format: config.fileNameFormat}))))
.pipe(production(gulp.dest(config.targetDir)))
.pipe(filelist())
.pipe(textTransformation(function (cssFiles) {
sources.css = eval(cssFiles);
})());
}));
gulp.task('font', function () {
return gulp.src([
'node_modules/open-sans-fontface/fonts/**/*.{eot,svg,ttf,woff,woff2}',
'node_modules/typeface-rubik/files/**/*.{eot,svg,ttf,woff,woff2}'
])
.pipe(gulp.dest(targetPath(config.targetFonts)));
});
gulp.task('clean', function () {
del.sync([config.targetDir, config.targetHtml]);
});
gulp.on('err', function(e) {
console.log(e.err.stack);
});
gulp.task('html', gulp.series(gulp.parallel('js', 'css'), function() {
return gulp.src(config.htmlTemplate)
.pipe(htmlReplace({
css: {
src: sources.css,
tpl: '<link rel="stylesheet" type="text/css" href="%s" />\n'
},
js: {
src: sources.js,
tpl: '\n<script src="%s"></script>'
},
config: {
src: params,
tpl: '<script nonce="<%- nonce %>">window.Config = %s</script>'
},
'config-backend': {
src: params,
tpl: '<script nonce="<%- nonce %>">' +
"window.Config = %s;" +
"window.EnvironmentConfig = JSON.parse('<%- config %>');" +
'</script>'
}
}))
.pipe(preprocess({
extension: 'js'
}))
.pipe(rename(config.targetHtml))
.pipe(gulp.dest(''));
}));
gulp.task('watch:less', gulp.series('less', function() {
return gulp.watch('less/**/*.less', gulp.parallel(['less']));
}));
'js:config'].concat(production() ? 'font' : [])));
gulp.task('default', gulpSequence('clean', gulp.series(['html', 'js:config', production() && 'font'].filter(Boolean))));
Could you please help me?
I need gulp 4 because gulp 3 has vulnerable.
I tried to use allowEmpty: true. No results.
This code works with gulp 3.
Any help would be appreciated.
Sorry for my bad English, it is not my native language.

NodeJS creating JSON using all JSONs uploaded by user

I am trying to make a JSON file using all the JSON files in a directory. Every time a user uploads a new JSON a new combined JSON should be generated. I want the new JSON to have a custom structure hence cant use any libraries. I have the following code:
router.post('/upload', function(req, res) {
var sampleFile;
var bbbid = req.body.bbbid;
DDLFile = req.files.DDLFile;
j++;
DDLFile.mv('/uploads/' + bbbid + '/device' + j + '.json', function (err) {
if (err) {
res.status(500).send(err);
}
else {
res.redirect("fileuploaded");
}
});
var myfiles = [];
var fs = require('fs');
var arrayOfFiles = fs.readdirSync('/uploads/' + bbbid);
arrayOfFiles.forEach(function (file) {
myfiles.push(file);
console.log(myfiles);
});
console.log('No of Files:', myfiles.length);
var files = myfiles.length;
console.log('Files:', files);
console.log('J', j);
var cddl = "{ BBBID:" + bbbid;
if (files == 0) {
cddl = cddl + '}';
console.log('Entered if loop');
}
else {
var i = 0;
/*var obj;
fs.readFile('/uploads/' + bbbid + '/device' + j + '.json', 'utf8', function (err, data) {
if (err) throw err;
obj = JSON.parse(data);
});*/
for (i = 0; i < files; i++) {
console.log('Entered For loop');
console.log('Count:', count);
console.log('Sensor:', sensor);
try{
var obj = fs.readFileSync('/uploads/' + bbbid + '/device' + count + '.json', 'utf8');}
catch(err){
console.log(err);
}
console.log('everything good');
var obj1 = JSON.parse(obj);
console.log('hi');
//JSON.stringify(obj);
var ddl = require('/uploads/' + bbbid + '/device' + count + '.json');
console.log('o');
cddl = cddl + ", {" + obj1.DDL.Sensor.Description.Verbose_Description + ":" + JSON.stringify(ddl) + "}"
JSON.stringify(cddl);
console.log(cddl);
count++;
sensor++;
console.log('Count:', count);
console.log('Sensor:', sensor);
}
cddl = cddl + '}';
JSON.stringify(cddl);
console.log(cddl);
}
});
I want to generate a new cddl everytime a new file is uploaded. Having a lot of problems. Help please!
I see two problems. First instead of this:
var obj = fs.readFileSync('/uploads/' + bbbid + '/device' + count + '.json', 'utf8');}
catch(err){
console.log(err);
}
console.log('everything good');
var obj1 = JSON.parse(obj);
You can write(fix path, if necessary):
var obj1 = require('./uploads/' + bbbid + '/device' + count + '.json')
Then, when you call:
JSON.stringify(cddl);
You're not saving the result anywhere. So you should save it in the place, you need to:
var a = JSON.stringify(cddl);
And when all set, dont forget to write to file back using fs.writeFileSync or async one fs.writeFile.

Express REST API response methods are not recognized

I have this really simple get request that returns json that I am trying to implement. I have followed the tutorials for Express Web Framework REST API, but for some reason I keep getting the same error
ERROR:
TypeError: res.status is not a function
or
TypeError: res.json is not a function
index.js:
var express = require('express');
var router = express.Router();
var pg = require('pg');
var connectionString = 'pg://postgres:postgres#postgres/feed';
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index', { title: 'Express' });
});
router.get('/api/leaderboard', function(resp, req){
var results = [];
pg.connect(connectionString, function(err, client, done){
if(err){
done();
console.log(err);
return res.status(500).json({ success: false, data: err});
}
var query = client.query("SELECT * FROM log WHERE (logged >= date_trunc('week', CURRENT_TIMESTAMP - interval '1 week') AND logged <= date_trunc('week', CURRENT_TIMESTAMP));");
var counter = 0;
var b1 = {};
var b2 = {};
var b3 = {};
var b4 = {};
b1.energy_sum_week = 0;
b2.energy_sum_week = 0;
b3.energy_sum_week = 0;
b4.energy_sum_week = 0;
b1.zne_sum_week = 30000;
b2.zne_sum_week = 30000;
b3.zne_sum_week = 30000;
b4.zne_sum_week = 30000;
query.on('row', function(row){
//results.push(row);
if(row['address'] == 215){
b1.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
}
else if (row['address'] == 1590) {
b2.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
} else if (row['address'] == 1605) {
console.log(row);
b3.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
} else if (row['address'] == 1715) {
b4.energy_sum_week = row['kitchen'] + row['plugload'] + row['lights'] + row['ev'] + row['hvac'] + row['instahot'] - row['solar'];
}
});
query.on('end', function(){
done();
//make zne lower than everything
results.push(b1);
results.push(b2);
results.push(b3);
results.push(b4);
resp.json(results);
});
});
});
module.exports = router;
It seems like it can't recognize the response object. Tried a bunch of different things like passing in the request and response's to the query callbacks, and using promises.
Getting kinda desperate here :/
The res variable doesn't exist in the current context, you probably expect that the line
router.get('/api/leaderboard', function(resp, req){
had this form
router.get('/api/leaderboard', function(req, res){
You are passing resp as the req object and the req as the resp object.
Try changing the order.
router.get('/api/leaderboard', function(req, resp){...}

multiple async mongo request generate messed up returns

I'm trying to build a JSON out of multiple requests on my mongodb.
since I'm not using DBRef, I have to build the "table joints" by myself, and that's how I ended up in this mess.
This is the code that is giving me the headaches from a couple of days now.
(the mongo part is done with mongoskin)
var getUserFeed = function(thelimit, out) {
userfeed = db.collection("userfeed");
apparel = db.collection("apparel");
store = db.collection("stores");
if(thelimit)
args = {limit:thelimit, sort: [['date',-1]]};
userfeed.find({},args).toArray(function(e, feed) {
if (e) console.log("error: ", e);
// gather aparel infos
var i=0;
var ret_feeds = [];
feed.forEach(function(cur_feed) {
var outfits=[];
console.log("beginning with: " + cur_feed.url);
var resfeed = "";
resfeed = cur_feed;
resfeed.url = baseurl + snapurl + resfeed.url + "_small.jpg";
i=0;
cur_feed.apparel_ids.forEach(function(item) {
/*>>*/ apparel.find({"_id": item},{limit:1}).toArray(function(e, results) {
console.log(">>>>>>>>>>> APPAREL_FIND { i:" + i + "}");
if (e) console.log("error: ", e);
results = results[0];
if(results.apparel_cat == 1)
url_subcat = "pants/";
else if(results.apparel_cat == 2)
url_subcat = "shirts/";
else if(results.apparel_cat == 2)
url_subcat = "tshirts/";
results.thumb = baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg";
results.size = "M"; ///// TOBE REAL VERY SOON
results.gallery = [
baseurl + outfiturl + url_subcat + results.apparel_id + "/model.jpg",
baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg"
];
outfits.push(results); // quick and dirty, 2 b refined..
i++;
if(i>=cur_feed.apparel_ids.length)
{
// pack it up
// resfeed.url = resfeed.url;
resfeed.outfits = outfits;
resfeed.fav = false;
resfeed.bough = false;
// retrieve store infos
/*>>>*/ store.find({"_id":resfeed.store_id}, {limit: 1}).toArray(function(e, resstore) {
console.log("\t############# STORE_FIND { i:" + i + "}");
if (e) console.log("error: ", e);
resfeed.store = resstore[0];
resfeed.store.class = "hem";
ret_feeds.push(resfeed);
if(ret_feeds.length >= feed.length)
{
console.log("\t\t######################calling return [ ret_feeds.length = " + ret_feeds.length + " feed.length = " + feed.length);
out.send(ret_feeds);
}
});
}
});
});
});
});
}
This code fails, because returns the json before finishing its task, so the next time that it tries to return another json it crashes miserably due to the fact the the headers have already been sent.
Now as you can see, I have 3 collections: userfeed, apparel and stores.
the goal of this function is to retrieve all the items in the userfeed collection, extract the outfits (based on the outfit_id array that is part of the userfeed collection), and also extract the store infos related in the same way to each userfeed entry, like so:
I know that async.js or equivalent is the way to go: I've red like a gazillion of other posts here on SO, but I still can't get my head around it, probably because the whole mechanism behind the async.js or flow control in general it's still out of focus in my mind.
I'm still a noob at node :)
UPDATE
I think I found the right path for understanding here: http://www.sebastianseilund.com/nodejs-async-in-practice
this guy made a terrific job in describing use-case by use-case all the ways to apply async.js to your code.
I'll post the solution as soon as I get around it.
UPDATE 2
Thanks to the above dude I could work out a working solution, below is the answer.
After so much struggling I have finally managed to get a solution.
async.js was the answer as I was (obviously) suspecting.
FYI here's the working code.
If you like to point out improvements or anything else, you are more than welcome
var getUserFeed = function(thelimit, out) {
userfeed = db.collection("userfeed");
apparel = db.collection("apparel");
store = db.collection("stores");
var args;
if(thelimit)
args = {limit:thelimit, sort: [['date',-1]]};
var outfits=[];
var feeds = array();
async.series([
// userfeed find
function(callback) {
userfeed.find({},args).toArray(function(e, feed) {
if(e) callback(e);
feeds = array(feed);
console.log(feeds.length + " retrieved. stepping in");
callback(null, null);
});
},
// join
function(callback) {
async.forEach(feeds, function(thefeed, callback) {
var i = feeds.indexOf(thefeed);
async.parallel([
// load apparel infos
function(callback) {
console.log("\t >>> analyzing thefeed id " + thefeed._id);
async.forEach(thefeed.apparel_ids, function(apparel_id, callback) {
apparel.find({"_id": apparel_id},{limit:1}).toArray(function(e, results) {
if (e) console.log("error: ", e);
results = results[0];
if(results.apparel_cat == 1)
url_subcat = "pants/";
else if(results.apparel_cat == 2)
url_subcat = "shirts/";
else if(results.apparel_cat == 2)
url_subcat = "tshirts/";
results.thumb = baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg";
results.size = "M"; ///// TOBE REAL VERY SOON
results.gallery = [
baseurl + outfiturl + url_subcat + results.apparel_id + "/model.jpg",
baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg"
];
console.log("\t\t### pushing data into thefeed_index: " + i);
if(!util.isArray(feeds[i].oufits)) feeds[i].outfits = array();
feeds[i].outfits.push(results);
callback(null, null);
});
}, callback);
},
// load store infos
function(callback) {
store.find({"_id":thefeed.store_id}, {limit: 1}).toArray(function(e, resstore) {
console.log("\t### STORE_FIND");
if (e) console.log("error: ", e);
feeds[i].store = resstore[0];
feeds[i].store.class = "hem";
callback(null, null);
});
}
], callback);
}, callback);
}
// MAIN
], function(err, result) {
console.log("feed retrieval completed. stepping out");
if (err) return next(err);
out.send(feeds);
});
};