I am trying to migrate our build process to gulp from existing custom bash build script. We concatenate several unminified opensource JS files like bootstrap, lazyload, ... and our own JS files. We uglify each JS file (removing their licenses as well) in an order, prepend custom license text to some of them as required and concatenate to create the output JS file. The custom license text are currently kept as strings in the bash script.
How to achieve this in gulp without creating intermediate files?
Will it also be possible to selectively avoid uglifying some JS scripts?
Ok, I spent some time learning up gulp and it's plugins and here is a working version. The points here are using the foreach on each JS retrieved from the JSON config file, pushing the streams to an array and finally using merge on the array streams.
Here are the plugins used and the JSON structure defined:
var gulp = require('gulp');
var each = require('foreach');
var debug = require('gulp-debug');
var gulpif = require('gulp-if');
var jshint = require('gulp-jshint');
var uglify = require('gulp-uglify');
var concat = require('gulp-concat-util');
var es = require('event-stream');
var cache = require('gulp-cached');
var remember = require('gulp-remember');
// Structure that holds the various JS files and their handling
var Config = {
js: {
output_dir: 'path/to/output/file/',
output_file: 'outputfile.js',
src: [{
name: 'bootstrap',
src: ['path/to/bootstrap.js'],
run_lint: false,
run_uglify: true,
license: '/* bootstrap license */'
}, {
name: 'lazyload',
src: ['path/to/lazyload.js'],
run_lint: false,
run_uglify: true,
license: '/* lazyload license */'
}, {
name: 'inhouse-js',
src: ['path/to/inhouse/ih-1.js', 'path/to/inhouse/ot/*.js'],
run_lint: true,
run_uglify: true,
license: ''
}]
}
}
The build task, with caching as we will be using it in development also:
gulp.task('build', ['build:js']);
gulp.task('build:js', function() {
var streams = [];
each(Config.js.src, function(val, key, array) {
var stream = gulp.src(val.src)
.pipe(cache('scripts'))
.pipe(gulpif(val.run_lint, jshint('.jshintrc')))
.pipe(gulpif(val.run_lint, jshint.reporter('jshint-stylish')))
.pipe(gulpif(val.run_uglify, uglify({
compress: {
drop_console: true
}
})))
.pipe(concat.header(val.license + '\n'));
streams.push(stream);
});
es.merge.apply(this, streams)
.pipe(remember('scripts')) // add back all files to the stream
.pipe(concat(Config.js.output_file))
.pipe(gulp.dest(Config.js.output_dir));
});
If you would like to debug, a good option will be to insert debug plugin like this example around the 'gulp-remember' plugin call above:
.pipe(debug({title: 'before remember:'}))
.pipe(remember('scripts')) // add back all files to the stream
.pipe(debug({title: 'after remember:'}))
And here's the watch task:
gulp.task('watch', function() {
var watch_list = [];
each(Config.js.src, function(val, key, array) {
watch_list.push.apply(watch_list, val.src);
});
// Watch .js files
var watcher = gulp.watch(watch_list, ['build']);
watcher.on('change', function(event) {
console.log('File '+ event.path +' was '+ event.type +', running tasks..');
if (event.type === 'deleted') { // if a file is deleted, forget it
delete cache.caches['scripts'][event.path];
remember.forget('scripts', event.path);
}
})
});
You can use lazypipe() to reuse parts of the build:js task with normal build.
Related
Had to rewrite my gulpfile to gulp 4 but I'm facing problems of gulp-inject injecting nothing.
var gulp = require('gulp');
var svgstore = require('gulp-svgstore');
var svgmin = require('gulp-svgmin');
var rename = require('gulp-rename');
var cheerio = require('gulp-cheerio');
var inject = require('gulp-inject');
var path = require('path');
gulp.task('svgstore', function () {
var svgs = gulp
.src('static/svg/icons/src/*.svg')
.pipe(svgmin(function (file) {
var prefix = path.basename(file.relative, path.extname(file.relative));
return {
plugins: [{
convertPathData: false
}, {
cleanupIDs: {
prefix: prefix + '-',
minify: true
}
}]
}
}))
.pipe(cheerio(function ($, file) {
$('[fill]').attr('fill', 'currentColor');
}))
.pipe(svgstore({ inlineSvg: true }))
.pipe(gulp.dest('dest'));
function fileContents(filePath, file) {
return file.contents.toString();
}
return gulp
.src('layouts/partials/src/inline-svg.html')
.pipe(inject(svgs, { transform: fileContents }))
.pipe(gulp.dest('layouts/partials'));
});
// Watch asset folder for changes
gulp.task("watch", function () {
var watcher = gulp.watch("static/svg/icons/src/**/*")
watcher.on('all', gulp.series('svgstore'))
})
// Set watch as default task
gulp.task("default", gulp.series("watch"))
I can't figure out why. I tried logging fileContents and it just says [Function: fileContents].
I faced the same issue. When I debugged the gulp-inject package, I figured out that the gulp-inject plugin look up for the startTag and the endTag in the source file/s (in your case, the source file/s is gulp.src('layouts/partials/src/inline-svg.html')) to inject the content, and when I printed out the tags to the console, the result is:
startTag: /<!\-\-\s*inject\b:svg\b\s*\-\->/gi
endTag: /<!\-\-\s*endinject\s*\-\->/gi
Conclusion:
In the src file(i.e., inline-svg.html), insert the comments like below:
<div class="sr-only">
<!-- inject:svg --><!-- endinject -->
</div>
Then, try to run the gulp task and it should work.
Reference: https://github.com/w0rm/gulp-svgstore#inlining-svgstore-result-into-html-body
Gulp Version Used: ^5.0.4
At first I thought this was related to dependency of tasks so I went with run-sequence and even tried defining dependencies within tasks themselves. But I cannot get the compress task to run after copy. Or, even if it says it did finish the compress task, the compression only works if I run compress in the task runner inside visual studio by itself. What else can I try to get it to compress after copy?
/// <binding BeforeBuild='default' />
/*
This file is the main entry point for defining Gulp tasks and using Gulp plugins.
Click here to learn more. https://go.microsoft.com/fwlink/?LinkId=518007
*/
var gulp = require("gulp");
var debug = require("gulp-debug");
var del = require("del");
var uglify = require("gulp-uglify");
var pump = require("pump");
var runSequence = require("run-sequence");
var paths = {
bower: "./bower_components/",
lib: "./Lib/"
};
var modules = {
"store-js": ["store-js/dist/store.legacy.js"],
"bootstrap-select": [
"bootstrap-select/dist/css/bootstrap-select.css",
"bootstrap-select/dist/js/bootstrap-select.js",
"bootstrap-select/dist/js/i18n/*.min.js"
]
}
gulp.task("default", function (cb) {
runSequence("clean", ["copy", "compress"], cb);
});
gulp.task("clean",
function () {
return del.sync(["Lib/**", "!Lib", "!Lib/ReadMe.md"]);
});
gulp.task("compress",
function (cb) {
pump([
gulp.src(paths.lib + "**/*.js"),
uglify(),
gulp.dest(paths.lib)
], cb);
});
gulp.task("copy",
function (cb) {
prefixPathToModules();
copyModules();
cb();
});
function prefixPathToModules() {
for (var moduleIndex in modules) {
for (var fileIndex in modules[moduleIndex]) {
modules[moduleIndex][fileIndex] = paths.bower + modules[moduleIndex][fileIndex];
}
}
}
function copyModules() {
for (var files in modules) {
gulp.src(modules[files], { base: paths.bower })
.pipe(gulp.dest(paths.lib));
}
}
You use run-sequence and your code
runSequence("clean", ["copy", "compress"], cb);
run in such order
clean
copy and compress in parallel // that's why your code compresses nothing, because you have not copied files yet
cb
Write like this and compress will be after copy
runSequence("clean", "copy", "compress", cb);
I am not familiar with runSequence. But why don't you try the following. By this way your default task depends on compress and compress depends on copy. So, 'copy' will run first and then 'compress'
gulp.task('default', ['copy','compress'], function(cb){});
gulp.task('compress',['copy'], function(cb){});
Gulp returns a steam , since you are calling it in a for loop the stream is returned during the first iteration itself.
Update your copyModule to the following and you can try either runSequence like posted by Kirill or follow my approach
function copyModules() {
var inputFileArr = [];
for (var files in modules) {
inputFileArr = inputFileArr.concat(modules[files]);
};
return gulp.src(inputFileArr, { base: paths.bower })
.pipe(gulp.dest(paths.lib));
}
Is it possible to get the list of files coming from a gulp.src stream as an array, e.g.:
var files = convertToArray(gulp.src('**/*.js'));
Update:
I was trying to move away from the gulp-karma plugin:
gulp.task('test', function () {
return gulp.src(files)
.pipe($.order(ordering))
.pipe($.karma({
karma.conf.js'
});
});
So my idea was:
gulp.task('test', function (done) {
var karmaFiles = convertToArray(gulp.src(files)
.pipe($.order(ordering)));
new Server({
configFile: karma.conf.js',
files: karmaFiles
}, done).start();
});
But as pointed out, this won't work because of it being async. Here's my solution:
gulp.task('test', function (done) {
gulp.src(files)
.pipe($.order(ordering)))
.pipe(gutil.buffer())
.on('data', function(data) {
var karmaFiles = data.map(function(f) { return f.path; });
new Server({
configFile: __dirname + '/karma.conf.js',
files: karmaFiles
}, done).start();
});
});
Gulp streams are always asynchronous so your hypothetical convertToArray function (which takes a stream and returns an array) is impossible.
The only way to get all the files in a stream is through some kind of callback function. The gulp-util package, which bundles various helper functions, provides the nice gutil.buffer() :
var gutil = require('gulp-util');
gulp.src('**/*.js').pipe(gutil.buffer(function(err, files) {
console.log('Path of first file:');
console.log(files[0].path);
console.log('Contents of first file:');
console.log(files[0].contents.toString());
}));
In the above files will be an array of vinyl files. That means for each file you have access to both the contents and the path of the file.
If you don't care about the file contents and only want the path of each file you shouldn't be using gulp.src() at all. You should be using glob instead (which is what gulp is using internally). It gives you a synchronous method that returns an array of matching file paths:
var glob = require('glob');
var files = glob.sync('**/*.js');
console.log(files);
The code bellow doesn't merge correctly rev-manifest.json file.
I loop several JS tasks and just one is merged, although hash files are being created and stored correctly.
I already tried a ton of things, I checked gulp-rev and some users seam to have similar problems. Some of them are creating several manifest files and proceed with the actual merge at the end. I would like to discard this solutions since it's slow and ugly.
If I comment the concat(...) line the manifest file registers all the JS tasks.
Is this a BUG or am I missing something here?
gulp 3.9.1
gulp-concat 2.6.0
gulp-rev 7.0.0
var gulp = require('gulp');
var less = require('gulp-less');
var minifycss = require('gulp-minify-css');
var jshint = require('gulp-jshint');
var uglify = require('gulp-uglify');
var concat = require('gulp-concat');
var rev = require('gulp-rev');
var jsFiles = {
task1: [
'./path/file1.js'
],
task2: [
'./path/file2.js',
'./path/file2.js'
]
};
function jsTask(key) {
gulp.task(key, function() {
gulp.src(jsFiles[key])
.pipe(jshint())
.pipe(jshint.reporter('default'))
.pipe(uglify())
// IT WORKS WHEN I COMMENT THIS LINE
.pipe(concat(key + '.min.js'))
.pipe(rev())
.pipe(gulp.dest('./public/js'))
.pipe(rev.manifest({merge:true }))
.pipe(gulp.dest('.'));
});
}
gulp.task('less', function() {
return gulp.src(['./path/less/*.less'])
.pipe(less({errLogToConsole: true}))
.pipe(minifycss())
.pipe(rev())
.pipe(gulp.dest('./path/public/css'))
.pipe(rev.manifest({merge:true }))
.pipe(gulp.dest('.'));
});
for (var key in jsFiles) {
jsTask(key);
}
var defaultTasks = ['less'];
for (var key in jsFiles) {
defaultTasks.push(key);
}
gulp.task('default', defaultTasks);
You can pass the name of the manifest file you want to create(different for each gulp task) to manifest function of the gulp-rev-all module like below
gulp.task('productionizeCss', function () {
return gulp
.src(['dist/prod/**/*.css'])
.pipe(revAll.revision({
fileNameManifest: 'css-manifest.json'
}))
.pipe(gulp.dest('dist/prod/'))
.pipe(revAll.manifestFile())
.pipe(gulp.dest('dist/prod/'));
});
gulp.task('productionizeJS', function () {
return gulp
.src(['dist/prod/**/*.js'])
.pipe(revAll.revision({
fileNameManifest: 'js-manifest.json'
}))
.pipe(gulp.dest('dist/prod/'))
.pipe(revAll.manifestFile())
.pipe(gulp.dest('dist/prod/'));
});
Here, I have two gulp tasks, one to revise all JS and one for CSS.So, I have created two manifest files css-manifest.json, js-manifest.json.
Then I specified both the manifest files in src of the rev-replace module as shown below:
gulp.task('revReplaceIndexHtml', function () {
var manifest = gulp.src(["dist/prod/js-manifest.json", 'dist/prod/css-manifest.json']);
return gulp.src('dist/dev/referralswebui/index.html')
.pipe(revReplace({ manifest: manifest, replaceInExtensions: ['.html']}))
.pipe(gulp.dest('dist/prod/referralswebui/'));
});
I would suggest using gulp-useref instead of gulp-concat.
Given your setup, I think key references a glob path, or at least I hope so. Otherwise you are trying to concatenate a single file, or no files which may crash the concat plug-in. Emphasis on may.
Also, since you are using gulp-rev, I suggest using gulp-rev-replace which will automatically update your index references to the reved files.
Edit
Sometimes rev.manifest behaves in ways that I would describe as buggy. Just to exhaust all possibilities remove the merge option for the manifest and run concat. Or run concat and remove manifest altogether.
I am following the Fast browserify builds with watchify recipe and get it working, but would like my bundles to be written to the original source folder.
For example, I have the following code (taken directly from the recipe, but modified slightly)
// add custom browserify options here
var customOpts = {
entries: glob.sync("./dev/www/**/*-root.js"),
debug: true
};
var opts = _.assign({}, watchify.args, customOpts);
var b = watchify(browserify(opts));
gulp.task('js', bundle); // so you can run `gulp js` to build the file
b.on('update', bundle); // on any dep update, runs the bundler
b.on('log', gutil.log); // output build logs to terminal
function bundle() {
return b.bundle()
// log errors if they happen
.on('error', gutil.log.bind(gutil, 'Browserify Error'))
.pipe(source('bundle.js'))
// optional, remove if you don't need to buffer file contents
.pipe(buffer())
// optional, remove if you dont want sourcemaps
.pipe(sourcemaps.init({ loadMaps: true })) // loads map from browserify file
// Add transformation tasks to the pipeline here.
.pipe(sourcemaps.write('./')) // writes .map file
.pipe(gulp.dest((file) => {return file.base;}));
//.pipe(gulp.dest('./dist'));
}
as you can see in the entries: glob.sync("./dev/www/**/*-root.js"), line, I am scanning for multiple files to be bundled.
The problem I am encountering is the .pipe(gulp.dest((_file: any) => {return file.base;})); line, which returns the root project folder's path, not the original source folder's path.
How can I obtain the source folder path for writing to?
Edit
I found http://fettblog.eu/gulp-browserify-multiple-bundles/ which describes how to create multiple bundles, but it doesn't use watchify. it looks like this issue might be a limitation of vinyl-source-stream?
It took a lot of effort, but I figured out a solution.
Generally I followed the pattern found at http://fettblog.eu/gulp-browserify-multiple-bundles/ but added caching of each browserify object created (one per page) plus added watchify to it.
Other things added in the following code:
livereload
dev server launch
typescript (tsx) compile (including chained sourcemaps)
full solution (typescript code):
import gulp = require("gulp");
import browserify = require("browserify");
var watchify = require("watchify");
import source = require("vinyl-source-stream");
import buffer = require("vinyl-buffer");
import gutil = require("gulp-util");
import sourcemaps = require("gulp-sourcemaps");
var sourcemapsApply = require("vinyl-sourcemaps-apply");
import _ = require("lodash");
import glob = require("glob");
import vinyl = require("vinyl");
import rename = require("gulp-rename");
var minifyify = require("minifyify");
import path = require("path");
var tsify = require("tsify");
var livereload = require("gulp-livereload");
var notify = require("gulp-notify");
var closureCompiler = require("gulp-closure-compiler");
import uglify = require("gulp-uglify");
import http = require("http");
var st = require("st"); //module for serving static files. used to create dev server. https://www.npmjs.com/package/st
var eventStream = require("event-stream"); //module for merging multiple vinyl streams to return one when finishing task. see http://fettblog.eu/gulp-browserify-multiple-bundles/
var rootPath = __dirname;
gulp.task("default", () => {
gulp.start("tsxDevWatch");
});
gulp.task("devServer", (done) => {
var rootPath = __dirname;
//start server
http.createServer(st({
path: rootPath,
index: true, //"index.html",
cache: false,
})).listen(8080,"localhost", done);
});
gulp.task("tsxDevWatch",["devServer"], () => {
livereload.listen();
//browserify+watchify of *-main.js files and compiles into *-main.bundle.js files IN SAME FOLDER
//created mostly following the pattern described here: http://fettblog.eu/gulp-browserify-multiple-bundles/
//but adds stupid source-filepath workaround for use with "source" (vinyl-source-stream needed for watchify)
/** the files we are going to browserify bundle*/
var entries = glob.sync("./dev/www/**/*-main.tsx", {
realpath: true, //so paths are absolute. This is required so our "bMaps" mappings stay intact, because watchify.on("update") always provides full filepath,
});
/** we create one browserify instance for each file we are bundling. this caches the browserify instance so it can be reused on watchify updates (decreasing compile time by A LOT) */
var bMaps: { [key: string]: BrowserifyObject } = {};
var tasks = entries.map((entry) => {
process.chdir(path.dirname(entry));
var browserifyOptions = {
entries: [entry],
debug: true,
plugin: [
watchify,
//tsify,
],
cache: {}, packageCache: {}, fullPaths: true // Requirement of watchify
};
var b = browserify(browserifyOptions);
b.plugin(tsify, { //options from here: http://json.schemastore.org/tsconfig
jsx: "react",
//inlineSourceMap: false, //sourcemap options don't seem to matter, seems to be set by browserify or something.
//sourceMap:true,
module: "commonjs",
target: "es5",
});
bMaps[entry] = b;
b.on('update', (updatedFiles: string[]) => {
console.log("!!!!!!!!!!!!!! \n!!!!!!!!!!!!!!!!!!!\n UPDATE CALLED FOR", JSON.stringify(updatedFiles));
var rebuildAll = false;
_.forEach(updatedFiles, (updatedFile) => {
if (bMaps[updatedFile] == null) {
//a dependency needs to be rebuilt, skip rebuilding anything that changed and do EVERYTHING
rebuildAll = true;
return false;
}
});
if (rebuildAll === false) {
_.forEach(updatedFiles, (updatedFile) => {
console.log(" ============= update()", updatedFile);
//find the b for this file
var _b = bMaps[updatedFile];
//do a bundle for it
_createTsXBundle(_b, updatedFile);
});
} else {
//this is a dependency, rebuild EVERYTHING!!!
_.forEach(bMaps, (value_b, key_entry) => {
_createTsXBundle(value_b, key_entry);
});
}
}); // on any dep update, runs the bundler
b.on('log', gutil.log); // output build logs to terminal
return _createTsXBundle(b, entry);
});
return eventStream.merge.apply(null, tasks);
});
/** worker to create a tsx bundle. used by a task */
function _createTsXBundle(b: BrowserifyObject, entry: string) {
process.chdir(path.dirname(entry));
console.log("================= doBundle()", entry);
var bundledStream = b.bundle();
bundledStream = <any>bundledStream.on('error', gutil.log.bind(gutil, 'Browserify Error'));
var currentSource: vinyl;
var targetName = path.basename(entry, ".tsx") + ".bundle.js";
bundledStream
.pipe(source(targetName))
.pipe(buffer()) //need this to support chaining our vinyl source file
////////////////////////////////////////
//// optional, remove if you dont want sourcemaps
// .pipe(sourcemaps.init({
// loadMaps: true,
// //debug: true
// })) // loads map from browserify file
///////////////////////// WORKS, BUT NEED TO ENABLE SOURCEMAPS plugin TO GET SOURCEMAPS
//// uglify
//.pipe(uglify({
// //preserveComments: "all",
// output:<any> {
// max_line_len: 300,
// //screw_ie8: false,
// //beautify: true,
// //comments: true,
// //bracketize: true,
// //quote_keys: true,
// //width: 120,
// //semicolons:true,
// },
// compress: {
// unsafe:true,
// },
//}))
//// Add transformation tasks to the pipeline here.
// .pipe(sourcemaps.write())
.pipe(gulp.dest((file: vinyl) => {
//console.log("GULP.DEST(file)\n base=", file.base, "\n cwd=", file.cwd, "\n path=", file.path, "\n relative=", file.relative);
return file.base;
}))
.pipe(livereload())
;
return bundledStream;
}