I use gulp to configure complex local setup and need to auto-edit files.
The scenario is:
determine if certain file contains certain lines after certain other line (found using regular expression)
if line is not found, insert the line.
optionally, delete some lines found in the file.
I need this to amend system configuration files and compile scenarios.
What would be the best way to do it in gulp?
Gulp is plain javascript. So what I would do if I were you is to create a plugin to pipe to the original config file.
Gulp streams emit Vinyl files. So all you really got to do is to create a "pipe factory" that transforms the objects.
It would look something like this (using EventStream):
var es = require('event-stream');
// you could receive params in here if you're using the same
// plugin in different occasions.
function fixConfigFile() {
return es.map(function(file, cb) {
var fileContent = file.contents.toString();
// determine if certain file contains certain lines...
// if line is not found, insert the line.
// optionally, delete some lines found in the file.
// update the vinyl file
file.contents = new Buffer(fileContent);
// send the updated file down the pipe
cb(null, file);
});
}
gulp.task('fix-config', function() {
return gulp.src('path/to/original/*.config')
.pipe(fixConfigFile())
.pipe(gulp.dest('path/to/fixed/configs');
});
Or you can use vinyl-map:
const map = require('vinyl-map')
const gulp = require('gulp')
const modify = map((contents, filename) => {
contents = contents.toString()
// modify contents somehow
return contents
})
gulp.task('modify', () =>
gulp.src(['./index.js'])
.pipe(modify)
.pipe(gulp.dest('./dist'))
})
Related
How can I validate CSV files (encoding, headline, delimiter, column count) in my Grunt build? I had a look at CSVLint but neither get it to work, nor know how to include it in Grunt.
Edit: PapaParse looks promising, but has no Grunt integration either.
Although grunt integration for PapaParse does not exist its API can be utilized by configuring a custom Function Task inside your Gruntfile.js.
Install papaparse via npm
Firstly, cd to your project directory, install papaparse via npm and add it to the devDependencies section of your projects package.json. To do this run the following command via your CLI tool:
$ npm i -D papaparse
Gruntfile.js
The following gist shows how to configure a custom Function Task named validateCSV in your Gruntfile.js.
module.exports = function(grunt) {
// Requirements
var fs = require('fs');
var Papa = require('papaparse');
// Other project configuration tasks.
grunt.initConfig({
// ...
});
/**
* Register a custom Function task to validate .csv files using Papa Parse.
*/
grunt.registerTask('validateCSV', 'Lint .csv files via Papa Parse', function() {
var glob = './csv/*.csv'; // <-- Note: Edit glob pattern as required.
var success = true;
// Create an Array of all .csv files using the glob pattern provided.
var csvFiles = grunt.file.expand(glob).map(function(file) {
return file;
});
// Report if no .csv files were found and return early.
if (csvFiles.length === 0) {
grunt.log.write('No .csv files were found');
return;
}
// Loop over each .csv file in the csvFiles Array.
csvFiles.forEach(function(csvFile) {
// Read the contents of the .csv file.
var csvString = fs.readFileSync(csvFile, {
encoding: 'utf8'
});
// Parse the .csv contents via Papa Parse.
var papa = Papa.parse(csvString, {
delimiter: ',',
newline: '',
quoteChar: '"',
header: true,
skipEmptyLines: true
// For additional config options visit:
// http://papaparse.com/docs#config
});
// Basic error and success logging.
if (papa.errors.length > 0) {
grunt.log.error('Error(s) in file: '['red'] + csvFile['red']);
// Report each error for a single .csv file.
// For additional Papa Parse errors visit:
// http://papaparse.com/docs#errors
papa.errors.forEach(function(error) {
grunt.log.write('\n type: ' + error.type);
grunt.log.write('\n code: ' + error.code);
grunt.log.write('\n message: ' + error.message);
grunt.log.write('\n row: ' + error.row + '\n\n');
});
// Indicate that a .csv file failed validation.
success = false;
} else {
grunt.log.ok('No errors found in file: ' + csvFile);
}
});
// If errors are found in any of the .csv files this will
// prevent subsequent defined tasks from being processed.
if (!success) {
grunt.fail.warn('Errors(s) were found when validating .csv files');
}
});
// Register the custom Function task.
grunt.registerTask('default', [
'validateCSV'
// ...
]);
};
Notes
The following line of code (taken from the Gruntfile.js above) that reads:
var glob = './csv/*.csv';
... will need to be changed/edited according to your project requirements. Currently the globbing pattern assumes all .csv files reside inside a folder named csv.
You may also need to set the config options as per your requirements.
The custom Function Task also includes some basic error and success reporting that will be logged to the CLI.
Running the Task
To run the grunt task simply execute the following via your CLI tool:
$ grunt validateCSV
EDIT: Updated Answer (based on the following comment...)
Would it also be possible to "configure" the task from within the
grunt.initConfig()? For example linting different CSV directories?
To achieve this you can create a separate Javascript module that exports a Registered MutliTask.
Lets call it papaparse.js and save it to a directory named custom-grunt-tasks which resides in the same top level directory as your Gruntfile.js
Note: This .js file and directory name can be any name that you prefer, however you will need to updated the references inside Gruntfile.js.
papaparse.js
module.exports = function(grunt) {
'use strict';
// Requirements
var fs = require('fs');
var Papa = require('papaparse');
grunt.registerMultiTask('papaparse', 'Misc Tasks', function() {
// Default options. These are used when no options are
// provided via the initConfig({...}) papaparse task.
var options = this.options({
quotes: false,
delimiter: ',',
newline: '',
quoteChar: '"',
header: true,
skipEmptyLines: true
});
// Loop over each path provided via the src array.
this.data.src.forEach(function(dir) {
// Append a forward slash If a directory path
// provided does not end in with one.
if (dir.slice(-1) !== '/') {
dir += '/';
}
// Generate the globbin pattern.
var glob = [dir, '*.csv'].join('');
// Create an Array of all .csv files using the glob pattern.
var csvFiles = grunt.file.expand(glob).map(function(file) {
return file;
});
// Report if no .csv files were found and return early.
if (csvFiles.length === 0) {
grunt.log.write(
'>> No .csv files found using the globbing '['yellow'] +
'pattern: '['yellow'] + glob['yellow']
);
return;
}
// Loop over each .csv file in the csvFiles Array.
csvFiles.forEach(function(csvFile) {
var success = true;
// Read the contents of the .csv file.
var csvString = fs.readFileSync(csvFile, {
encoding: 'utf8'
});
// Parse the .csv contents via Papa Parse.
var papa = Papa.parse(csvString, options);
// Basic error and success logging.
if (papa.errors.length > 0) {
grunt.log.error('Error(s) in file: '['red'] + csvFile['red']);
// Report each error for a single .csv file.
// For additional Papa Parse errors visit:
// http://papaparse.com/docs#errors
papa.errors.forEach(function(error) {
grunt.log.write('\n type: ' + error.type);
grunt.log.write('\n code: ' + error.code);
grunt.log.write('\n message: ' + error.message);
grunt.log.write('\n row: ' + error.row + '\n\n');
});
// Indicate that a .csv file failed validation.
success = false;
} else {
grunt.log.ok('No errors found in file: ' + csvFile);
}
// If errors are found in any of the .csv files this will prevent
// subsequent files and defined tasks from being processed.
if (!success) {
grunt.fail.warn('Errors(s) found when validating .csv files');
}
});
});
});
};
Gruntfile.js
Your Gruntfile.js can then be configured something like this:
module.exports = function(grunt) {
grunt.initConfig({
// ...
papaparse: {
setOne: {
src: ['./csv/', './csv2']
},
setTwo: {
src: ['./csv3/'],
options: {
skipEmptyLines: false
}
}
}
});
// Load the custom multiTask named `papaparse` - which is defined in
// `papaparse.js` stored in the directory named `custom-grunt-tasks`.
grunt.loadTasks('./custom-grunt-tasks');
// Register and add papaparse to the default Task.
grunt.registerTask('default', [
'papaparse' // <-- This runs Targets named setOne and setTwo
// ...
]);
// `papaparse.js` allows for multiple targets to be defined, so
// you can use the colon notation to just run one Target.
// The following only runs the setTwo Target.
grunt.registerTask('processOneTarget', [
'papaparse:setTwo'
// ...
]);
};
Running the Task
The papaparse Task has been added to the taskList Array of the default Task, so it can be executed by entering the following via your CLI tool:
$ grunt
Notes
Running the example gist by entering $ grunt via your CLI will process all .csv files inside the directories named csv, csv2, and csv3.
Running $ grunt processOneTarget via your CLI will process only .csv files inside the directory named csv3.
As the papaparse.js utilizes a MultiTask you'll notice that in the papaparse Task defined in Gruntfile.js it includes two Targets. Namely setOne and setTwo.
The setOne Target src Array defines paths to two directories that should be processed. I.e. Directories ./csv/ and ./csv2. All .csv files found in these paths will be processed using the default papaparse options defined in papaparse.js as the Target does not define any custom options.
The setTwo target src Array defines a path to one directory. (I.e. ./csv3/). All .csv files found in this path will be processed using the default papaparse options defined in papaparse.js with the exception of the skipEmptyLines option as it's set to false.
You may find that simply defining one Target in Gruntfile.js with multiple paths in the src Array, without any custom options, meets your requirement. For Example:
// ...
grunt.initConfig({
// ...
papaparse: {
myTask: {
src: ['./csv/', './csv2', './csv3']
}
}
// ...
});
// ...
Hope this helps!
I have a multiple build set up with my gulp file(dev,uat etc) and I need to define certain variables in my app.js file then concat them into my main bundle file.
I dont want to have to create the edited file then concat that file. Is it possible to make the variable change then take the file changed in memory and use that in the concat?
gulp.src('_app.js').pipe(replace("'baseURL':", "'baseURL':'www.test.com'"))
.pipe(concat(['list of files','file in memory']));
You can use gulp-replace to do the actual replacement in your app.js, then use merge-stream to combine it with your other source files:
var replace = require('gulp-replace');
var merge = require('merge-stream');
var order = require('gulp-order');
gulp.task('concat', function() {
var appStream = gulp.src('app.js')
.pipe(replace(/'baseURL':/, "'baseURL':'www.test.com'"));
return merge(appStream, gulp.src(['list of files']))
.pipe(order(['app.js', '*.js']))
.pipe(concat('concatenatedFile.js'))
.pipe(gulp.dest('dist'));
});
I'm having problems with my gulp-rev-all task. Everytime I change the code, it will generate a new revision file, but leave the old one there.
Here is my gulp task:
var gulp = require('gulp');
var RevAll = require('gulp-rev-all');
gulp.task('js', function() {
var revAll = new RevAll();
return gulp.src(opt.Src + 'scripts.js')
// Add a hash to the file
.pipe(revAll.revision())
// Save the hashed css file
.pipe(gulp.dest(path.js))
// Write the manifest file
.pipe(revAll.manifestFile())
.pipe(gulp.dest(path.js + 'rev'));
});
So, this works like a charm.
It will give me a file with a rev (like: scripts.0ad8ecf1.js) and a manifest.json file.
The challange is, whenever I change my code, it will generate a new scripts.js file with a different hash and not overwrite or remove the old one. So, my folder looks like this now:
scripts.0ad8ecf1.js
scripts.7e3fa506.js
scripts.056ddda0.js
I can't seem to replace the old file for the new one.
Can anybody help me or point me in the right direction to accomplish this?
You need to delete your files with another plugin, since gulp-rev-all doesn't do this for you.
You could for example use the 'del' package (https://www.npmjs.com/package/del)
And then create a "delete Task" something like this:
var del = require('del');
/**
* Deletes all files inside the /foo/scripts/ folder
*/
gulp.task('purge:foo', function() {
return del.sync(['foo/scripts/**'], function (err, deletedFiles) {
if (err) {
console.log(err);
}
if (deletedFiles) {
console.log('Elements deleted:', deletedFiles.join(', '));
}
});
});
I've started using Gulp JS and must admit I'm finding it really useful.
One of the tasks I need to perform is zip up a collection of folders into individual zip files, one for each folder and then zip all this zipped files up into one single zip file. Using Gulp-Zip I've managed to get this far:
var modelFolders = [
'ELFH_Check',
'ELFH_DDP',
'ELFH_Free'
];
gulp.task('zipModels', function () {
for (var i = 0; i < modelFolders.length; i++) {
var model = modelFolders[i];
gulp.src('**/*', {cwd: path.join(process.cwd(), '/built_templates/' + model) })
.pipe(zip(model + '.zip'))
.pipe(gulp.dest('./built_templates'));
};
});
This works and outputs ELFH_Check.zip, ELFH_DDP.zip and ELFH_Free.zip. However, I then need to zip up these zip files into one zip file called "Templates.zip" and I've not managed to get this task to work:
// zip up model files
gulp.task('zipTemplate', ['zipModels'], function () {
gulp.src('*.zip', {cwd: path.join(process.cwd(), './built_templates/') })
.pipe(zip('Templates_.zip'))
.pipe(gulp.dest('./built_templates'));
});
Does anyone know if this is possible or what I'm doing wrong?
I saw the problem as well, and it seems to be related to the cwd option somehow. I'll investigate further.
After #OverZealous comment, I investigated further and found two issues:
As he said, you need to hint gulp to wait until the end of the dependency task (zipModels), by returning a stream from it. As you have multiple streams, you can use event-stream.merge to return a bundle stream.
The reason why the bundle zip wouldn't work, is because you cwd points to /built_templates/, and the second slash is causing some problem. To work properly, you need to remove the trailing slash, so it should be path.join(process.cwd(), '/built_templates').
IMPORTANT
Anyway, you should avoid temporary files. Gulp philosophy is to try using pipes to avoid IO. In that direction, what you want to do is to cut the intermediary dest steps, merge the streams, zip them, and finally, output them.
Something like that:
var es = require('event-stream');
var modelFolders = [
'ELFH_Check',
'ELFH_DDP',
'ELFH_Free'
];
gulp.task('zipModels', function () {
var zips = [],
modelZip;
for (var i = 0; i < modelFolders.length; i++) {
var model = modelFolders[i];
modelZip = gulp.src('**/*', {cwd: path.join(process.cwd(), '/built_templates/' + model) })
.pipe(zip(model + '.zip'));
// notice we removed the dest step and store the zip stream (still in memory)
zips.push(modelZip);
};
// we finally merge them (the zips), zip them again, and output.
return es.merge.apply(null, zips)
.pipe(zip('templates.zip'))
.pipe(gulp.dest('./'));
});
By the name of your folder (built_templates), it seems you have some other task that will generate the temporary built files. Preferably, you don't want these as well. You should pipe their streams directly to the ZIP stream, a finally, to the bundle-zip stream. By doing that, you would have a simple stream flow, with one disk read, and one disc write at the end, with no temporary files.
If you need them to be different tasks, consider having a function that will generate the stream up to the step before the gulp.dest pipe, and use this function on all subtasks.
Additionally, always try to hint your async tasks by returning a stream, a promise or receiving a callback function, and advise the end of the task.
In my gulp.js file I'm streaming all HTML files from the examples folder into the build folder.
To create the gulp task is not difficult:
var gulp = require('gulp');
gulp.task('examples', function() {
return gulp.src('./examples/*.html')
.pipe(gulp.dest('./build'));
});
But I can't figure out how retrieve the file names found (and processed) in the task, or I can't find the right plugin.
I'm not sure how you want to use the file names, but one of these should help:
If you just want to see the names, you can use something like gulp-debug, which lists the details of the vinyl file. Insert this anywhere you want a list, like so:
var gulp = require('gulp'),
debug = require('gulp-debug');
gulp.task('examples', function() {
return gulp.src('./examples/*.html')
.pipe(debug())
.pipe(gulp.dest('./build'));
});
Another option is gulp-filelog, which I haven't used, but sounds similar (it might be a bit cleaner).
Another options is gulp-filesize, which outputs both the file and it's size.
If you want more control, you can use something like gulp-tap, which lets you provide your own function and look at the files in the pipe.
I found this plugin to be doing what I was expecting: gulp-using
Simple usage example: Search all files in project with .jsx extension
gulp.task('reactify', function(){
gulp.src(['../**/*.jsx'])
.pipe(using({}));
....
});
Output:
[gulp] Using gulpfile /app/build/gulpfile.js
[gulp] Starting 'reactify'...
[gulp] Finished 'reactify' after 2.92 ms
[gulp] Using file /app/staging/web/content/view/logon.jsx
[gulp] Using file /app/staging/web/content/view/components/rauth.jsx
Here is another simple way.
var es, log, logFile;
es = require('event-stream');
log = require('gulp-util').log;
logFile = function(es) {
return es.map(function(file, cb) {
log(file.path);
return cb(null, file);
});
};
gulp.task("do", function() {
return gulp.src('./examples/*.html')
.pipe(logFile(es))
.pipe(gulp.dest('./build'));
});
You can use the gulp-filenames module to get the array of paths.
You can even group them by namespaces:
var filenames = require("gulp-filenames");
gulp.src("./src/*.coffee")
.pipe(filenames("coffeescript"))
.pipe(gulp.dest("./dist"));
gulp.src("./src/*.js")
.pipe(filenames("javascript"))
.pipe(gulp.dest("./dist"));
filenames.get("coffeescript") // ["a.coffee","b.coffee"]
// Do Something With it
For my case gulp-ignore was perfect.
As option you may pass a function there:
function condition(file) {
// do whatever with file.path
// return boolean true if needed to exclude file
}
And the task would look like this:
var gulpIgnore = require('gulp-ignore');
gulp.task('task', function() {
gulp.src('./**/*.js')
.pipe(gulpIgnore.exclude(condition))
.pipe(gulp.dest('./dist/'));
});
If you want to use #OverZealous' answer (https://stackoverflow.com/a/21806974/1019307) in Typescript, you need to import instead of require:
import * as debug from 'gulp-debug';
...
return gulp.src('./examples/*.html')
.pipe(debug({title: 'example src:'}))
.pipe(gulp.dest('./build'));
(I also added a title).