validate CSV files in Grunt build - csv

Question

How can I validate CSV files (encoding, headline, delimiter, column count) in my Grunt build? I had a look at CSVLint but neither get it to work, nor know how to include it in Grunt.
Edit: PapaParse looks promising, but has no Grunt integration either.

Answer 1

Although grunt integration for PapaParse does not exist its API can be utilized by configuring a custom Function Task inside your Gruntfile.js.
Install papaparse via npm
Firstly, cd to your project directory, install papaparse via npm and add it to the devDependencies section of your projects package.json. To do this run the following command via your CLI tool:
$ npm i -D papaparse
Gruntfile.js
The following gist shows how to configure a custom Function Task named validateCSV in your Gruntfile.js.
module.exports = function(grunt) {
// Requirements
var fs = require('fs');
var Papa = require('papaparse');
// Other project configuration tasks.
grunt.initConfig({
// ...
});
/**
* Register a custom Function task to validate .csv files using Papa Parse.
*/
grunt.registerTask('validateCSV', 'Lint .csv files via Papa Parse', function() {
var glob = './csv/*.csv'; // <-- Note: Edit glob pattern as required.
var success = true;
// Create an Array of all .csv files using the glob pattern provided.
var csvFiles = grunt.file.expand(glob).map(function(file) {
return file;
});
// Report if no .csv files were found and return early.
if (csvFiles.length === 0) {
grunt.log.write('No .csv files were found');
return;
}
// Loop over each .csv file in the csvFiles Array.
csvFiles.forEach(function(csvFile) {
// Read the contents of the .csv file.
var csvString = fs.readFileSync(csvFile, {
encoding: 'utf8'
});
// Parse the .csv contents via Papa Parse.
var papa = Papa.parse(csvString, {
delimiter: ',',
newline: '',
quoteChar: '"',
header: true,
skipEmptyLines: true
// For additional config options visit:
// http://papaparse.com/docs#config
});
// Basic error and success logging.
if (papa.errors.length > 0) {
grunt.log.error('Error(s) in file: '['red'] + csvFile['red']);
// Report each error for a single .csv file.
// For additional Papa Parse errors visit:
// http://papaparse.com/docs#errors
papa.errors.forEach(function(error) {
grunt.log.write('\n type: ' + error.type);
grunt.log.write('\n code: ' + error.code);
grunt.log.write('\n message: ' + error.message);
grunt.log.write('\n row: ' + error.row + '\n\n');
});
// Indicate that a .csv file failed validation.
success = false;
} else {
grunt.log.ok('No errors found in file: ' + csvFile);
}
});
// If errors are found in any of the .csv files this will
// prevent subsequent defined tasks from being processed.
if (!success) {
grunt.fail.warn('Errors(s) were found when validating .csv files');
}
});
// Register the custom Function task.
grunt.registerTask('default', [
'validateCSV'
// ...
]);
};
Notes
The following line of code (taken from the Gruntfile.js above) that reads:
var glob = './csv/*.csv';
... will need to be changed/edited according to your project requirements. Currently the globbing pattern assumes all .csv files reside inside a folder named csv.
You may also need to set the config options as per your requirements.
The custom Function Task also includes some basic error and success reporting that will be logged to the CLI.
Running the Task
To run the grunt task simply execute the following via your CLI tool:
$ grunt validateCSV
EDIT: Updated Answer (based on the following comment...)
Would it also be possible to "configure" the task from within the
grunt.initConfig()? For example linting different CSV directories?
To achieve this you can create a separate Javascript module that exports a Registered MutliTask.
Lets call it papaparse.js and save it to a directory named custom-grunt-tasks which resides in the same top level directory as your Gruntfile.js
Note: This .js file and directory name can be any name that you prefer, however you will need to updated the references inside Gruntfile.js.
papaparse.js
module.exports = function(grunt) {
'use strict';
// Requirements
var fs = require('fs');
var Papa = require('papaparse');
grunt.registerMultiTask('papaparse', 'Misc Tasks', function() {
// Default options. These are used when no options are
// provided via the initConfig({...}) papaparse task.
var options = this.options({
quotes: false,
delimiter: ',',
newline: '',
quoteChar: '"',
header: true,
skipEmptyLines: true
});
// Loop over each path provided via the src array.
this.data.src.forEach(function(dir) {
// Append a forward slash If a directory path
// provided does not end in with one.
if (dir.slice(-1) !== '/') {
dir += '/';
}
// Generate the globbin pattern.
var glob = [dir, '*.csv'].join('');
// Create an Array of all .csv files using the glob pattern.
var csvFiles = grunt.file.expand(glob).map(function(file) {
return file;
});
// Report if no .csv files were found and return early.
if (csvFiles.length === 0) {
grunt.log.write(
'>> No .csv files found using the globbing '['yellow'] +
'pattern: '['yellow'] + glob['yellow']
);
return;
}
// Loop over each .csv file in the csvFiles Array.
csvFiles.forEach(function(csvFile) {
var success = true;
// Read the contents of the .csv file.
var csvString = fs.readFileSync(csvFile, {
encoding: 'utf8'
});
// Parse the .csv contents via Papa Parse.
var papa = Papa.parse(csvString, options);
// Basic error and success logging.
if (papa.errors.length > 0) {
grunt.log.error('Error(s) in file: '['red'] + csvFile['red']);
// Report each error for a single .csv file.
// For additional Papa Parse errors visit:
// http://papaparse.com/docs#errors
papa.errors.forEach(function(error) {
grunt.log.write('\n type: ' + error.type);
grunt.log.write('\n code: ' + error.code);
grunt.log.write('\n message: ' + error.message);
grunt.log.write('\n row: ' + error.row + '\n\n');
});
// Indicate that a .csv file failed validation.
success = false;
} else {
grunt.log.ok('No errors found in file: ' + csvFile);
}
// If errors are found in any of the .csv files this will prevent
// subsequent files and defined tasks from being processed.
if (!success) {
grunt.fail.warn('Errors(s) found when validating .csv files');
}
});
});
});
};
Gruntfile.js
Your Gruntfile.js can then be configured something like this:
module.exports = function(grunt) {
grunt.initConfig({
// ...
papaparse: {
setOne: {
src: ['./csv/', './csv2']
},
setTwo: {
src: ['./csv3/'],
options: {
skipEmptyLines: false
}
}
}
});
// Load the custom multiTask named `papaparse` - which is defined in
// `papaparse.js` stored in the directory named `custom-grunt-tasks`.
grunt.loadTasks('./custom-grunt-tasks');
// Register and add papaparse to the default Task.
grunt.registerTask('default', [
'papaparse' // <-- This runs Targets named setOne and setTwo
// ...
]);
// `papaparse.js` allows for multiple targets to be defined, so
// you can use the colon notation to just run one Target.
// The following only runs the setTwo Target.
grunt.registerTask('processOneTarget', [
'papaparse:setTwo'
// ...
]);
};
Running the Task
The papaparse Task has been added to the taskList Array of the default Task, so it can be executed by entering the following via your CLI tool:
$ grunt
Notes
Running the example gist by entering $ grunt via your CLI will process all .csv files inside the directories named csv, csv2, and csv3.
Running $ grunt processOneTarget via your CLI will process only .csv files inside the directory named csv3.
As the papaparse.js utilizes a MultiTask you'll notice that in the papaparse Task defined in Gruntfile.js it includes two Targets. Namely setOne and setTwo.
The setOne Target src Array defines paths to two directories that should be processed. I.e. Directories ./csv/ and ./csv2. All .csv files found in these paths will be processed using the default papaparse options defined in papaparse.js as the Target does not define any custom options.
The setTwo target src Array defines a path to one directory. (I.e. ./csv3/). All .csv files found in this path will be processed using the default papaparse options defined in papaparse.js with the exception of the skipEmptyLines option as it's set to false.
You may find that simply defining one Target in Gruntfile.js with multiple paths in the src Array, without any custom options, meets your requirement. For Example:
// ...
grunt.initConfig({
// ...
papaparse: {
myTask: {
src: ['./csv/', './csv2', './csv3']
}
}
// ...
});
// ...
Hope this helps!