I have a folder of HTML files that contain a comment at the top with metadata. I would like to run one gulp-replace operation if the metadata matches one regex, and another gulp-replace operation if it doesn't match, then continue on with the rest of the tasks pipeline. If tried various iterations using gulp-if but it always results in "TypeError: undefined is not a function" errors
import gulp from 'gulp';
import plugins from 'gulp-load-plugins';
const $ = plugins();
function preprocess() {
var template_data = new RegExp('<!-- template_language:(\\w+)? -->\n', 'i');
var handlebars = new RegExp('<!-- template_language:handlebars -->', 'i');
var primaryColor = new RegExp('#dc002d', 'gi');
var mailchimpColorTag = '*|PRIMARY_COLOR|*';
var handlebarsColorTag = '{{PRIMARY_COLOR}}';
var replaceCondition = function (file) {
return file.contents.toString().match(handlebars);
}
return gulp.src('dist/**/*.html')
.pipe($.if(
replaceCondition,
$.replace(primaryColor, handlebarsColorTag),
$.replace(primaryColor, mailchimpColorTag)
))
.pipe($.replace, template_data, '')
.pipe(gulp.dest('dist'));
}
What's the most efficient way to go about this?
gulp-filter was the answer. Whereas gulp-if can be used to decide whether a particular operation should be applied to the whole stream, gulp-filter can be used to decide which files in a stream an operation should be applied to.
import gulp from 'gulp';
import plugins from 'gulp-load-plugins';
const $ = plugins();
function preprocess() {
var template_language = new RegExp('<!-- template_language:(\\w+)? -->\n', 'i');
var handlebars = 'handlebars';
var primaryColor = new RegExp('#dc002d', 'gi');
var handlebarsColorTag = '{{PRIMARY_COLOR}}';
var handlebarsCondition = function (file) {
var match = file.contents.toString().match(template_language);
return (match && match[1] == handlebars);
}
var handlebarsFilter = $.filter(handlebarsCondition, {restore: true});
var mailchimpColorTag = '*|PRIMARY_COLOR|*';
var mailchimpCondition = function (file) {
return !handlebarsCondition(file);
}
var mailchimpFilter = $.filter(mailchimpCondition, {restore: true});
return gulp.src('dist/**/*.html')
.pipe(handlebarsFilter)
.pipe($.replace(primaryColor, handlebarsColorTag))
.pipe($.debug({title: 'Applying ' + handlebarsColorTag}))
.pipe(handlebarsFilter.restore)
.pipe(mailchimpFilter)
.pipe($.replace(primaryColor, mailchimpColorTag))
.pipe($.debug({title: 'Applying ' + mailchimpColorTag}))
.pipe(mailchimpFilter.restore)
.pipe($.replace(template_language, ''))
.pipe(gulp.dest('dist'));
}
Related
I'm attempting to use the lastmod function in conjunction with cheerio (or gulp-cheerio) to pull a value from a meta tag from each file in the stream to populate this lastmod entry.
var gulp = require('gulp');
var cheerio = require('cheerio');
var sitemap = require('gulp-sitemap');
gulp.task('sitemap', function() {
return gulp.src(['/prod/**/*.html',])
.pipe(sitemap({
siteUrl: 'https://www.somewhere.com',
lastmod: function(file) {
var $ = cheerio.load(body);
var lastmodValue = $('head > meta[name="dc.date"]').attr('content');
return lastmodValue.toString().trim();
}
}))
.pipe(gulp.dest('/prod'));
});
This works fine if I hardcode a var before the return and comment out the cheerio statements. Gulp-sitemap lastmod instructions here: https://github.com/pgilad/gulp-sitemap#lastmod
Found my issue - Here's the code that works:
.pipe(sitemap({
siteUrl: 'https://www.somewhere.com',
lastmod: function(file) {
var $ = cheerio.load(file.contents);
var lastmodValue = $('head > meta[name="dc.date"]').attr('content');
return lastmodValue.toString().trim();
}
}))
I'm seeing a similar issue as this post (gulp-concat twice the content).
However, I'm dumping the concatenated file into a different directory, so it's not pulling in the resulting concatenated file into task, yet I'm seeing the contents of each file doubling up for some reason.
My gulp file is as follows:
/* jshint node: true */
module.exports = function (gulp, options, plugins) {
var merge = require('merge-stream');
var uglify = require('gulp-uglify');
var pump = require('pump');
var gp_concat = require('gulp-concat');
var gp_rename = require('gulp-rename');
var gp_ignore = require('gulp-ignore');
var ngAnnotate = require('gulp-ng-annotate');
var paths = require('../paths');
var utils = require('../utils');
var base = [
paths.APP,
paths.ETC,
paths.DESIGN
];
gulp.task('scripts:clean', function () {
var srcOptions = {
read: false
};
var tasks = base.map(function (folder) {
return gulp.src(folder + '/**/' + paths.GENERATED_SUBPATH + '/js/**/*.js', srcOptions)
.pipe(plugins.clean({force: true}));
});
return merge(tasks);
});
gulp.task('compress', function () {
var filesToInclude = ['**/app/components/**/*.js'
];
var excludeCondition = '**/*.spec*.js'
var fileToDest = paths.GLOBAL + '/'+paths.GENERATED_SUBPATH + '/js';
return gulp.src(filesToInclude)
.pipe(gp_ignore.exclude(excludeCondition))
.pipe(ngAnnotate({add: true}))
.pipe(gp_concat('all.concat.js'))
.pipe(gulp.dest('dist'))
.pipe(gp_rename('all.min.js'))
.pipe(uglify())
.pipe(gulp.dest(fileToDest));
});
gulp.task('scripts:build', ['scripts:clean', 'compress']);
};
Can someone help me understand why the
var filesToInclude = ['**/app/components/**/*.js];
would bring in each file twice? I've checked the files and no, the files are not duplicated anywhere in there.
It seems that the issue was with the definition of the filesToInclude, with it starting out with a wildcard. Since the fileToDest puts the file in a separate target directory, but the structure is the same, the process picks up the files twice.
The situation: in my crontab, I have a job that runs a .sh script. This script in turn executes my, say, casperjs casper.js - all works fine. The script executes when I run it from the command line. The shell script, when called from the command line, executes fine. I run the job from crontab and output to a log file, I can see that it's working.
However, I never get the expected output: ie. casper_output.json, here is the complete casper.js file that runs:
var casper = require('casper').create({
waitTimeout: 10000,
stepTimeout: 10000,
verbose: true,
//debug, write out results, errors
logLevel: "debug",
loadImages: true,
loadPlugins: true,
pageSettings: {
javascriptEnabled: true,
webSecurityEnabled: false
},
onWaitTimeout: function() {
this.echo('** Wait-TimeOut **');
},
onStepTimeout: function() {
this.echo('** Step-TimeOut **');
}
});
//vars for this page
var url = 'http://urltoscrape.com';
var results = [];
var menuResults = [];
var fs = require('fs');
var terminate = function() {
this.echo("Exiting..").exit();
};
var getContent = function() {
//grab the values that we need for json file, store in empty results array
var results = [];
var URL = document.querySelectorAll('#results .event-details-link');
var title = document.querySelectorAll('#results h3');
var date = document.querySelectorAll('#results span');
var desc = document.querySelectorAll('#results p');
for (var i = 0;
//count through URL, title, date and description
i < URL.length
&& i < title.length
&& i < date.length
&& i < desc.length;
i++) {
var innerURL = URL[i].getAttribute("href");;
var innerTitle = title[i].innerText;
var innerDate = date[i].innerText;
//messy regex, TODO: make more efficient
var newDate = innerDate.replace(/\s[at].*$/g,"");
var strip = newDate.replace(/[on]/g,"");
var nows = strip.replace(/\s/g, "");
var innerDesc = desc[i].innerText;
var dept = "Dept";
//push into json, TODO: order
results.push({ URL: innerURL, title: innerTitle, date: nows, desc: innerDesc, dept: dept});
}
return results;
}
var processPage = function() {
//crude but waits for page to load
this.wait(2000, function() {
//run the get content function
results = this.evaluate(getContent);
require('utils').dump(results);
//write to file
fs.write("casper_output.json", JSON.stringify(results, null, ' '), 'w');
});
}
casper.start(url, function() {
this.waitForSelector('#results', processPage, terminate);
});
casper.run();
The shell script that calls it, casper_run.sh:
#!/bin/bash
PATH=/output/of/path/MAMP/Library/bin
PHANTOMJS_EXECUTABLE=/usr/local/bin/phantomjs ; /usr/local/bin/casperjs /Applications/AMPPS/www/test/casper.js 2>&1
And, finally, in my crontab:
* * * * * /Applications/AMPPS/www/test/casper_run.sh
Any insight on why that casper_output.json does not write from my cron job (but does when run from command line) would be so welcome. I have been researching this for far too long.
Thanks!
Cron (or rather CasperJS) probably doesn't have write permission to write into the current working directory.
You can change the working directory through PhantomJS' file system module or you can use a full path:
fs.write("/Applications/AMPPS/www/test/casper_output.json", JSON.stringify(results, null, ' '), 'w');
I am managing a website displaying a lot of tabular data (language stuff) and running on Jekyll. I really like to display content based on a CSV file stored in the _data folder of Jekyll.
I would like to be able to edit / add / remove content from this CSV directly on Google and then reference it to Jekyll (like a shortcut or something that sync the CSV content from Google to my static folder).
Which way would be the simplest to reference an external file (either in the _data folder or directly in my templace). I can find the CSV file with this kind of link but downloading it every time is a hassle (https://docs.google.com/spreadsheets/d//export?format=csv).
How can Jekyll understand data from external stored file (maybe in javascript ?).
Thank you.
Getting datas from google docs is becoming harder ;-(
I've tried with jquery.ajax but I met the CORS limitation.
Then I found tabletop and it works !
go to your google spreadsheet and File > Publish to the web > Start publishing
note the publish url
download tabletop script and save it to eg: js/tabletop.js
put a link at the bottom of your _includes/header.html eg
<script src="`{{ site.baseurl }}`/js/tabletop.js"></script>
in a data.html page put
---
title: csv to json
layout: page
---
<div id="csvDatas"></div>
you can now get your datas with a js/script.js file that you've also included at the very end of you _includes/footer.html
var csvParse = function() {
// put you document url here
var sharedDocUrl = 'https://docs.google.com/spreadsheets/d/1Rk9RMD6mcH-jPA321lFTKmZsHebIkeHx0tTU0TWQYE8/pubhtml'
// can also be only the ID
// var sharedDocUrl = '1Rk9RMD6mcH-jPA321lFTKmZsHebIkeHx0tTU0TWQYE8'
var targetDiv = 'csvDatas';
// holds datas at a closure level
// this then can be accessed by closure's functions
var dataObj;
function showInfo(data, tabletop) {
dataObj = data;
var table = generateTable();
var target = document.getElementById(targetDiv);
target.appendChild(table);
}
function generateTable(){
var table = document.createElement("table");
var head = generateTableHeader();
table.appendChild(head);
var body = generateTableBody();
table.appendChild(body);
return table;
}
function generateTableHeader(){
var d = dataObj[0];
var tHead = document.createElement("thead");
var colHeader = [];
$.each(d, function( index, value){
console.log(index + ' : ' + value);
colHeader.push(index);
});
var row = generateRow(colHeader, 'th');
tHead.appendChild(row);
return tHead;
}
// this can be factorized with generateTableHeader
function generateTableBody(){
var tBody = document.createElement("tbody");
$.each(dataObj, function( index, value ){
var rowVals = [];
$.each(value, function(colnum, colval){
rowVals.push(colval);
});
var row = generateRow(rowVals);
tBody.appendChild(row);
});
return tBody;
}
function generateRow(headersArray, cellTag){
cellTag = typeof cellTag !== 'undefined' ? cellTag : 'td';
var row = document.createElement("tr");
$.each(headersArray, function( index, value){
if( value != "rowNumber"){
var cell = document.createElement(cellTag);
var cellText = document.createTextNode(value);
cell.appendChild(cellText);
row.appendChild(cell);
}
});
return row;
}
return {
init: function() {
if( $('#' + targetDiv).length ){
Tabletop.init( { key: sharedDocUrl ,
callback: showInfo,
simpleSheet: true } );
}else{
console.log('Not the good page to parse csv datas');
}
}
};
}();
$( document ).ready(function() {
csvParse.init();
});
I'm trying to write a minify function that can be used to minifiy html, css, and js depending on file type. I would like to use the existing gulp plugins for these 3 minification processes to do the actual minification. The problem I'm having is I don't know how to call a plugin on a single vinyl file. Here is what I have so far:
var cssmin = require('gulp-cssmin');
var htmlmin = require('gulp-minify-html');
var uglify = require('gulp-uglify');
var minifiers = {
js: uglify,
css: cssmin,
html: htmlmin
};
function minify(options) {
var options = options || {};
return tap(function(file){
var fileType = file.path.split('.').pop();
options = options[fileType] || options
var minifier = minifiers[fileType];
if(!minifier)
console.error("No minifier for " + fileType + " - " + file.path);
// WHAT DO I DO HERE? This doesn't work but I want to do something similar
file.pipe(minifier(options));
});
}
I would like to be able to call the minify function like this:
gulp.src(['test.html', 'test.css', 'test.js'])
.pipe(minify());
Use gulp-filter.
var gulpFilter = require('gulp-filter');
var jsFilter = gulpFilter('**/*.js');
var cssFilter = gulpFilter('**/*.css');
var htmlFilter = gulpFilter('**/*.html');
gulp.task('default', function () {
gulp.src('assets/**')
.pipe(jsFilter)
.pipe(uglify())
.pipe(jsFilter.restore())
.pipe(cssFilter)
.pipe(cssmin())
.pipe(cssFilter.restore())
.pipe(htmlFilter)
.pipe(htmlmin())
.pipe(htmlFilter.restore())
.pipe(gulp.dest('out/'));
});
Will work for single files too but globs are more futureproof :)
SOLUTION:
I ended up using gulp-filter to solve the issue, but it was fairly tricky to get it working in a reusable way. Here is my final code:
var cssmin = require('gulp-cssmin');
var htmlmin = require('gulp-htmlmin');
var uglify = require('gulp-uglify');
var lazypipe = require('lazypipe');
function getFilter(type) {
// create a filter for the specified file type
return filter('**/*.' + type);
}
var minify = function() {
var jsFilter = getFilter('js'),
cssFilter = getFilter('css'),
htmlFilter = getFilter('html');
var min = lazypipe()
.pipe(function(){return jsFilter;})
.pipe(uglify)
.pipe(jsFilter.restore)
.pipe(function(){return cssFilter;})
.pipe(cssmin)
.pipe(cssFilter.restore)
.pipe(function(){return htmlFilter;})
.pipe(htmlmin)
.pipe(htmlFilter.restore);
return min();
};
To run gulp plugin on a single file you need to do the following:
var stream = minifier(options);
stream.once('data', function(newFile) {
file.contents = newFile.contents;
})
stream.write(file);