Gulp control output based on data from an earlier pipe - gulp

I have been trying to find a way with gulp to only write out certain files based on yaml data I am collecting in the pipe. I have been able to see the file's data, but not able to get the output I expect.
In this task, I am collecting a glob of markdown files, and passing them into a pipe that reads the yaml using gulp-data, and then adds some other data to it. I then pipe it through Swig.
I'm trying to add some sort of conditional element before I pipe into gulp.dest. I found this example which got me to where I am currently.
The closest I've gotten is below:
.pipe(tap(function(file) {
if (new Date(file.data.date) >= new Date(buildRange)) {
console.log(file.path);
gulp.src(file.path)
.pipe(gulp.dest(config.paths.dest + '/underreview/'));
}
}))
What I've gotten from the console.log command is correct (it shows 2 of 50 files). But nothing gets written to the destination. If I move the gulp.dest outside of this pipe, all files get written.
I've tried using gulp-if or gulp-ignore, but have been unable to get the file.data.date into either of those modules.
Edited: Here's the complete task
module.exports = function(gulp, config, env) {
var gulpSwig = require('gulp-swig'),
swig = require('swig'),
data = require('gulp-data'),
matter = require('gray-matter'),
runSequence = require('run-sequence'),
// BrowserSync
reload = config.browserSync.reload,
_ = require('lodash'),
Path = require('path'),
requireDir = require('require-dir'),
marked = require('marked'),
readingTime = require('reading-time'),
postsData = [],
postsTags = [],
pdate = null,
buildRange = new Date(new Date().setDate(new Date().getDate()-14));
sitebuilddate = null,
through = require('through2'),
gutil = require('gulp-util'),
rename = require('gulp-rename'),
File = require('vinyl'),
$if = require('gulp-if'),
ignore = require('gulp-ignore'),
tap = require('gulp-tap');
var opts = {
defaults: {
cache: false
},
setup: function(Swig) {
Swig.setDefaults({
loader: Swig.loaders.fs(config.paths.source + '/templates')});
}
};
// Full of the compiled HTML file
function targetPathFull(path, data) {
return Path.join(Path.dirname(path), targetPath(data));
}
gulp.task('templates2:under', function() {
return gulp.src(config.paths.source + '/content/**/*.md')
.pipe(data(function(file) {
postData = [];
var matterObject = matter(String(file.contents)), // extract front matter data
type = matterObject.data.type, // page type
body = matterObject.content,
postData = matterObject.data,
moreData = requireDir(config.paths.data),
data = {},
bodySwig;
bodySwig = swig.compile(body, opts);
// Use swig to render partials first
body = bodySwig(data);
// Process markdown
if (Path.extname(file.path) === '.md') {
body = marked(body);
}
// Inherit the correct template based on type
if (type) {
var compiled = _.template(
"{% extends 'pages/${type}.html' %}{% block body %}${body}{% endblock %}"
// Always use longform until a different template for different types is needed
//"{% extends 'pages/longform.html' %}{% block body %}${body}{% endblock %}"
);
body = compiled({
"type": type,
"body": body
});
}
file.path = targetPathFull(file.path, postData);
moreData.path = targetPath(postData);
_.merge(data, postData, moreData);
data.url = data.site.domain + "/" + data.slug;
// Copy the processed body text back into the file object so Gulp can keep piping
file.contents = new Buffer(body);
return data;
}))
.pipe(gulpSwig(opts))
.pipe(tap(function(file) {
if (new Date(file.data.date) >= new Date(buildRange)) {
console.log(file.path);
gulp.src(file.path)
.pipe(gulp.dest(config.paths.dest + '/underreview/'));
}
}))
.pipe(gulp.dest(config.paths.dest + '/underreview/'));
});
}

So, possibly not the best solution, but after some re-factoring I came up with this:
.pipe(gulp.dest(config.paths.dest + '/underreview/'))
.pipe(tap(function(file) {
if (new Date(file.data.date) < new Date(buildRange)) {
console.log(file.data.path);
del(config.paths.dest + '/underreview/' + file.data.path)
}
}))
I moved the gulp-tap to after the output, and then I am deleting the file that was just written.

Related

Gulpfile end of task with two different operations

I am kind of lacking imagination on that one.
My goal is to retrieve a json object so I can run a replace string on all the files I want to translate, I have looked into a lot of translation libraries but this way is the best i can think of for my use.
Anyway my issue here is Once I got my json object, I have to run on all the files and when it is done, finish the task 'trad'.
I have done some research and tried a lot of things but there is something that I miss, something that I didn't understood about the good way to do that ?
Please help !
gulp.task('trad', gulp.series( 'createTradFile', 'copyBeforeTrad', function( done ) {
var data = require('gulp-data');
var path = require('path');
var fs = require('fs');
var replace2 = require('gulp-string-replace');
var transObj = null;
var translateAll = function()
{
var files = gulp.src(['fr/**/*.html', 'fr/**/*.js']);
for (var k in transObj)
{
if (transObj[k].ID)
{
console.log("TRAD " + transObj[k].ID + " TO " + transObj[k].LANG1);
files.pipe(replace2(new RegExp('\\+' + transObj[k].ID + '\\+', 'g'),
transObj[k].LANG1,
{'logs': {'enabled': true}}))
.pipe(chmod(755));
}
}
files.pipe(gulp.dest("fr"))
.on('end', done);
};
gulp.src('distTemp/wording.json')
.pipe(data(function(file) {
transObj = JSON.parse( fs.readFileSync('distTemp/' + path.basename(file.path)));
console.log("TRAD first part OK");
translateAll();
}));
}));
So this code will translate like I want it too, but the task does not end :
[16:38:34] The following tasks did not complete: trad, <anonymous>
[16:38:34] Did you forget to signal async completion?
So, after a bit of research I found this ( almost crappy ) solution, which do the trick ( please answer if you hava a better solution )
var transObj = null;
gulp.task("retrieveTradObject", function(){
var data = require('gulp-data');
var path = require('path');
var fs = require('fs');
return gulp.src('distTemp/wording.json')
.pipe(data(function(file) {
transObj = JSON.parse( fs.readFileSync('distTemp/' + path.basename(file.path)));
console.log("TRAD first part OK");
}));
});
gulp.task('trad', gulp.series( 'createTradFile', 'copyBeforeTrad', 'retrieveTradObject', function( done ) {
var replace2 = require('gulp-string-replace');
var files = gulp.src(['fr/**/*.html', 'fr/**/*.js']);
for (var k in transObj)
{
if (transObj[k].ID)
{
console.log("TRAD " + transObj[k].ID + " TO " + transObj[k].LANG1);
files = files.pipe(replace2(new RegExp('\\+' + transObj[k].ID + '\\+', 'g'),
transObj[k].LANG1,
{'logs': {'enabled': true}}))
.pipe(chmod(755));
}
}
files.pipe(gulp.dest("fr"));
return files;
}));
So main idea here was to separate the two promises into task ( mainly for a better understanding of the code for later ) and then to do the files = files.pipe( ... ) Which is explained here : How to create repeating pipe in gulp?
Hope this can help !
I'm not sure I understand the question 100% so I'll take the dv's, but are talking about something like gulp-run-sequence?
You can do all sorts of tasking stuff like this
var gulp = require('gulp');
//webp images for optimization on some browsers
const webp = require('gulp-webp');
//responsive images!
var responsive = require('gulp-responsive-images');
//gulp delete for cleaning
var del = require('del');
//run sequence to make sure each gulp command completes in the right order.
var runSequence = require('run-sequence');
// =======================================================================//
// ! Default and bulk tasks //
// =======================================================================//
//default runs when the user types 'gulp' into CLI
//first clean is ran, then webp, then the rest are ran async.
//If you want something ran after, you can add something like 'example'
gulp.task('default',function(callback){
runSequence('clean','webp',['responsive-jpg','responsive-webp','copy-data','copy-sw'],'example'),callback
});
// =======================================================================//
// Images and fonts //
// =======================================================================//
gulp.task('responsive-jpg',function(){
gulp.src('src/images/*')
.pipe(responsive({
'*.jpg':[
{width:1600, suffix: '_large_1x', quality:40},
{width:800, suffix: '_medium_1x', quality:70},
{width:550, suffix: '_small_1x', quality:100}
]
}))
.pipe(gulp.dest('build/images'));
});
gulp.task('responsive-webp',function(){
gulp.src('src/images/*')
.pipe(responsive({
'*.webp':[
{width:1600, suffix: '_large_1x', quality:40},
{width:800, suffix: '_medium_1x', quality:70},
{width:550, suffix: '_small_1x', quality:80}
]
}))
.pipe(gulp.dest('build/images'));
});
gulp.task('webp', () =>
gulp.src('src/images/*.jpg')
.pipe(webp())
.pipe(gulp.dest('src/images'))
);
gulp.task('copy-data', function () {
gulp.src('./src/data/*.json')
.pipe(gulp.dest('./build/data'));
});
gulp.task('copy-sw', function () {
gulp.src('./src/sw.js')
.pipe(gulp.dest('./build/'));
});
In my example here, I clear out old files, then I convert any images that need to be converted to webp, then I async the tasks that can be run together. You can do this in any arrangement you need. You could create then a gulp task that even points to two gulp run sequence tasks to double down on the effectiveness.

Gulp task to replace all Root-Relative URLs with Document-Relative URLs

There are 4 distinct URL types;
Absolute http://www.example.com/images/icons.png
Document-Relative ../images/icons.png
Root-Relative /images/icons.png
Protocol-Relative //www.example.com/images/icons.png
I have a large static file site (html, css, js) which is built with Jigsaw. This framework takes PHP templates and compiles them into static HTML. I am also using Gulp tasks to compile assets (sass, js..etc).
Using Jigsaw's build process I can either have the site built with full Absolute paths/urls (http://example.com/path-to/page) or Root-Relative (/path-to/page).
This is great but now the client wants the site to use Document-Relative as they are now hosting the site in a subdirectory with a URL pointing to that subdirectory.
E.g. http://example.com would point to http://xx.server.com/hosted-files/directory
My issue is that Jigsaw doesn't allow for Document-Relative URLs. Is there a gulp/node script I can use to convert all references (image sources, links, css paths..etc)? Or is there another solution (e.g. using .htacccess)?
TLDR;
I need to replace any Absolute or Root-Relative references in multiple HTML files and directories with Document-Relative paths and URLs. Or is there another solution (e.g. using .htacccess)?
I have managed to solve my own issue with what I feel is a "hacky" fix.
I've basically created a custom gulp plugin replaces URLs/paths..etc with Document-Relative paths.
gulpfile.js - relative-urls task runs after all other tasks have completed.
const relative = require('./tasks/document-relative');
gulp.task('relative-urls', function() {
return gulp.src('build/**/*.html')
.pipe( relative({
directory: 'build',
url: 'http://localhost:8000',
}) )
.pipe( gulp.dest('build') );
});
./tasks/document-relative.js - plugin
'use strict';
const fs = require('fs');
const PluginError = require('plugin-error');
const through = require('through2');
const PLUGIN_NAME = 'document-relative';
let count = 0;
module.exports = function(options) {
// Remove slashes from beginning and end of string
const strip_slashes = (string) => {
return string ? string.replace(/^\/|\/$/g, '') : null;
}
// Users options object
options = options || {};
// Cleanup options
const base_dir = strip_slashes(options.directory);
const url = strip_slashes(options.url) + '/';
return through({
objectMode: true,
writable: true,
readable: true
},
function(file, enc, callback) {
count++;
// Check for null file
if (file.isNull()) {
return callback(null, file);
}
if (file.isStream()) {
this.emit('error', new PluginError(PLUGIN_NAME, 'Stream not supported!'));
return callback(null, file);
}
if (file.isBuffer()) {
// Get contents of this file
let html = file.contents.toString(enc);
// This files full path (/home/usr/project/build/page/example/index.html)
const path = file.path;
// Path task was run from (/home/usr/project/)
const cwd = file.cwd+( base_dir ? '/'+base_dir : '' );
// Project specific path (/page/example/index.html)
const relative = path.replace(cwd, '');
// Get array of directories ['page', 'example', 'index.html']
let paths = strip_slashes(relative).split('/');
// Remove last item ['page', 'example']
paths.pop();
// Add ../ for nth number of paths in array
let rel_path = paths.length === 0 ? '' : ('../'.repeat(paths.length));
// Replace dom attributes (e.g. href="/page/example")
html = html.replace( /(?:(?!="\/\/)="\/)/g, '="'+rel_path );
// Replace inline background (e.g. background: url('/image/something.jpg'))
html = html.replace( /url\(\'\//g, 'url(\''+rel_path );
html = html.replace( /url\('\//g, 'url(''+rel_path );
// If user defined URL, match and remove
if (url && url.length) {
html = html.replace( new RegExp(url, 'g'), rel_path );
}
// Overwrite file
fs.writeFileSync(file.path, html, {
encoding: enc,
flag:'w'
});
return callback();
}
});
};
This basically opens all .html files in my build folder, calculates how many paths deep each file is (/folder1/folder2/index.html) and replaces any instances of url (http://localhost:8000) with ../ repeated for the number of paths calculated.
Node has path.relative.
Read Levi Coles' own answer to understand where url and directory come from.
const path = require("path");
// create a regular expression from your url property.
const domain_expression = new RegExp(url);
// Once you have an offending 'href' you can do this.
// - Here 'href' is Absolute, but this same code would work
// with Root-Relative paths too.
const href = "http://localhost:8000/images/icons.png";
const file_index = href.lastIndexOf("/") + 1;
const file_component = href.substring(file_index);
const root_relative = href.replace(domain_expression, "");
const relative_href = path.relative(`${directory}/${root_relative}`, directory);
const _href = relative_href + file_component;
// _href = ../../icons.png

Imported generated JSON in JSX causes Webpack build loop

I've got a small postcss plugin I've made that generates a JSON file off a colors.css variable file during webpack build.
My postcss plugin
const fs = require('fs');
const postcss = require('postcss');
const capitalize = (string) => string.charAt(0).toUpperCase() + string.slice(1);
const getPropName = (string) => {
let name = clean(string.split('-'));
name.shift();
for(let k = 1; k < name.length; k++){ //start at 1 to skip 'color' prefix
name[k] = capitalize(name[k].toString());
}
return name.join('');
};
const clean = (array) => {
let i = array.length;
while(i--){
if (!array[i]) {
array.splice(i, 1);
i++;
}
}
return array;
};
module.exports = postcss.plugin('cssobject', (files, filters, options) =>
(css) => {
options = options || {
destination: ''
};
// Processing code will be added here
const getVariable = (variable) => {
let result;
css.walkRules((rules) => {
rules.walkDecls((decl) => {
const pointer = variable.replace('var(', '').replace(')','');
if(!decl.prop.match(pointer)) return;
result = decl.value;
});
});
return result;
};
css.walkRules((rules) => { //hooks into CSS stream
let i = files.length;
let cssObject = {};
while (i--) {
if(!rules.source.input.from.match(files[i])) return; //scrubs against requested files
rules.walkDecls((decl) => {
let j = filters.length;
while(j--){
if(!decl.prop.match(filters[j])) return; //scrubs against requested rules
let prop = getPropName(decl.prop);
cssObject[prop] = (decl.value.match('var'))? getVariable(decl.value) : decl.value;
}
});
}
if (options.destination) {
fs.writeFile(options.destination, JSON.stringify(cssObject), 'utf8');
}
});
}
);
I'm then importing this JSON file into a react component JSX file to then parse JSON data into a visual guide of project's used colors under AA and AAA requirements... anywho
The problem I'm having is my webpack-dev-server keeps re-building over and over again cause it thinks a change has been made to the JSX file, when in fact it's only ever a change to the JSON file being imported.
Is there a standard way of importing generated files in to a JSX without causing infinite build loops?
I've already tried having the JSON file be saved well outside of the webpack dev's watch location, and still build loop remains.
Thanks in advance!
you can change you file's timestamp, the webpack will not build after you change your file
const now = Date.now() / 1000;
const lastModifyTime = now - 11;
const lastAccessTime = now - 11;
fs.utimesSync(jsonPath, lastModifyTime, lastAccessTime);
Have a try, hope to help you.

gulp-replace if content of files match regex

I have a folder of HTML files that contain a comment at the top with metadata. I would like to run one gulp-replace operation if the metadata matches one regex, and another gulp-replace operation if it doesn't match, then continue on with the rest of the tasks pipeline. If tried various iterations using gulp-if but it always results in "TypeError: undefined is not a function" errors
import gulp from 'gulp';
import plugins from 'gulp-load-plugins';
const $ = plugins();
function preprocess() {
var template_data = new RegExp('<!-- template_language:(\\w+)? -->\n', 'i');
var handlebars = new RegExp('<!-- template_language:handlebars -->', 'i');
var primaryColor = new RegExp('#dc002d', 'gi');
var mailchimpColorTag = '*|PRIMARY_COLOR|*';
var handlebarsColorTag = '{{PRIMARY_COLOR}}';
var replaceCondition = function (file) {
return file.contents.toString().match(handlebars);
}
return gulp.src('dist/**/*.html')
.pipe($.if(
replaceCondition,
$.replace(primaryColor, handlebarsColorTag),
$.replace(primaryColor, mailchimpColorTag)
))
.pipe($.replace, template_data, '')
.pipe(gulp.dest('dist'));
}
What's the most efficient way to go about this?
gulp-filter was the answer. Whereas gulp-if can be used to decide whether a particular operation should be applied to the whole stream, gulp-filter can be used to decide which files in a stream an operation should be applied to.
import gulp from 'gulp';
import plugins from 'gulp-load-plugins';
const $ = plugins();
function preprocess() {
var template_language = new RegExp('<!-- template_language:(\\w+)? -->\n', 'i');
var handlebars = 'handlebars';
var primaryColor = new RegExp('#dc002d', 'gi');
var handlebarsColorTag = '{{PRIMARY_COLOR}}';
var handlebarsCondition = function (file) {
var match = file.contents.toString().match(template_language);
return (match && match[1] == handlebars);
}
var handlebarsFilter = $.filter(handlebarsCondition, {restore: true});
var mailchimpColorTag = '*|PRIMARY_COLOR|*';
var mailchimpCondition = function (file) {
return !handlebarsCondition(file);
}
var mailchimpFilter = $.filter(mailchimpCondition, {restore: true});
return gulp.src('dist/**/*.html')
.pipe(handlebarsFilter)
.pipe($.replace(primaryColor, handlebarsColorTag))
.pipe($.debug({title: 'Applying ' + handlebarsColorTag}))
.pipe(handlebarsFilter.restore)
.pipe(mailchimpFilter)
.pipe($.replace(primaryColor, mailchimpColorTag))
.pipe($.debug({title: 'Applying ' + mailchimpColorTag}))
.pipe(mailchimpFilter.restore)
.pipe($.replace(template_language, ''))
.pipe(gulp.dest('dist'));
}

Reference Google Spreadsheet (CSV) in Jekyll Data

I am managing a website displaying a lot of tabular data (language stuff) and running on Jekyll. I really like to display content based on a CSV file stored in the _data folder of Jekyll.
I would like to be able to edit / add / remove content from this CSV directly on Google and then reference it to Jekyll (like a shortcut or something that sync the CSV content from Google to my static folder).
Which way would be the simplest to reference an external file (either in the _data folder or directly in my templace). I can find the CSV file with this kind of link but downloading it every time is a hassle (https://docs.google.com/spreadsheets/d//export?format=csv).
How can Jekyll understand data from external stored file (maybe in javascript ?).
Thank you.
Getting datas from google docs is becoming harder ;-(
I've tried with jquery.ajax but I met the CORS limitation.
Then I found tabletop and it works !
go to your google spreadsheet and File > Publish to the web > Start publishing
note the publish url
download tabletop script and save it to eg: js/tabletop.js
put a link at the bottom of your _includes/header.html eg
<script src="`{{ site.baseurl }}`/js/tabletop.js"></script>
in a data.html page put
---
title: csv to json
layout: page
---
<div id="csvDatas"></div>
you can now get your datas with a js/script.js file that you've also included at the very end of you _includes/footer.html
var csvParse = function() {
// put you document url here
var sharedDocUrl = 'https://docs.google.com/spreadsheets/d/1Rk9RMD6mcH-jPA321lFTKmZsHebIkeHx0tTU0TWQYE8/pubhtml'
// can also be only the ID
// var sharedDocUrl = '1Rk9RMD6mcH-jPA321lFTKmZsHebIkeHx0tTU0TWQYE8'
var targetDiv = 'csvDatas';
// holds datas at a closure level
// this then can be accessed by closure's functions
var dataObj;
function showInfo(data, tabletop) {
dataObj = data;
var table = generateTable();
var target = document.getElementById(targetDiv);
target.appendChild(table);
}
function generateTable(){
var table = document.createElement("table");
var head = generateTableHeader();
table.appendChild(head);
var body = generateTableBody();
table.appendChild(body);
return table;
}
function generateTableHeader(){
var d = dataObj[0];
var tHead = document.createElement("thead");
var colHeader = [];
$.each(d, function( index, value){
console.log(index + ' : ' + value);
colHeader.push(index);
});
var row = generateRow(colHeader, 'th');
tHead.appendChild(row);
return tHead;
}
// this can be factorized with generateTableHeader
function generateTableBody(){
var tBody = document.createElement("tbody");
$.each(dataObj, function( index, value ){
var rowVals = [];
$.each(value, function(colnum, colval){
rowVals.push(colval);
});
var row = generateRow(rowVals);
tBody.appendChild(row);
});
return tBody;
}
function generateRow(headersArray, cellTag){
cellTag = typeof cellTag !== 'undefined' ? cellTag : 'td';
var row = document.createElement("tr");
$.each(headersArray, function( index, value){
if( value != "rowNumber"){
var cell = document.createElement(cellTag);
var cellText = document.createTextNode(value);
cell.appendChild(cellText);
row.appendChild(cell);
}
});
return row;
}
return {
init: function() {
if( $('#' + targetDiv).length ){
Tabletop.init( { key: sharedDocUrl ,
callback: showInfo,
simpleSheet: true } );
}else{
console.log('Not the good page to parse csv datas');
}
}
};
}();
$( document ).ready(function() {
csvParse.init();
});