git log in JSON - send changelog to browser - json

In order to know what's actually been committed to a project in production, we are going to display the Git log to admins - this serve as a makeshift changelog.
I have this routine on a Node.js Express server:
router.get('/changelog/json', ac.allow('ROLE_ADMIN'), function (req, res, next) {
const k = cp.spawn('bash');
k.stdin.write('git log --pretty=format:\'{%n "commit": "%H",%n "abbreviated_commit": "%h",%n "tree": "%T",%n "abbreviated_tree": "%t",%n "parent": "%P",%n "abbreviated_parent": "%p",%n "refs": "%D",%n "encoding": "%e",%n "subject": "%s",%n "sanitized_subject_line": "%f",%n "body": "%b",%n "commit_notes": "%N",%n "verification_flag": "%G?",%n "signer": "%GS",%n "signer_key": "%GK",%n "author": {%n "name": "%aN",%n "email": "%aE",%n "date": "%aD"%n },%n "commiter": {%n "name": "%cN",%n "email": "%cE",%n "date": "%cD"%n }%n},\'\n')
k.stdin.end();
k.stdout.pipe(res);
});
this sort of works, but we don't actually get a JSON array, we just get comma separate JSON strings.
I got this info form here:
https://gist.github.com/varemenos/e95c2e098e657c7688fd
https://git-scm.com/docs/pretty-formats
does anyone know how I can construct a JSON array from the stdout from the Git command?
I tried this:
router.get('/json', ac.allow('ROLE_ADMIN'), function (req, res, next) {
const p = createParser();
const k = cp.spawn('bash', [], {
cwd: global.cdtProjectRoot
});
const items = [];
k.stdin.write('git log --pretty=format:\'{%n "commit": "%H",%n "abbreviated_commit": "%h",%n "tree": "%T",%n "abbreviated_tree": "%t",%n "parent": "%P",%n "abbreviated_parent": "%p",%n "refs": "%D",%n "encoding": "%e",%n "subject": "%s",%n "sanitized_subject_line": "%f",%n "body": "%b",%n "commit_notes": "%N",%n "verification_flag": "%G?",%n "signer": "%GS",%n "signer_key": "%GK",%n "author": {%n "name": "%aN",%n "email": "%aE",%n "date": "%aD"%n },%n "commiter": {%n "name": "%cN",%n "email": "%cE",%n "date": "%cD"%n }%n},\'\n')
k.stdin.end();
k.stdout.pipe(p).on('data', function (d) {
// d would be a parsed JSON object
items.push(d);
})
.once('error', next)
.once('end', function () {
res.json({
success: items
})
})
});
My parser transform works, because I use it in another project, so it's something about the format of the JSON coming from the stdout that's causing the problem - the 'data' event handler never sees any data.

This worked for me, I had to improve the git log format option:
router.get('/json', ac.allow('ROLE_ADMIN'), function (req, res, next) {
const p = createParser();
const k = cp.spawn('bash');
let items = [];
k.stdin.write(`git log -300 --pretty=format:'{"commit":"%H","sanitized_subject_line":"%f","commit_notes":"%N","author":"%aN","date":"%aD"}'`);
k.stdin.end('\n');
k.stdout.pipe(p).on('data', function (d) {
items.push(JSON.parse(d));
})
.once('error', next)
.once('end', function () {
res.json({
success: items
})
})
});
createParser looks like this:
const stream = require("stream");
exports.createParser = function () {
let lastLineData = '';
let strm = new stream.Transform({
objectMode: true,
transform: function (chunk, encoding, cb) {
let _this = this;
let data = String(chunk);
if (lastLineData) {
data = lastLineData + data;
}
let lines = data.split('\n');
lastLineData = lines.splice(lines.length - 1, 1)[0];
lines.forEach(function (l) {
l && _this.push(l);
});
cb();
},
flush: function (cb) {
if (lastLineData) {
this.push(lastLineData);
}
lastLineData = '';
cb();
}
});
return strm;
};
this might barf if there are certain characters in the git log fields, like Author, but I think the built-in field "sanitized_subject_line" will save our skin a bit when it comes to removing bad chars from commit messages.

Related

Struggling to get data from AWS lambda JSON

I'm working on a lambda project and getting data from an API inside the function which looks like this
{ "Title": "300", "Year": "2006", "Rated": "R", "Released": "09 Mar 2007", "Runtime": "117 min", "Genre": "Action, Fantasy, War", "Director": "Zack Snyder", "Writer": "Zack Snyder (screenplay), Kurt Johnstad (screenplay), Michael B. Gordon (screenplay), Frank Miller (graphic novel), Lynn Varley (graphic novel)", "Actors": "Gerard Butler, Lena Headey, Dominic West, David Wenham", "Plot": "King Leonidas of Sparta and a force of 300 men fight the Persians at Thermopylae in 480 B.C.", "Language": "English", "Country": "USA, Canada, Bulgaria", "Awards": "17 wins & 45 nominations.", "Poster": "https://m.media-amazon.com/images/M/MV5BMjc4OTc0ODgwNV5BMl5BanBnXkFtZTcwNjM1ODE0MQ##._V1_SX300.jpg", "Ratings": [ { "Source": "Internet Movie Database", "Value": "7.7/10" }, { "Source": "Rotten Tomatoes", "Value": "60%" }, { "Source": "Metacritic", "Value": "52/100" } ], "Metascore": "52", "imdbRating": "7.7", "imdbVotes": "691,774", "imdbID": "tt0416449", "Type": "movie", "DVD": "31 Jul 2007", "BoxOffice": "$210,500,000", "Production": "Warner Bros. Pictures", "Website": "http://300themovie.warnerbros.com/", "Response": "True" }
I've tried dot notation, indexing all sorts but no matter what I try, the console log just comes out with
2019-06-14T18:33:46.394Z ecc5d247-6475-464e-8dd7-bec310d98c4a INFO undefined
Has anyone else had the same issue before with lambda and lex?
Thanks
const https = require('https')
let url = "http://www.omdbapi.com/?t=300&r&apikey=3ecc35a"
let reply;
const http = require('http')
let test;
http.get(url, res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
console.log(body);
reply = JSON.parse(body);
});
});
This currently produces a perfectly good JSON in the console but it's impossible to actually extract anything. I've tried reply.Year, reply["Year"], reply.[0].Year almost any combination I can think off.
Full Code
'use strict';
'use fetch';
// Close dialog with the customer, reporting fulfillmentState of Failed or Fulfilled ("Thanks, your pizza will arrive in 20 minutes")
function close(sessionAttributes, fulfillmentState, message) {
return {
sessionAttributes,
dialogAction: {
type: 'Close',
fulfillmentState,
message,
},
};
}
// --------------- Events -----------------------
function dispatch(intentRequest, callback) {
console.log(`request received for userId=${intentRequest.userId}, intentName=${intentRequest.currentIntent.name}`);
const sessionAttributes = intentRequest.sessionAttributes;
//const film = intentRequest.currentIntent.film;
const film = intentRequest.currentIntent.slots.film.toString();
console.log(intentRequest.currentIntent.slots.film.toString());
const https = require('https')
let url = "http://www.omdbapi.com/?t=300&r&apikey=3ecc35a"
let reply;
const http = require('http')
let test;
http.get(url, res => {
res.setEncoding("utf8");
let body = "";
res.on("data", data => {
body += data;
});
res.on("end", () => {
console.log(body);
reply = JSON.parse(body);
});
});
//const rating = reply.imdbRating;
console.log(reply);
callback(close(sessionAttributes, 'Fulfilled',
{'contentType': 'PlainText', 'content': `The film ${film} has a rating of `}));
}
// --------------- Main handler -----------------------
// Route the incoming request based on intent.
// The JSON body of the request is provided in the event slot.
exports.handler = (event, context, callback) => {
try {
dispatch(event,
(response) => {
callback(null, response);
});
} catch (err) {
callback(err);
}
};
I tried to reproduce the issue with that code and got the following error
Response:
{
"errorType": "TypeError",
"errorMessage": "Cannot read property 'name' of undefined",
"trace": [
"TypeError: Cannot read property 'name' of undefined",
" at dispatch (/var/task/index.js:20:112)",
" at Runtime.exports.handler (/var/task/index.js:65:9)",
" at Runtime.handleOnce (/var/runtime/Runtime.js:63:25)",
" at process._tickCallback (internal/process/next_tick.js:68:7)"
]
}
Line 20 of index.js for me is:
console.log(`request received for userId=${intentRequest.userId}, intentName=${intentRequest.currentIntent.name}`);
However when using the test event in the question event.currentIntent doesn't exist and the name property of the event object doesn't exist either.
If I remove part of the console.log statement and change it to reference the Title attribute which exists in the test event I get:
console.log(`request received for Title=${intentRequest.Title}`);
INFO request received for Title=300
Seems like the function's code is referencing attributes fine but the function's just not receiving it's expected event objects.
HTH
-James

GULP Create JSON file from another JSON file

I am trying to create a local lang file that will be formatted as json. I have the following navigation in json format below. And I need to create a new JSON file using GULP to create a lang file (see below)
"lists": [
{
"title": "Application Intel",
"items": [
{
"title": "Analytics Dashboard",
"href": "intel_analytics_dashboard.html"
},
{
"title": "Marketing Dashboard",
"href": "intel_marketing_dashboard.html"
},
{
"title": "CEO Dashboard",
"href": "intel_ceo_dashboard.html"
},
{
"title": "Introduction",
"href": "intel_introduction.html"
},
{
"title": "Build Notes",
"href": "intel_build_notes.html",
"text": "Build Notes",
"span": {
"class": "",
"text": "v{{version}}"
}
}
]
}
I need to create a file that looks like the following json:
"nav": {
"application_intel": "Application Intel",
"intel_analytics_dashboard": "Analytics Dashboard",
"intel_marketing_dashboard": "Marketing Dashboard",
"intel_ceo_dashboard": "CEO Dashboard",
"intel_introduction": "Introduction",
"intel_build_notes": "Build Notes",
}
Whats the best way to go about this?
Here is solution.
Let's say you have nav.json file inside src and you want to change its shape and place it into dest directory. You can achieve this from within gulpfile.js
const { src, dest } = require("gulp");
const through = require("through2");
// gulp task
function json() {
return src("src/nav.json")
.pipe(
through.obj((file, enc, cb) => {
// get content of json file
const rawJSON = file.contents.toString();
// parse raw json into javscript object
const parsed = JSON.parse(rawJSON);
// transform json into desired shape
const transformed = transformJson(parsed);
// make string from javascript obj
const stringified = JSON.stringify(transformed, null, 2);
// make bufer from string and attach it as current file content
file.contents = Buffer.from(stringified);
// pass transformed file into next gulp pipe
cb(null, file);
})
)
.pipe(dest("dest"));
}
// transformation
function transformJson(input) {
const result = { nav: {} };
// read json field by field
Object.keys(input).forEach(topLevelKey => {
// current object
const topLevelItem = input[topLevelKey];
// in your design topLevelItems are arrays
topLevelItem.forEach(menuItem => {
if (menuItem.title) {
// make url either from item href or title
const itemUrl = makeUrl(menuItem.href || menuItem.title);
result.nav[itemUrl] = menuItem.title;
}
// prcoess children
if (menuItem.items) {
menuItem.items
.filter(child => !!child.title) // process only child items with title
.forEach(child => {
const childUrl = makeUrl(child.href || child.title);
result.nav[childUrl] = child.title;
});
}
});
});
return result;
}
// helper func
function makeUrl(href) {
return href
.toLowerCase()
.replace(/\.html$/, "")
.replace(/\s/g, "_");
}
// export for use in command line
exports.json = json;
json transformation function is bit forEachy and if you have deep nested navigation structure, maybe you should change it into something recursive

API Gateway + Lambda download CSV file

I want to do a csv download link with API Gateway + Lambda.
But there is a problem that lambda always return JSON.stringify. Is there a way to resolve this?
s-function.json
"responses": {
"default": {
"statusCode": "200",
"responseParameters": {
"method.response.header.Content-disposition": "'attachment; filename=testing.csv'"
},
"responseTemplates": {
"text/csv": ""
}
}
}
handler.js
var json2csv = require('json2csv');
module.exports.handler = function(event, context, cb) {
var fields = ['car', 'price', 'color'];
var myCars = [
{
"car": "Audi",
"price": 40000,
"color": "blue"
}, {
"car": "BMW",
"price": 35000,
"color": "black"
}, {
"car": "Porsche",
"price": 60000,
"color": "green"
}
];
var csv = json2csv({ data: myCars, fields: fields });
return cb(null, csv);
};
In the downloaded csv file.
"\"car\",\"price\",\"color\"\n\"Audi\",40000,\"blue\"\n\"BMW\",35000,\"black\"\n\"Porsche\",60000,\"green\""
Updated:
I still trying but thank you at least I have direction.
By the way, I can't find API Gateway doc about $input.body.replaceAll. replaceAll is Java function?
Finally, I resolve this by below code in Api Gateway template.
$input.body.replaceAll("\\""","").replaceAll("""","").replaceAll("\\n","
")
s-function escaped double quotes.
"responseTemplates": {
"text/csv": "$input.body.replaceAll(\"\\\\\"\"\",\"\").replaceAll(\"\"\"\",\"\").replaceAll(\"\\\\n\",\"\n\")"
}
return data:
car,price,color
Audi,40000,blue
BMW,35000,black
Porsche,60000,green
The template final replaceAll is weird. CSV don't recognize \n or \r\n, but I try copy new line in IDE and pass to code. It work and it's magical.
Serverless has changed a bit since you asked this question but if you're using the lambda_proxy method of integration, you can use a handler like:
module.exports.handler = (event, context, callback) => {
const csvRows = [
'1,"blah",123',
'2,"qwe",456'
]
const result = csvRows.reduce((prev, curr) => {
return prev + '\n' + curr
})
callback(null, {
headers: {
'Content-Type': 'text/csv',
'Content-disposition': 'attachment; filename=testing.csv'
},
body: result,
statusCode: 200
})
}
Note: I've used ES6 features so you'll want to use Node 6 or higher to directly copy-paste this example.
If you can't fix it on the Lambda function, you could probably do a replaceAll() in the API Gateway mapping template. I think this could work just to replace the escaped double quotes:
$input.body.replaceAll("\\""","")
Edit: So the swagger would be (if I got the escaping right):
"responses": {
"default": {
"statusCode": "200",
"responseParameters": {
"method.response.header.Content-disposition": "'attachment; filename=testing.csv'"
},
"responseTemplates": {
"text/csv": "$input.body.replaceAll(\"\\\"\"\",\"\")"
}
}
}

Spotify API top-tracks broken

I'm trying to get the first top-track preview url from an artist but everytime I do the search it returns a broken json. I can parse it as a string to get what I need but a json would be a lot easier. Here is my code:
const https = require('https');
var open = require('open')
function songError(){
console.log('There was some kind of error fetching your artist ;(');
}
function getTopSong(p_id){
https.get('https://api.spotify.com/v1/artists/'+p_id+'/top-tracks?country=BR', function(res){
res.on("data", function(chunk){
var json = JSON.parse(chunk.toString('utf8'));
console.log(json);
});
});
}
function getArtistID(p_name) {
https.get('https://api.spotify.com/v1/search?q='+encodeURI(p_name)+'&type=artist', function(res){
res.on("data", function(chunk) {
var json = JSON.parse(chunk.toString('utf8'));
if(json['artists']['items'][0]['id'] != undefined || json['artists']['items'][0]['id'] != null){
console.log('id: ',json['artists']['items'][0]['id']);
getTopSong(json['artists']['items'][0]['id']);
}else
{
songError();
}
});
});
}
getArtistID("rage against the machine");
There seems to be an error in line 329:
undefined:329
"available_markets" : [ "AR", "AU", "AT", "BE", "BO", "BR", "BG", "CA", "CL", "CO", "CR", "CY", "CZ", "DK", "DO", "DE", "EC", "EE", "SV", "FI", "FR", "GR", "
My question is, am I doing something wrong or is it really broken?
Thanks!
I could curl it without any problems at least:
$ curl -s 'https://api.spotify.com/v1/artists/2d0hyoQ5ynDBnkvAbJKORj/top-tracks?country=BR' | python -mjson.tool | tail
"id": "25CbtOzU8Pn17SAaXFjIR3",
"name": "Take The Power Back - Remastered",
"popularity": 58,
"preview_url": "https://p.scdn.co/mp3-preview/b44e8f96a219871587d0559970ca5dce71c891f2",
"track_number": 3,
"type": "track",
"uri": "spotify:track:25CbtOzU8Pn17SAaXFjIR3"
}
]
}
I don't know much about nodejs, but don't you need to concatenate all callbacks to res.on("data"?
https://nodejs.org/api/http.html#http_http_request_options_callback
https.get('https://api.spotify.com/v1/artists/' + p_id + '/top-tracks?country=BR', function(res) {
var body = [];
res.on("data", function(chunk) {
body.push(chunk);
});
res.on("end", function() {
var json = JSON.parse(Buffer.concat(body).toString("utf8"));
console.log(json);
});
});
If the response is long and Spotify's servers decides to send the response back chunked transfer encoding, then the nodejs http module probably splits the response up as well.

how do I get all my recent commit messages from github?

I would like to display all of my recent commit messages from github on a website. Is this possible?
To get the public events of a user, you should use the /users/:user/events endpoint (Events performed by a user):
curl https://api.github.com/users/IonicaBizau/events
This will give you back a JSON response like this:
[
{
"type": "IssueCommentEvent",
...
}
{
"id": "3349705833",
"type": "PushEvent",
"actor": {...},
"repo": {...},
"payload": {
"push_id": 868451162,
"size": 13,
"distinct_size": 1,
"ref": "refs/heads/master",
"head": "0ea1...12162",
"before": "548...d4bd",
"commits": [
{
"sha": "539...0892e",
"author": {...},
"message": "Some message",
"distinct": false,
"url": "https://api.github.com/repos/owner/repo/commits/53.....92e"
},
...
]
},
"public": true,
"created_at": "2015-11-17T11:05:04Z",
"org": {...}
},
...
]
Now, you only need to filter the response to include only the PushEvent items.
Since you want to display these events on a website, probably you want to code it in javascript. Here is an example how to do it using gh.js–an isomorphic GitHub API wrapper for JavaScript/Node.js written by me:
// Include gh.js
const GitHub = require("gh.js");
// Create the GitHub instance
var gh = new GitHub();
// Get my public events
gh.get("users/IonicaBizau/events", (err, res) => {
if (err) { return console.error(err); }
// Filter by PushEvent type
var pushEvents = res.filter(c => {
return c.type === "PushEvent";
});
// Show the date and the repo name
console.log(pushEvents.map(c => {
return "Pushed at " + c.created_at + " in " + c.repo.name;
}).join("\n"));
// => Pushed at 2015-11-17T11:05:04Z in jillix/jQuery-json-editor
// => Pushed at 2015-11-16T18:56:05Z in IonicaBizau/html-css-examples
// => Pushed at 2015-11-16T16:36:37Z in jillix/node-cb-buffer
// => Pushed at 2015-11-16T16:35:57Z in jillix/node-cb-buffer
// => Pushed at 2015-11-16T16:34:58Z in jillix/node-cb-buffer
// => Pushed at 2015-11-16T13:39:33Z in IonicaBizau/ghosty
});