Updating JSON file NodeJS - json

This is my code:
const Discord = require('discord.js');
const client = new Discord.Client();
const TOKEN = "***********";
const PREFIX = "!";
client.on("ready", function () {
console.log("Ready!");
});
client.on("message", function (message) {
if (message.author.equals(client.user)) return;
if (!message.content.startsWith(PREFIX)) return;
var args = message.content.substring(PREFIX.length).split(" ");
switch (args[0]) {
case "rules":
var _embed = new Discord.RichEmbed()
.setTitle("Ruleset")
.addField("Where is my order?", "Theres only one proper way to recive an order and help. Its a command .ticket")
.addField("Why AZATEJ is such a bitch?", "If my status is 'dont disturb' and hue is way more red than green it means I have a reason to do so, im not a dick, but i recive a shitload of messages on daily route with stupid quiestions.")
.addField("Dont ask stupid questions", "Stupid doesnt mean basic, we are up to help you but before you'll contact anyone read twice explanation documents and use a ticket.")
.setColor(0x00FFFF)
.setFooter("This message is coool !")
.setThumbnail(message.author.avatarURL);
message.channel.send(_embed);
break;
case "spotify":
var uID = message.author.id;
for (let i = 0; i < ftpr.buyers.length; i++) {
if (uID === ftpr.buyers[i].id) {
var _embed = new Discord.RichEmbed()
.setTitle("Spotify")
.addField("Username", "testsda#yahoo.com")
.addField("Password", "ithastobe8")
.setColor(0x00FFFF)
.setFooter("Sincerely, LajgaardMoneyService")
.setThumbnail(message.author.avatarURL);
message.author.send(_embed);
console.log(message.author.username + "(" + JSON.stringify(ftpr.buyers[i].id) + ") Just used the command !spotify");
break;
}
else {
message.channel.send(message.author + "You haven't got a valid subscription. This command is locked until a new one is obtained!");
break;
}
}
break;
}
});
client.on('guildMemberAdd', function(member) {
console.log("User " + member.id + " has joined the server!");
//var role = member.guild.roles.find("name", "Google!");
var myRole = member.guild.roles.find("name", "Google!");
member.addRole(myRole);
});
client.login(TOKEN);
This is the JSON file:
{
"buyers": [
{
"id": "1331499609509724162"
},
{
"id": "181336616164392960"
},
{
"id": "266389854122672128"
}
]
}
When the bot is running and im changing one of the ID's the check function in case "spotify": is still using the old id. I do not want to restart the program every time the json file updates as it should be running 24/7. I have tried const fs = require("fs"); method but it gave me this error: TypeError: Cannot read property 'buyers' of undefined json
Sincerely, Oscar

const fs = require("fs"); just loads the module. Put that at the top of your file.
To read the json file each time you need to check user IDs (inefficient, but should get things working), put this at the top of your spotify case:
case "spotify":
var yourjsonfile = fs.readFileSync("yourjsonfile.json");
var ftpr = JSON.parse(yourjsonfile);
var uID = message.author.id;
for (let i = 0; i < ftpr.buyers.length; i++) {
if (uID === ftpr.buyers[i].id) {
Again, this is very inefficient - you are reloading the file every time you need to check it, and it uses readFileSync(), which blocks until the file is read (it is better to utilize node's asynchronous features). So as the JSON file grows larger, this will run slower. But at that point you probably need a database or some other mechanism for persisting and querying your data.

Related

How to fix "Service Documents failed while accessing document" while inserting a lot of data?

This is a follow-up question derivated from How to solve error when adding big number of tables
With the code below, I get the following message when, for 500 tables. BUt it works fine for 200 for example.
Exception: Service Documents failed while accessing document with id
The error happens on line 22, inside de if body = DocumentApp.getActiveDocument().getBody();
You also have the table template id to try, but here is an image
Image Table Template
function RequirementTemplate_Copy() {
var templatedoc = DocumentApp.openById("1oJt02MfOIQPFptdWCwDpj5j-zFdO_Wrq-I48mUq9I-w");
return templatedoc.getBody().getChild(1).copy()
}
function insertSpecification_withSection(){
// Retuns a Table Template Copied from another Document
reqTableItem = RequirementTemplate_Copy();
var body = DocumentApp.getActiveDocument().getBody();
// Creates X number of separated tables from the template
for (var i = 1; i < 501; i++){
table = reqTableItem.copy().replaceText("#Title#",String(i))
body.appendTable(table);
if((i % 100) === 0) {
DocumentApp.getActiveDocument().saveAndClose();
body = DocumentApp.getActiveDocument().getBody()
}
}
}
It looks that the error message isn't related to the number of tables to be inserted because it occurs before adding the tables.
Just wait a bit an try again. If the problem persist try your code using a different account if the code runs on the second account it's very possible that you first account exceeded a limit... there are some limits to prevent abuse that aren't published and that might change without any announcement.
Using the fix suggested for the code from my answer to the previous question and changing the number for iteration limit to 1000 and 2000 works fine
The following screenshot shows the result for 1000
Here is the code used for the tests
function insertSpecification_withSection(){
startTime = new Date()
console.log("Starting Function... ");
// Retuns a Table Template Copied from another Document
reqTableItem = RequirementTemplate_Copy();
var body = DocumentApp.getActiveDocument().getBody();
// Creates X number of separated tables from the template
for (var i = 0; i < 2000; i++){
table = body.appendTable(reqTableItem.copy());
// if((i % 100) === 0) {
// DocumentApp.getActiveDocument().saveAndClose();
// }
//
}
endTime = new Date();
timeDiff = endTime - startTime;
console.log("Ending Function..."+ timeDiff + " ms");
}
function RequirementTemplate_Copy() {
//---------------------------------------------------------------------------------------------------------------------------------------------------
var ReqTableID = PropertiesService.getDocumentProperties().getProperty('ReqTableID');
try{
var templatedoc = DocumentApp.openById(ReqTableID);
} catch (error) {
DocumentApp.getUi().alert("Could not find the document. Confirm it was not deleted and that anyone have read access with the link.");
//Logger.log("Document not accessible", ReqTableID)
}
var reqTableItem = templatedoc.getChild(1).copy();
//---------------------------------------------------------------------------------------------------------------------------------------------------
return reqTableItem
}
function setReqTableID(){
PropertiesService.getDocumentProperties().setProperty('ReqTableID', '1NS9nOb3qEBrqkcAQ3H83OhTJ4fxeySOQx7yM4vKSFu0')
}

Getting response with NodeJS request module

I just started using the twitch kraken api and I have a few questions.
Whenever I attempt to get a JSON object there is no response. I am attempting to run this function through Amazon AWS Lambda, and don't have access to a console.
In the code below my callback function will always print out "SUCCESS got streamers ERROR". I am pretty certain right now the "ERROR" comes from my initial setting of result.
How come result does not get changed into the proper JSON?
I have used postman and it returns the proper thing with the query and param, and headers:
function getJSON(callback){
var result = "ERROR";
request.get(url(games[0]),function(error,response,body){
console.log("requested for url: " + url(games[0]));
var d = JSON.parse(body);
result = d.streams[0];//.channel.display_name;
// for(var i = 0; i < limit; i++){
// streamers.push(d.streams[i].channel.display_name)
// }
streamers.push(result);
});
if (streamers.length < 0){
callback("ERROR");
}else{
callback("SUCCESS got streamers " + result);
}
}
function url(game){
return {
url: "https://api.twitch.tv/kraken/streams/",//twitchlimit,
qs : {
'game' : 'overwatch',
'limit' : 2
},
headers: {
'Client-ID': clientID,
'Accept': 'application/json',
'Accept-Charset': 'utf-8',
}
};
}
I think your streamers code
if (streamers.length < 0){
callback("ERROR");
}else{
callback("SUCCESS got streamers " + result);
}
should be included in the request callback because currently it's not waiting for the request to finish, it's just carrying on so therefore the value of result will not change. Also the array length cannot be less than 0 so it will always go to the else and say "SUCCESS got streamers ERROR"
Thank you guys for the suggestions. I did have a few oversights and attempted to fix them.
I have implemented you suggestions and it seems to have worked a bit. I ended up putting the json.parse into a try/catch block, and moved the if/else statements inside the getJSON method. However, now I don't get any output.
This is how I am invoking the getJSON method:
function handleGameResponse(intent,session,callback){
//gets the game
var game = intent.slots.game.value;
if (!games.includes(game)){
var speechOutput = "You asked for: " + intent.slots.game.value;
//var speechOutput = "You asked for: " + games[game] + " That game is currently not an option. These are your current options: " + arrayToString(games)
var repromptText = "Please ask one from the current options.";
var header = "Invalid Game";
}else {
getJSON(function(data){
if(data !== "ERROR"){
var speechOutput = data; //capitalizeFirst(game) + " top three streamers are: " + arrayToString(streamers) + '.';
var repromptText = "Do you want to hear more about games?";
var header = capitalizeFirst(game);
}else{
var speechOutput = "I'm sorry, something went wrong and I could not get the streamers.";
}
//speechOutput = data;
});
//speechOutput = games[0] + " games[0], game= " + game; //this executes so the getJSON isn't executing
}
var shouldEndSession = false;
callback(session.attributes,buildSpeechletResponse(header,speechOutput,repromptText,shouldEndSession));
}
Does the above execute the same way? As in the shouldEndSession and callback execute before the getJSON has time to give a response?
For ref, this is the getJSON method now:
function getJSON(callback){
var result = "ERROR";
request.get(url(games[0]),function(error,response,body){
try{
var d = JSON.parse(body);
} catch (err){
callback("Sorry, something seems to have malfunctioned while getting the streamers");
}
result = d.streams[0].channel.display_name;
// for(var i = 0; i < limit; i++){
// streamers.push(d.streams[i].channel.display_name)
// }
streamers.push(result);
if (streamers.length <= 0){
callback("ERROR");
}else{
callback("SUCCESS got streamers " + result);
}
});
}

How to parse unstructured JSON file in Node?

I have to parse a JSON file that has many objects but no structure to the file. It looks like this:
{"obj1": "john"}
{"obj2": "sally"}
{"obj3": "veronica"}
Each object is on on it's own there is no container. So when I open the file and try to iterate through it I get the error Unexpected token { in JSON
Aside from wrapping the objects in an array and then manually going through the whole file to add commas, how can I parse this?
If it's really one-object-per-line, it's fairly straightforward to take the string, break it into lines, and JSON.parse each line:
const str =
'{"obj1": "john"}\n' +
'{"obj2": "sally"}\n' +
'{"obj3": "veronica"}';
const array = str.split(/[\r\n]+/)
.map(entry => JSON.parse(entry));
console.log(array);
...but that's assuming it really is one object per line.
If you're reading the file, you don't have to start out with all in one string as above; just read line by line as Kevin B points out.
(Since you're using Node, I've happily used ES2015+ features above...)
If you assume each line of the input file is complete, self-standing JSON, then a split-into-lines-then-parse-each strategy works well.
But even if the data isn't limited to a single line, not all is lost. You can heuristically parse the file. It isn't hyper-efficient, but except for very large files you'll probably never know the difference:
function incrementallyParseJSON(filepath) {
var lines = fs.readFileSync(filepath)
.toString()
.split(/\n/g);
var result = [];
var [start, stop] = [0, 1];
while (stop <= lines.length) {
try {
var part = JSON.parse(lines.slice(start, stop).join('\n'));
result.push(part);
[start, stop] = [stop, stop+1];
} catch (e) {
stop += 1;
}
}
return result;
}
So if your file is:
{"obj1": "john"}
{"obj2": "sally",
"more": "other"}
{"obj3": "veronica"}
"something"
12
The result will be:
[ { obj1: 'john' },
{ obj2: 'sally', more: 'other' },
{ obj3: 'veronica' },
'something',
12 ]
Example:
function incrementallyParseJSON(str) {
var lines = str.split(/\n/g);
var result = [];
var [start, stop] = [0, 1];
while (stop <= lines.length) {
try {
var part = JSON.parse(lines.slice(start, stop).join('\n'));
result.push(part);
[start, stop] = [stop, stop+1];
} catch (e) {
stop += 1;
}
}
return result;
}
var str =
'{"obj1": "john"}\n' +
'{"obj2": "sally",\n' +
' "more": "other"}\n' +
'{"obj3": "veronica"}\n' +
'"something"\n' +
'12';
console.log(incrementallyParseJSON(str));

Kettle PDI - Modified JavaScript - Json function not available

I'm using Kettle PDI 6.0 running on Windows Server 2012. I need to use the Modified Java Script Value to handle on Json object. I try something like this:
var jsondata = JSON.parse(result);
And get that:
"TypeError: Cannot find function parse in object test value test value test value test value test value test value test value test value test value test value. (script#3)"
I already try to looking for a solution on google, but not looks like that. I think that can be something wrong with my installation.
Note: I already try to use the command:
import java.util.*;
But that command is not recognized (Is not marked in bold).
I get:
missing ; before statement (script#2)
Maybe the Java functions not available.
I made my own function to resolve the problem. I will post here to help who has the same problem. If anyone want to help to solve the initial problem, I am still interested.
You can paste the code bellow on your "Modified Java Script Value" step after receive the Json response from service or get that on file. Note that you need to change the name of variables that you want to find on Json.
Result field is a Json Value.
//Script here
function findInArray(myValue, myArray){
var myResult='';
if(myArray.indexOf(myValue) > -1){
myResult = true;
} else {
myResult = false;
}
return myResult;
}
function getAttributeValue(Atribute, Object)
{
start = indexOf(Object,Atribute);
for (i= start; i < Object.length; i++)
{
if (substr(Object,i,1) == ":")
{
start_value = i+1;
break;
}
}
for (i= start_value; i < Object.length; i++)
{
end_value = i;
if (substr(Object,i,1) == ",")
{
break;
}
}
AttributeValue = replace(substr(Object, start_value, end_value-start_value),'"','');
if (indexOf(AttributeValue, "null") >= 0)
{
AttributeValue = null;
}
return AttributeValue ;
}
// Recupera Status
if (findInArray("status",result))
{
var status = getAttributeValue("status", result);
}
else
{
var status = "";
}
// Recupera _ID
if (findInArray("_id",result))
{
var mandrill_id = getAttributeValue("_id", result);
}
else
{
var mandrill_id = "";
}
// Recupera reject_reason
if (findInArray("reject_reason",result))
{
var reject_reason = replace(getAttributeValue("reject_reason", result),"}","");
}
else
{
var reject_reason = "";
}
yes, the parse json function is not available on the ex4 ecmascript of js rhino engine build in kettle, but you can handle json in kettle using eval.
var resultObj = eval('('+result+')');
//now you can iterate the foo elements of result original json
for(i=0;i< resultObj.length;i++){
Alert('foo number ' + i ' value = ' + resultObj[i].foo);
}
This is not javascript for the browser so eval is perfectly safe.

Export multiple html tables to Excel

I've scavenged the inter web for answers and though I found some, they were mostly incomplete or not working.
What I'm trying to do is: I have a info page which displays information about a customer or server (or something else), this information is displayed in a table, sometimes multiple tables (I sometimes create my own table for some of the data and use Html.Grid(Model.list) to create tables for the rest of the data stored in lists, all on 1 page).
I found this website which is an awesome: http://www.excelmashup.com/ and does exactly what I want for 1 table, though I need this for multiple tables (they must all be in the same Excel file). I know I can create multiple files (1 for each table) but this is not the desired output.
So I kept on searching and I found a post on stackoverflow: Export multiple HTML tables to Excel with JavaScript function
This seemed promising so I tried using it but the code had some minor errors which I tried to fix:
var tableToExcel = (function () {
var uri = 'data:application/vnd.ms-excel;base64,'
, template = '<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns="http://www.w3.org/TR/REC-html40"><head><!--[if gte mso 9]><xml><x:ExcelWorkbook><x:ExcelWorksheets><x:ExcelWorksheet><x:Name>{worksheet}</x:Name><x:WorksheetOptions><x:DisplayGridlines/></x:WorksheetOptions></x:ExcelWorksheet></x:ExcelWorksheets></x:ExcelWorkbook></xml><![endif]--></head><body><table>{table}</table></body></html>'
, base64 = function (s) { return window.btoa(unescape(encodeURIComponent(s))) }
, format = function (s, c) { return s.replace(/{(\w+)}/g, function (m, p) { return c[p]; }) }
return function (table, name) {
if (!table.nodeType) table = document.getElementById(table)
var ctx = { worksheet: name || 'Worksheet', table: table.innerHTML }
window.location.href = uri + base64(format(template, ctx))
}
})()
The button I use to trigger it:
<input type="button" onclick="tableToExcel('InformatieTable', 'W3C Example Table')" value="Export to Excel">
but alas to no avail (I did not know what to do with the if (!table.nodeType) table = table line so I just commented it since it seemed to do nothing special).
Now I get an error, or well not really an error but this is what it says when I try to run this code:
Resource interpreted as Document but transferred with MIME type application/vnd.ms-excel: "data:application/vnd.ms-excel;base64,PGh0bWwgeG1sbnM6bz0idXJuOnNjaGVtYXMtbW…JzZXQ9VVRGLTgiLz48L2hlYWQ+PGJvZHk+PHRhYmxlPjwvdGFibGU+PC9ib2R5PjwvaHRtbD4=".
And I get an Excel file as download in my browser but when I try to open it I get an error about the content and file extension not matching and if I would still like to open it. So if I click ok it opens a empty Excel sheet and that's it.
I am currently trying to fix that error, though i don't think it will make any difference to the content of the Excel file.
Is there anyone that can help me fix this? Or provide an other way of doing this?
I do prefer it to be run client side (so jQuery/java) instead of server side to minimize server load.
EDIT
I've found a better example of the jQuery (one that does work) on http://www.codeproject.com/Tips/755203/Export-HTML-table-to-Excel-With-CSS
This converts 1 table into an excel file which is obviously not good enough. But now I have the code to do this so I should be able to adapt it to loop trough all tables on the web page.
Also updated the code in this example to the correct version I'm using now.
I also still get the same error yet when I click on ok when trying to open the Excel file it does show me the content of the table, so I'm just ignoring that for now. anyone who has a solution for this please share.
Thanks to #Axel Richter I got my answer, he reffered me to the following question
I have adapted the code a bit so it would Take all the tables on the web page so it now looks like this:
<script type="text/javascript">
var tablesToExcel = (function () {
var uri = 'data:application/vnd.ms-excel;base64,'
, tmplWorkbookXML = '<?xml version="1.0"?><?mso-application progid="Excel.Sheet"?><Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet">'
+ '<DocumentProperties xmlns="urn:schemas-microsoft-com:office:office"><Author>Axel Richter</Author><Created>{created}</Created></DocumentProperties>'
+ '<Styles>'
+ '<Style ss:ID="Currency"><NumberFormat ss:Format="Currency"></NumberFormat></Style>'
+ '<Style ss:ID="Date"><NumberFormat ss:Format="Medium Date"></NumberFormat></Style>'
+ '</Styles>'
+ '{worksheets}</Workbook>'
, tmplWorksheetXML = '<Worksheet ss:Name="{nameWS}"><Table>{rows}</Table></Worksheet>'
, tmplCellXML = '<Cell{attributeStyleID}{attributeFormula}><Data ss:Type="{nameType}">{data}</Data></Cell>'
, base64 = function (s) { return window.btoa(unescape(encodeURIComponent(s))) }
, format = function (s, c) { return s.replace(/{(\w+)}/g, function (m, p) { return c[p]; }) }
return function (wsnames, wbname, appname) {
var ctx = "";
var workbookXML = "";
var worksheetsXML = "";
var rowsXML = "";
var tables = $('table');
for (var i = 0; i < tables.length; i++) {
for (var j = 0; j < tables[i].rows.length; j++) {
rowsXML += '<Row>'
for (var k = 0; k < tables[i].rows[j].cells.length; k++) {
var dataType = tables[i].rows[j].cells[k].getAttribute("data-type");
var dataStyle = tables[i].rows[j].cells[k].getAttribute("data-style");
var dataValue = tables[i].rows[j].cells[k].getAttribute("data-value");
dataValue = (dataValue) ? dataValue : tables[i].rows[j].cells[k].innerHTML;
var dataFormula = tables[i].rows[j].cells[k].getAttribute("data-formula");
dataFormula = (dataFormula) ? dataFormula : (appname == 'Calc' && dataType == 'DateTime') ? dataValue : null;
ctx = {
attributeStyleID: (dataStyle == 'Currency' || dataStyle == 'Date') ? ' ss:StyleID="' + dataStyle + '"' : ''
, nameType: (dataType == 'Number' || dataType == 'DateTime' || dataType == 'Boolean' || dataType == 'Error') ? dataType : 'String'
, data: (dataFormula) ? '' : dataValue.replace('<br>', '')
, attributeFormula: (dataFormula) ? ' ss:Formula="' + dataFormula + '"' : ''
};
rowsXML += format(tmplCellXML, ctx);
}
rowsXML += '</Row>'
}
ctx = { rows: rowsXML, nameWS: wsnames[i] || 'Sheet' + i };
worksheetsXML += format(tmplWorksheetXML, ctx);
rowsXML = "";
}
ctx = { created: (new Date()).getTime(), worksheets: worksheetsXML };
workbookXML = format(tmplWorkbookXML, ctx);
console.log(workbookXML);
var link = document.createElement("A");
link.href = uri + base64(workbookXML);
link.download = wbname || 'Workbook.xls';
link.target = '_blank';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
})();
</script>
so now when ever I want a page to have an option to be exported to excel i add a refference to that script and i add the following button to my page:
<button onclick="tablesToExcel(['ServerInformatie', 'Relaties'], 'VirtueleMachineInfo.xls', 'Excel')">Export to Excel</button>
so the method:
tablesToExcel(WorksheetNames, fileName, 'Excel')
Where worksheetNames is an array which needs to contain as much names (or more) as there are tables on the page. You could ofcourse chose to create the worksheet names in a different way.
And where fileName is ofcourse the name of the file you'll be downloading.
Not having it all in 1 worksheet is a shame but at least this will do for now.
Here is the code that I used to put multiple HTML tables in the same Excel sheet:
import TableExport from 'tableexport';
const tbOptions = {
formats: ["xlsx"], // (String[]), filetype(s) for the export, (default: ['xlsx', 'csv', 'txt'])
bootstrap: true, // (Boolean), style buttons using bootstrap, (default: true)
exportButtons: false, // (Boolean), automatically generate the built-in export buttons for each of the specified formats (default: true)
position: "bottom", // (top, bottom), position of the caption element relative to table, (default: 'bottom')
}
DowlandExcel = (key) => {
const table = TableExport(document.getElementById(key), tbOptions);
var exportData = table.getExportData();
var xlsxData = exportData[key].xlsx;
console.log(xlsxData); // Replace with the kind of file you want from the exportData
table.export2file(xlsxData.data, xlsxData.mimeType, xlsxData.filename, xlsxData.fileExtension, xlsxData.merges, xlsxData.RTL, xlsxData.sheetname)
}
DowlandExcelMultiTable = (keys) => {
const tables = []
const xlsxDatas = []
keys.forEach(key => {
const selector = document.getElementById(key);
if (selector) {
const table = TableExport(selector, tbOptions);
tables.push(table);
xlsxDatas.push(table.getExportData()[key].xlsx)
}
});
const mergeXlsxData = {
RTL: false,
data: [],
fileExtension: ".xlsx",
filename: 'rapor',
merges: [],
mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
sheetname: "Rapor"
}
for (let i = 0; i < xlsxDatas.length; i++) {
const xlsxData = xlsxDatas[i];
mergeXlsxData.data.push(...xlsxData.data)
xlsxData.merges = xlsxData.merges.map(merge => {
const diff = mergeXlsxData.data.length - xlsxData.data.length;
merge.e.r += diff;
merge.s.r += diff;
return merge
});
mergeXlsxData.merges.push(...xlsxData.merges)
mergeXlsxData.data.push([null]);
}
console.log(mergeXlsxData);
tables[0].export2file(mergeXlsxData.data, mergeXlsxData.mimeType, mergeXlsxData.filename, mergeXlsxData.fileExtension, mergeXlsxData.merges, mergeXlsxData.RTL, mergeXlsxData.sheetname)
}