MY script pulls data from Bigquery and saves it in google sheets but every time my script is triggered the sheet gets cleared and values are updated from the start date to the end date
Is there a possibility to append and only update few days of data where there is a data discrepancy rather than clearing and updating the whole sheet?
my script :
/**
* Runs a BigQuery query and logs the results in a spreadsheet.
*/
function runQuery() {
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'xxxxx';
var request = {
query: 'SELECT date, month, channel_grouping, device_category ' +
'FROM `xxxxx` where date between "2020-01-01" and "2020-01-03" order by date ;',
useLegacySql : false
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
// Get all the rows of results.
var rows = queryResults.rows;
while (queryResults.pageToken) {
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId, {
pageToken: queryResults.pageToken
});
rows = rows.concat(queryResults.rows);
}
if (rows) {
var spreadsheet = SpreadsheetApp.openById('1yOhjzprF2FXt3dM-FktVyVWNW1fCBhRO63f-QK-y3CQ');
var sheet = spreadsheet.getActiveSheet();
sheet.clear();
// Append the headers.
var headers = queryResults.schema.fields.map(function(field) {
return field.name;
});
sheet.appendRow(headers);
// Append the results.
var data = new Array(rows.length);
for (var i = 0; i < rows.length; i++) {
var cols = rows[i].f;
data[i] = new Array(cols.length);
for (var j = 0; j < cols.length; j++) {
data[i][j] = cols[j].v;
}
}
sheet.getRange(2, 1, rows.length, cols.length).setValues(data);
Logger.log('Results spreadsheet created: %s',
spreadsheet.getUrl());
} else {
Logger.log('No rows returned.');
}
}
Related
I am trying to get data from Gmail body using GAS. To be specific, I get an email with a table content; I am trying to copy the table from gmail and write it to google sheet for my further analysis. Below is a sample email I get:
The output I am expecting in Google sheets:
UPDATE: I was able to make some modifications to the code I had by referring to Insert table from gmail to google spreadsheet by google script
Here's how the email body and output looks like now.
Email:
GSheet Output:
The issue occurs with merged cells in table. The code does not generate output as how it appears in the gmail body. Is there any workaround for this?
Final code:
var SEARCH_QUERY = "SearchKey";
function getEmailss_(q) {
var emails = [];
var threads = GmailApp.search(q);
if (threads.length == 0) {
console.log("No threads found that match the search query: " + q);
}
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
var arrStr = msgs[j].getBody()
.replace(/<\/tr>/gm, '[/tr]')
.replace(/<\/td>/gm, '[/td]')
.replace(/<.*?>/g, '\n')
.replace(/^\s*\n/gm, '')
.replace(/^\s*/gm, '')
.replace(/\s*\n/gm, '\n')
.split("[/tr]");
if (arrStr.length == 1) {
console.log("No data found in thread: " + threads[i].getFirstMessageSubject());
}
var line = [];
for (var i = 0; i < arrStr.length - 1; i++) {
line = arrStr[i].split("[/td]");
line.length -= 1;
emails.push(line);
}
}
}
if (emails.length == 0) {
console.log("No emails found that match the search query: " + q);
}
return convert2ArrayToRectangular_(emails);
}
function convert2ArrayToRectangular_(array2d)
{
// get max width
var res = [];
var w = 0;
for (var i = 0; i < array2d.length; i++)
{
if (array2d[i].length > w) {w = array2d[i].length;}
}
var row = [];
for (var i = 0; i < array2d.length; i++)
{
row = array2d[i];
if(array2d[i].length < w)
{
for (var ii = array2d[i].length; ii < w; ii++)
{
row.push('');
}
}
res.push(row);
}
return res;
}
function appendData_(sheet, array2d) {
var h = array2d.length;
var l = array2d[0].length;
sheet.getRange(1, 1, h, l).setValues(array2d);
}
function saveEmailsss() {
var array2d = getEmailss_(SEARCH_QUERY);
if (array2d) {
appendData_(SpreadsheetApp.getActive().getSheetByName('Sheet1'), convert2ArrayToRectangular_(array2d));
}
markArchivedAsRead();
}
function markArchivedAsRead() {
var threads = GmailApp.search('label:inbox is:unread to:me subject:importnumberlist');
GmailApp.markThreadsRead(threads);
};
As another approach, how about using Sheets API? When Sheets API is used, the HTML table can be parsed by including the merged cells. The sample script is as follows.
Sample script:
This script uses Sheets API. Please enable Sheets API at Advanced Google services.
var SEARCH_QUERY = "SearchKey";
function getEmailss_(q, sheetName) {
var emails = [];
var threads = GmailApp.search(q);
if (threads.length == 0) {
console.log("No threads found that match the search query: " + q);
}
var tables = [];
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
var arrStr = msgs[j].getBody();
var table = arrStr.match(/<table[\s\S\w]+?<\/table>/);
if (table) {
tables.push(table[0]);
}
}
}
if (emails.length == 0) {
console.log("No emails found that match the search query: " + q);
}
if (tables.length == 0) {
console.log("No tables.");
return
};
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName(sheetName);
var requests = [{ pasteData: { html: true, data: tables.join(""), coordinate: { sheetId: sheet.getSheetId() } } }];
Sheets.Spreadsheets.batchUpdate({ requests }, ss.getId());
}
function saveEmailsss() {
var sheetName = "Sheet1"; // Please set your sheet name.
getEmailss_(SEARCH_QUERY, sheetName);
markArchivedAsRead();
}
When this script is run, the HTML table included in the email message is retrieved and put to the active Spreadsheet.
References:
Method: spreadsheets.batchUpdate
PasteDataRequest
I have a program that filters and updates data from an existing sheet.
The program works as follows:
1. Find and filter out the required value
2. Enter data in [Adjustment] column then update to database in Record sheet.
I tried to try but my program doesn't seem to work.
I tried to edit the program code but when run it will affect the other columns and the [adjustment] column value is entered wrong.
This is my link program
function Searchold(){
var ss = SpreadsheetApp.getActiveSpreadsheet ();
var shtRecords = ss. getSheetByName ("RECORD");
var shtForm = ss. getSheetByName ("TEST") ;
var records = shtRecords. getDataRange () . getValues ();
var sField = shtForm. getRange ("A3").getValue ();
var sValue = shtForm.getRange ("A6").getValue();
var sCol = records [0].lastIndexOf(sField);
var results = records.filter(function(e){return sValue == e[sCol] });
if(results.length==0){SpreadsheetApp.getUi().alert("not found values");}
else{
shtForm.getRange(9,1,results.length,results[0].length).setValues(results);
}
}
function Updatenew(){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var shtRecords = ss.getSheetByName("RECORD");
var shtForm = ss.getSheetByName("TEST");
var LastRow = shtForm.getRange("A8").getNextDataCell(SpreadsheetApp.Direction.DOWN).getLastRow();
var newData = shtForm.getRange(9,1,LastRow -1,7).getValues();
for(var i =0; i<newData.length;i++){
var oldData= shtRecords.getDataRange().getValues();
for(var j= 0;j<oldData.length;j++){
if(newData[i][0] ==oldData[j][0]){
var newData2 = [newData[i]];
shtRecords.getRange(j + 1,1,1,newData2[0].length).setValues(newData2);
}
}
}
}
Can you help me with the update program? Sincerely thank you
Modification points:
When I saw your showing script of Updatenew, I think that each row of var oldData = shtRecords.getDataRange().getValues() is used in each loop of for (var i = 0; i < newData.length; i++) {}. By this, each row is overwritten by each row of newData. By this, all searched rows in "RECORD" sheet are the same value. I thought that this might be the reason for your issue.
var oldData = shtRecords.getDataRange().getValues(); can be used one call.
In order to avoid this issue by modifying your script, as one of several methods, how about the following modification?
From:
for (var i = 0; i < newData.length; i++) {
var oldData = shtRecords.getDataRange().getValues();
for (var j = 0; j < oldData.length; j++) {
if (newData[i][0] == oldData[j][0]) {
var newData2 = [newData[i]];
shtRecords.getRange(j + 1, 1, 1, newData2[0].length).setValues(newData2);
}
}
}
To:
var oldData = shtRecords.getDataRange().getValues();
for (var j = 0; j < oldData.length; j++) {
for (var i = 0; i < newData.length; i++) {
if (newData[0][0] == oldData[j][0]) {
var newData2 = newData.splice(0, 1);
shtRecords.getRange(j + 1, 1, 1, newData2[0].length).setValues(newData2);
break;
}
}
}
Note:
At the above modification, setValues is used in a loop. In this case, the process cost becomes high. If you want to reduce the process cost of the script, how about using Sheets API? When Sheets API is used, how about the following modification? Please enable Sheets API at Advanced Google services.
To
var temp = newData.slice();
var data = shtRecords.getDataRange().getValues().reduce((ar, r, i) => {
if (temp[0][0] == r[0]) {
var t = temp.splice(0, 1);
t[0][2] = Utilities.formatDate(t[0][2], Session.getScriptTimeZone(), "dd/MM/yyyy");
t[0][4] = Utilities.formatDate(t[0][4], Session.getScriptTimeZone(), "dd/MM/yyyy");
ar.push({ range: `'RECORD'!A${i + 1}`, values: t });
}
return ar;
}, []);
Sheets.Spreadsheets.Values.batchUpdate({ data, valueInputOption: "USER_ENTERED" }, ss.getId());
I received the following error on a script that we run time-based.
Exception: Bandwidth quota exceeded: https://app.enzyme.finance/api/vault-performance?vault=0x95fca2e84556443c1bc0c6416ee17be0e6844cd0¤cy=eur&network=ethereum. Limit the data transfer speed.
I have no clue which quota I'm exceeding. So how fixing this issue is quite a mystery.
In the code below I'm doing the following:
Delete any existing triggers deleteTriggers()
Schedule a trigger via function on time: 00:00 scheduledTrigger(0,0)
Fetch data from URL and parse that data into Google Sheets function_Triggered()
In run this code time-driven day timer at: 11 pm to midnight.
The code below:
function setTrigger() {
deleteTriggers();
scheduledTrigger(0,0);
}
function scheduledTrigger(hours,minutes){
var today_D = new Date();
var year = today_D.getFullYear();
var month = today_D.getMonth();
var day = today_D.getDate();
pars = [year,month,day,hours,minutes];
var scheduled_D = new Date(...pars);
scheduled_D.setDate(scheduled_D.getDate() + 1);
var hours_remain=Math.abs(scheduled_D - today_D) / 36e5;
ScriptApp.newTrigger("function_Triggered")
.timeBased()
.after(hours_remain * 60 * 60 * 1000)
.create()
}
function deleteTriggers() {
var triggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < triggers.length; i++) {
if ( triggers[i].getHandlerFunction() == "function_Triggered") {
ScriptApp.deleteTrigger(triggers[i]);
}
}
}
function function_Triggered() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var raw = ss.getSheetByName('raw');
var current = ss.getSheetByName('nav.current');
var database = ss.getSheetByName('nav.database');
var url = "https://app.enzyme.finance/api/vault-performance?vault=0x95fca2e84556443c1bc0c6416ee17be0e6844cd0¤cy=eur&network=ethereum";
var response = UrlFetchApp.fetch(url); // get feed
var dataAll = JSON.parse(response.getContentText());
var rows = [Object.keys(dataAll)]; // Retrieve headers.
var temp = [];
for (var i = 0; i < rows[0].length; i++) {
temp.push(dataAll[rows[0][i]]); // Retrieve values.
}
rows.push(temp);
raw.getRange(1,1,rows.length,rows[0].length).setValues(rows); // Put values to Spreadsheet.
// count rows to snap
var current_rows = current.getLastRow();
var database_rows = database.getLastRow() + 1;
var rows_new = current.getRange("A2:B" + current_rows).getValues();
// snap rows, can run this on a trigger to be timed
database.getRange("A" + database_rows + ":B" + database_rows).setValues(rows_new);
}
From my previous question here, I think it is the problem to my server-script function that queries the data from BigQuery. Following are my code to queries HOD email base on requester email address.
//To fetch requester's HOD email address
function BigQueryGetEmail(emailRequester){
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'xxxxxx-xxx-stg';
query = '#standardSQL \n SELECT SUPV_EMAIL_A FROM `xxxxx-xxx-stg.xxxxx.vw_HOD_email` WHERE EMP_EMAIL_A = "' + emailRequester + '";';
var request = {
query: query,
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
// Get all the rows of results.
var rows = queryResults.rows;
while (queryResults.pageToken) {
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId, {
pageToken: queryResults.pageToken
});
rows = rows.concat(queryResults.rows);
}
// Append the results.
var data = new Array(rows.length);
for (var i = 0; i < rows.length; i++) {
var cols = rows[i].f;
data[i] = new Array(cols.length);
for (var j = 0; j < cols.length; j++) {
data[i][j] = cols[j].v;
}
}
console.log('HOD email: ' + data);
return data.toString();
}
The weird thing is that when I console.log data it return the HOD email address. But the function won't return the data result when ever I call the BigQueryGetEmail function. Following is the caller function:
function getHOD(){
var ownerEmail = app.pages.Main.descendants.TextBox1.value;
var HODemailfield = app.pages.Main.descendants.TextBox2.value;
function successHandler(email){
//assign email value to widget;
HODemailfield = email;
}
google.script.run.withSuccessHandler(successHandler).BigQueryGetEmail(ownerEmail);
}
The widget TextBox2 doesn't reflect any data nor error which is I believe it is because of the BigQueryGetEmail function doesn't return any data.
Can anyone guide me on the correct way how to return the query result when I call the BigQueryGetEmail function? had tried several ways but still stuck.
I look on the internet to see how can I import bigquery data inside google spreadsheet.
I found this appscript sample, but it doesn'twork API are not at the same level, and I do not find how to query with API2 or API#beta1 in appscript.
function runQuery() {
var ss = SpreadsheetApp.getActive();
var range = ss.getRangeByName('query');
var query = range.getCell(1, 1).getValue();
//var results = bigquery.query(query);
var header = ss.getRangeByName('header');
header.clearContent();
var output = ss.getRangeByName('output');
output.clearContent();
for (var i = 0; i < results.fields.length; i++) {
var field = results.fields[i];
header.getCell(1, 1 + i).setValue(field.id);
}
for (var i = 0; i < results.rows.length; i++) {
var row = results.rows[i].f;
for (var j = 0; j < row.length; ++j) {
output.getCell(1 + i, 1 + j).setValue(row[j].v);
}
}
}
Thanks in advance for your ideas,
GQ
UPDATE: We just added a new BigQuery + Apps Script Tutorial that should walk you through the answer to this question here: https://developers.google.com/apps-script/articles/bigquery_tutorial
#GQuery: We've very recently updated AppsScript to have access to the latest BigQuery API version (v2). Here's a simple example to get started, will display results in the AppScript log. We are working on an update to the AppScript/BigQuery documentation.
function runQuery() {
var projectId = 'YOUR PROJECT';
var sql = 'select word, word_count from publicdata:samples.shakespeare limit 100';
var queryResults;
// Run the query
try {
queryResults = BigQuery.Jobs.query(projectId, sql);
}
catch (err) {
Logger.log(err);
return;
}
// Loop until successful job completion
while (queryResults.getJobComplete() == false) {
try {
queryResults = BigQuery.Jobs.getQueryResults(projectId, queryResults.getJobReference().getJobId());
}
catch (err) {
Logger.log(err);
return;
}
}
var tableRows = queryResults.getRows();
for (var i = 0; i < tableRows.length; i++) {
var rowString = '';
var cols = tableRows[i].getF();
for (var j = 0; j < cols.length; j++) {
rowString += cols[j].getV() + '\t';
}
Logger.log(rowString);
I dont have the reputation to comment on hurricaneditka16. Therefore, I posted this answer:
This line
queryResults = BigQuery.Jobs.query(projectId, sql);
Should be replaced by
.query(
resource,
projectId);
Resource is a slight transformation on the sql you used before. Try this transformation and it will work.
function getResource(sql) {
var resource = '{"query": "sql"}'
resource = resource.replace('sql', sql);
return resource
}