Accessing BigQuery with Google Spreadsheet - google-apps-script

I look on the internet to see how can I import bigquery data inside google spreadsheet.
I found this appscript sample, but it doesn'twork API are not at the same level, and I do not find how to query with API2 or API#beta1 in appscript.
function runQuery() {
var ss = SpreadsheetApp.getActive();
var range = ss.getRangeByName('query');
var query = range.getCell(1, 1).getValue();
//var results = bigquery.query(query);
var header = ss.getRangeByName('header');
header.clearContent();
var output = ss.getRangeByName('output');
output.clearContent();
for (var i = 0; i < results.fields.length; i++) {
var field = results.fields[i];
header.getCell(1, 1 + i).setValue(field.id);
}
for (var i = 0; i < results.rows.length; i++) {
var row = results.rows[i].f;
for (var j = 0; j < row.length; ++j) {
output.getCell(1 + i, 1 + j).setValue(row[j].v);
}
}
}
Thanks in advance for your ideas,
GQ

UPDATE: We just added a new BigQuery + Apps Script Tutorial that should walk you through the answer to this question here: https://developers.google.com/apps-script/articles/bigquery_tutorial
#GQuery: We've very recently updated AppsScript to have access to the latest BigQuery API version (v2). Here's a simple example to get started, will display results in the AppScript log. We are working on an update to the AppScript/BigQuery documentation.
function runQuery() {
var projectId = 'YOUR PROJECT';
var sql = 'select word, word_count from publicdata:samples.shakespeare limit 100';
var queryResults;
// Run the query
try {
queryResults = BigQuery.Jobs.query(projectId, sql);
}
catch (err) {
Logger.log(err);
return;
}
// Loop until successful job completion
while (queryResults.getJobComplete() == false) {
try {
queryResults = BigQuery.Jobs.getQueryResults(projectId, queryResults.getJobReference().getJobId());
}
catch (err) {
Logger.log(err);
return;
}
}
var tableRows = queryResults.getRows();
for (var i = 0; i < tableRows.length; i++) {
var rowString = '';
var cols = tableRows[i].getF();
for (var j = 0; j < cols.length; j++) {
rowString += cols[j].getV() + '\t';
}
Logger.log(rowString);

I dont have the reputation to comment on hurricaneditka16. Therefore, I posted this answer:
This line
queryResults = BigQuery.Jobs.query(projectId, sql);
Should be replaced by
.query(
resource,
projectId);
Resource is a slight transformation on the sql you used before. Try this transformation and it will work.
function getResource(sql) {
var resource = '{"query": "sql"}'
resource = resource.replace('sql', sql);
return resource
}

Related

Is it possible to export table from Gmail body while preserving the format, to Google sheets using GAS?

I am trying to get data from Gmail body using GAS. To be specific, I get an email with a table content; I am trying to copy the table from gmail and write it to google sheet for my further analysis. Below is a sample email I get:
The output I am expecting in Google sheets:
UPDATE: I was able to make some modifications to the code I had by referring to Insert table from gmail to google spreadsheet by google script
Here's how the email body and output looks like now.
Email:
GSheet Output:
The issue occurs with merged cells in table. The code does not generate output as how it appears in the gmail body. Is there any workaround for this?
Final code:
var SEARCH_QUERY = "SearchKey";
function getEmailss_(q) {
var emails = [];
var threads = GmailApp.search(q);
if (threads.length == 0) {
console.log("No threads found that match the search query: " + q);
}
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
var arrStr = msgs[j].getBody()
.replace(/<\/tr>/gm, '[/tr]')
.replace(/<\/td>/gm, '[/td]')
.replace(/<.*?>/g, '\n')
.replace(/^\s*\n/gm, '')
.replace(/^\s*/gm, '')
.replace(/\s*\n/gm, '\n')
.split("[/tr]");
if (arrStr.length == 1) {
console.log("No data found in thread: " + threads[i].getFirstMessageSubject());
}
var line = [];
for (var i = 0; i < arrStr.length - 1; i++) {
line = arrStr[i].split("[/td]");
line.length -= 1;
emails.push(line);
}
}
}
if (emails.length == 0) {
console.log("No emails found that match the search query: " + q);
}
return convert2ArrayToRectangular_(emails);
}
function convert2ArrayToRectangular_(array2d)
{
// get max width
var res = [];
var w = 0;
for (var i = 0; i < array2d.length; i++)
{
if (array2d[i].length > w) {w = array2d[i].length;}
}
var row = [];
for (var i = 0; i < array2d.length; i++)
{
row = array2d[i];
if(array2d[i].length < w)
{
for (var ii = array2d[i].length; ii < w; ii++)
{
row.push('');
}
}
res.push(row);
}
return res;
}
function appendData_(sheet, array2d) {
var h = array2d.length;
var l = array2d[0].length;
sheet.getRange(1, 1, h, l).setValues(array2d);
}
function saveEmailsss() {
var array2d = getEmailss_(SEARCH_QUERY);
if (array2d) {
appendData_(SpreadsheetApp.getActive().getSheetByName('Sheet1'), convert2ArrayToRectangular_(array2d));
}
markArchivedAsRead();
}
function markArchivedAsRead() {
var threads = GmailApp.search('label:inbox is:unread to:me subject:importnumberlist');
GmailApp.markThreadsRead(threads);
};
As another approach, how about using Sheets API? When Sheets API is used, the HTML table can be parsed by including the merged cells. The sample script is as follows.
Sample script:
This script uses Sheets API. Please enable Sheets API at Advanced Google services.
var SEARCH_QUERY = "SearchKey";
function getEmailss_(q, sheetName) {
var emails = [];
var threads = GmailApp.search(q);
if (threads.length == 0) {
console.log("No threads found that match the search query: " + q);
}
var tables = [];
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
var arrStr = msgs[j].getBody();
var table = arrStr.match(/<table[\s\S\w]+?<\/table>/);
if (table) {
tables.push(table[0]);
}
}
}
if (emails.length == 0) {
console.log("No emails found that match the search query: " + q);
}
if (tables.length == 0) {
console.log("No tables.");
return
};
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName(sheetName);
var requests = [{ pasteData: { html: true, data: tables.join(""), coordinate: { sheetId: sheet.getSheetId() } } }];
Sheets.Spreadsheets.batchUpdate({ requests }, ss.getId());
}
function saveEmailsss() {
var sheetName = "Sheet1"; // Please set your sheet name.
getEmailss_(SEARCH_QUERY, sheetName);
markArchivedAsRead();
}
When this script is run, the HTML table included in the email message is retrieved and put to the active Spreadsheet.
References:
Method: spreadsheets.batchUpdate
PasteDataRequest

How can Google Sheets Form Update Records from Results Using Google App script?

I have a program that filters and updates data from an existing sheet.
The program works as follows:
1. Find and filter out the required value
2. Enter data in [Adjustment] column then update to database in Record sheet.
I tried to try but my program doesn't seem to work.
I tried to edit the program code but when run it will affect the other columns and the [adjustment] column value is entered wrong.
This is my link program
function Searchold(){
var ss = SpreadsheetApp.getActiveSpreadsheet ();
var shtRecords = ss. getSheetByName ("RECORD");
var shtForm = ss. getSheetByName ("TEST") ;
var records = shtRecords. getDataRange () . getValues ();
var sField = shtForm. getRange ("A3").getValue ();
var sValue = shtForm.getRange ("A6").getValue();
var sCol = records [0].lastIndexOf(sField);
var results = records.filter(function(e){return sValue == e[sCol] });
if(results.length==0){SpreadsheetApp.getUi().alert("not found values");}
else{
shtForm.getRange(9,1,results.length,results[0].length).setValues(results);
}
}
function Updatenew(){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var shtRecords = ss.getSheetByName("RECORD");
var shtForm = ss.getSheetByName("TEST");
var LastRow = shtForm.getRange("A8").getNextDataCell(SpreadsheetApp.Direction.DOWN).getLastRow();
var newData = shtForm.getRange(9,1,LastRow -1,7).getValues();
for(var i =0; i<newData.length;i++){
var oldData= shtRecords.getDataRange().getValues();
for(var j= 0;j<oldData.length;j++){
if(newData[i][0] ==oldData[j][0]){
var newData2 = [newData[i]];
shtRecords.getRange(j + 1,1,1,newData2[0].length).setValues(newData2);
}
}
}
}
Can you help me with the update program? Sincerely thank you
Modification points:
When I saw your showing script of Updatenew, I think that each row of var oldData = shtRecords.getDataRange().getValues() is used in each loop of for (var i = 0; i < newData.length; i++) {}. By this, each row is overwritten by each row of newData. By this, all searched rows in "RECORD" sheet are the same value. I thought that this might be the reason for your issue.
var oldData = shtRecords.getDataRange().getValues(); can be used one call.
In order to avoid this issue by modifying your script, as one of several methods, how about the following modification?
From:
for (var i = 0; i < newData.length; i++) {
var oldData = shtRecords.getDataRange().getValues();
for (var j = 0; j < oldData.length; j++) {
if (newData[i][0] == oldData[j][0]) {
var newData2 = [newData[i]];
shtRecords.getRange(j + 1, 1, 1, newData2[0].length).setValues(newData2);
}
}
}
To:
var oldData = shtRecords.getDataRange().getValues();
for (var j = 0; j < oldData.length; j++) {
for (var i = 0; i < newData.length; i++) {
if (newData[0][0] == oldData[j][0]) {
var newData2 = newData.splice(0, 1);
shtRecords.getRange(j + 1, 1, 1, newData2[0].length).setValues(newData2);
break;
}
}
}
Note:
At the above modification, setValues is used in a loop. In this case, the process cost becomes high. If you want to reduce the process cost of the script, how about using Sheets API? When Sheets API is used, how about the following modification? Please enable Sheets API at Advanced Google services.
To
var temp = newData.slice();
var data = shtRecords.getDataRange().getValues().reduce((ar, r, i) => {
if (temp[0][0] == r[0]) {
var t = temp.splice(0, 1);
t[0][2] = Utilities.formatDate(t[0][2], Session.getScriptTimeZone(), "dd/MM/yyyy");
t[0][4] = Utilities.formatDate(t[0][4], Session.getScriptTimeZone(), "dd/MM/yyyy");
ar.push({ range: `'RECORD'!A${i + 1}`, values: t });
}
return ar;
}, []);
Sheets.Spreadsheets.Values.batchUpdate({ data, valueInputOption: "USER_ENTERED" }, ss.getId());

Pushing Bigquery data to Google sheets using app script

MY script pulls data from Bigquery and saves it in google sheets but every time my script is triggered the sheet gets cleared and values are updated from the start date to the end date
Is there a possibility to append and only update few days of data where there is a data discrepancy rather than clearing and updating the whole sheet?
my script :
/**
* Runs a BigQuery query and logs the results in a spreadsheet.
*/
function runQuery() {
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'xxxxx';
var request = {
query: 'SELECT date, month, channel_grouping, device_category ' +
'FROM `xxxxx` where date between "2020-01-01" and "2020-01-03" order by date ;',
useLegacySql : false
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
// Get all the rows of results.
var rows = queryResults.rows;
while (queryResults.pageToken) {
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId, {
pageToken: queryResults.pageToken
});
rows = rows.concat(queryResults.rows);
}
if (rows) {
var spreadsheet = SpreadsheetApp.openById('1yOhjzprF2FXt3dM-FktVyVWNW1fCBhRO63f-QK-y3CQ');
var sheet = spreadsheet.getActiveSheet();
sheet.clear();
// Append the headers.
var headers = queryResults.schema.fields.map(function(field) {
return field.name;
});
sheet.appendRow(headers);
// Append the results.
var data = new Array(rows.length);
for (var i = 0; i < rows.length; i++) {
var cols = rows[i].f;
data[i] = new Array(cols.length);
for (var j = 0; j < cols.length; j++) {
data[i][j] = cols[j].v;
}
}
sheet.getRange(2, 1, rows.length, cols.length).setValues(data);
Logger.log('Results spreadsheet created: %s',
spreadsheet.getUrl());
} else {
Logger.log('No rows returned.');
}
}

Querying data from BigQuery and return a single data into Google AppMaker

From my previous question here, I think it is the problem to my server-script function that queries the data from BigQuery. Following are my code to queries HOD email base on requester email address.
//To fetch requester's HOD email address
function BigQueryGetEmail(emailRequester){
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'xxxxxx-xxx-stg';
query = '#standardSQL \n SELECT SUPV_EMAIL_A FROM `xxxxx-xxx-stg.xxxxx.vw_HOD_email` WHERE EMP_EMAIL_A = "' + emailRequester + '";';
var request = {
query: query,
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
// Get all the rows of results.
var rows = queryResults.rows;
while (queryResults.pageToken) {
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId, {
pageToken: queryResults.pageToken
});
rows = rows.concat(queryResults.rows);
}
// Append the results.
var data = new Array(rows.length);
for (var i = 0; i < rows.length; i++) {
var cols = rows[i].f;
data[i] = new Array(cols.length);
for (var j = 0; j < cols.length; j++) {
data[i][j] = cols[j].v;
}
}
console.log('HOD email: ' + data);
return data.toString();
}
The weird thing is that when I console.log data it return the HOD email address. But the function won't return the data result when ever I call the BigQueryGetEmail function. Following is the caller function:
function getHOD(){
var ownerEmail = app.pages.Main.descendants.TextBox1.value;
var HODemailfield = app.pages.Main.descendants.TextBox2.value;
function successHandler(email){
//assign email value to widget;
HODemailfield = email;
}
google.script.run.withSuccessHandler(successHandler).BigQueryGetEmail(ownerEmail);
}
The widget TextBox2 doesn't reflect any data nor error which is I believe it is because of the BigQueryGetEmail function doesn't return any data.
Can anyone guide me on the correct way how to return the query result when I call the BigQueryGetEmail function? had tried several ways but still stuck.

Insert table from gmail to google spreadsheet by google script

I recive an email with table many times in one day it can be 10x2 or 40x2 or something like this. Every time I need to paste this table in sheet start from cell A1/ Now I have this code, but it paste whole table to cell A1:
var SEARCH_QUERY = "label:inbox is:unread to:me subject:importnumbers";
// Credit: https://gist.github.com/oshliaer/70e04a67f1f5fd96a708
function getEmails_(q) {
var emails = [];
var threads = GmailApp.search(q);
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
emails.push([msgs[j].getBody().replace(/<.*?>/g, '\n')
.replace(/^\s*\n/gm, '').replace(/^\s*/gm, '').replace(/\s*\n/gm, '\n')
]);
}
}
return emails;
}
function appendData_(sheet, array2d) {
sheet.getRange(1, 1).setValues(array2d);
}
function saveEmails() {
var array2d = getEmails_(SEARCH_QUERY);
if (array2d) {
appendData_(SpreadsheetApp.getActiveSpreadsheet().getSheetByName('numberlist'), array2d);
}
markArchivedAsRead ();
}
function markArchivedAsRead() {
var threads = GmailApp.search('label:inbox is:unread to:me subject:importnumberlist');
GmailApp.markThreadsRead(threads);
};
Please, help me to paste table with right dimensions.
Try this one.
Replace getEmails_(q) function
Code#1:
function getEmails_(q) {
var emails = [];
var threads = GmailApp.search(q);
for (var i in threads) {
var msgs = threads[i].getMessages();
for (var j in msgs) {
var arrStr = msgs[j].getBody()
.replace(/<\/tr>/gm, '[/tr]')
.replace(/<\/td>/gm, '[/td]')
.replace(/<.*?>/g, '\n')
.replace(/^\s*\n/gm, '')
.replace(/^\s*/gm, '')
.replace(/\s*\n/gm, '\n')
.split("[/tr]");
var line = [];
for (var i = 0; i < arrStr.length - 1; i++) {
line = arrStr[i].split("[/td]");
line.length -= 1;
emails.push(line);
}
}
}
return emails;
}
Replace appendData_ function
Code#2:
function appendData_(sheet, array2d) {
var h = array2d.length;
var l = array2d[0].length;
sheet.getRange(1, 1, h, l).setValues(array2d);
}
This code converts the html text into 2d array and then paste it into Spreadsheet.
Note 2020-06
Your email may (1) contain several tables or (2) a table with merged cells. In this case, the proposed script may cause an error:
The number of columns in the data does not match the number of columns
in the range
In this situation, you'll need to create another function to parse the message body. I suggest you see the HTML-code of your message and inspect it in Gmail:
Right Click > Instect
or [Ctrl]+[Shift]+[I]
The other approach in this situation is to paste data "as is", use sample function to convert any 2d array into rectangular:
function convert2ArrayToRectangular_(array2d)
{
// get max width
var res = [];
var w = 0;
for (var i = 0; i < array2d.length; i++)
{
if (array2d[i].length > w) {w = array2d[i].length;}
}
var row = [];
for (var i = 0; i < array2d.length; i++)
{
row = array2d[i];
if(array2d[i].length < w)
{
for (var ii = array2d[i].length; ii < w; ii++)
{
row.push('');
}
}
res.push(row);
}
return res;
}
Sample usage:
function test_get_email() {
var q = 'Test_table_to_sheets';
var array2d = getEmails_(q);
var sheet = SpreadsheetApp.openById('1BKkd5LwBYyGoi2um-S3pTCBKrUEko34m9vJu94K8uOQ').getSheetByName('Test_table_to_sheets2');
appendData_(sheet, convert2ArrayToRectangular_(array2d));
}