I'm using BigQuery.Jobs.insert to upload a csv blob to a BigQuery table (from a Google Sheets spreadsheet if that matters), but I need to know when the upload is complete so that I can run a query to process it. Here's the code I have...
csv_blob = Utilities.newBlob(csv,"application/octet-stream");
//Copy the csv to BigQuery
job = BigQuery.Jobs.insert(job, projectId, csv_blob );
var jobId=job.jobReference.jobId;
//Give it a half second to complete
var sleepTimeMs = 500;
Utilities.sleep(sleepTimeMs);
//see if it needs longer
var jobstatus = job.status;
while(jobstatus.state!="COMPLETE") {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
jobstatus = job.status;
}
The upload is working, but the job.status.state never changes from "RUNNING", no matter how long I wait. Even after I manually verify the uploaded table on the server, the job.status.state is still "RUNNING". What is the right way to know when the job is complete? Thanks
You need to fetch the current status of the job with BigQuery.Jobs.Get (REST reference), it appears that you're just rechecking the response you received from the initial job insertion.
An quick and dirty example of invoking this:
function myFunction() {
projectID = "my-project-id"
jobID = "some-job-id"
location = "us-central1"
job = BigQuery.Jobs.get(projectID, jobID, {location: location});
console.log("result state: " + job.status.state + " end time:" + job.statistics.endTime);
}
Related
I have build a google sheet that will be replicated 80 times for stores to enter operational data into.
This user generated data is then pushed to BigQuery with an Appscript via a button in the relevant sheet.
The script works fine, but I don't want to give each of the 80 users access to BigQuery, I would like to use a service account. As a bit of a newbie, I am not sure how to do this. I have some questions
Do I set this service account up in GCP IAM?
Do I assign permissions in IAM?
If I add service account to the GoogleSheet with editor permission- I assume that I would need to modify my appscript to use the service account. Otherwise I will get the error as per screenshot enter image description here
/**
* Loads the content of a Google Drive Spreadsheet into BigQuery
*/
function loadCogsPlayupHistory() {
// Enter BigQuery Details as variable.
var projectId = 'myproject';
// Dataset
var datasetId = 'my_Dataset';
// Table
var tableId = 'my table';
// WRITE_APPEND: If the table already exists, BigQuery appends the data to the table.
var writeDispositionSetting = 'WRITE_APPEND';
// The name of the sheet in the Google Spreadsheet to export to BigQuery:
var sheetName = 'src_cogs_playup_current';
Logger.log(sheetName)
var file = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('src_cogs_playup_current');
Logger.log(file)
// This represents ALL the data
var rows = file.getDataRange().getValues();
var rowsCSV = rows.join("\n");
var blob = Utilities.newBlob(rowsCSV, "text/csv");
var data = blob.setContentType('application/octet-stream');
Logger.log(rowsCSV)
// Create the data upload job.
var job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1,
writeDisposition: writeDispositionSetting
}
}
};
Logger.log(job)
// send the job to BigQuery so it will run your query
var runJob = BigQuery.Jobs.insert(job, projectId, data);
//Logger.log('row 61 '+ runJob.status);
var jobId = runJob.jobReference.jobId
Logger.log('jobId: ' + jobId);
Logger.log('row 61 '+ runJob.status);
Logger.log('FINISHED!');
// }
}
I am trying to push data using the Salesforce API to BigQuery using Google Scripts. For this example, I am trying to push Opportunities to BigQuery.
I have this variable called opps, and it is prepared by calling the Salesforce API and then retrieving only the data that I care about. This looks like this:
var opps = [];
for (var i in arrOpportunities.records) {
let data = arrOpportunities.records[i];
let createDate = Utilities.formatDate(new Date(data.CreatedDate), "GMT", "dd-MM-YYYY");
let modDate = Utilities.formatDate(new Date(data.LastModifiedDate), "GMT", "dd-MM-YYYY");
let a1 = 'C' + (parseInt(i, 10) + 2);
let companyFormula = '=IFERROR(INDEX(Accounts,MATCH(' + a1 + ',Accounts!$B2:$B,0),1),"")';
opps.push([data.Name, data.Id, data.AccountId, companyFormula, data.StageName, data.IsClosed, data.IsWon, createDate, modDate, data.ContactId, data['Region_by_manager__c'], data['Industry__c'], data['Customer_type__c'], data['acc__c'], data['Reason_for_lost_deal__c'], data['hs_deal_id__c'], data['Solution__c'], data['Solution_Elements__c']]);
}
Afterwards I check to see if there is already a table in BigQuery using guideInsertData('Opportunities', 'Opportunities', opps); which comes after the for loop above. It doesn't actually check itself, it has another function for that, which is irrelevant for this discussion. This function just guides the process.
The function here looks like this:
function guideInsertData(tableName, headers, data) {
const tableExists = checkTables(tableName);
if (tableExists == false) {
let createTable = prepareSchema(headers, tableName);
if (createTable == false) {
throw 'Unable to create table';
}
}
insertData(tableName,data);
}
and then finally we get to the actual function for inserting the data which is here:
function insertData(tableName, arrData) {
const projectId = 'lateral-scion-352013', datasetId = 'CRM_Data', tableId = tableName;
const job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1
}
}
};
const strData = arrData.join("\n");
const data = Utilities.newBlob(strData,"application/octet-stream");
Logger.log(strData);
Logger.log(data.getDataAsString());
try {
BigQuery.Jobs.insert(job, projectId, data);
let success = 'Load job started. Check on the status of it here: ' +
'https://console.cloud.google.com/home/activity?project='+projectId;
Logger.log(success);
return success;
} catch (err) {
Logger.log(err);
Logger.log('unable to insert job');
return 'unable to insert data';
}
}
This is based on https://developers.google.com/apps-script/advanced/bigquery#load_csv_data.
I convert everything into the blob as requested. I can't see anything wrong in the Log. Not only that, I get the message that the jobs have loaded successfully.
However, when I check the status of the activity in BigQuery, I get Failed:Complete BigQuery job, these are the error messages:
Invalid argument (HTTP 400): Error while reading data, error message: Too many values in row starting at position: 234. Found 23 column(s) while expected 18.
Error while reading data, error message: CSV processing encountered too many errors, giving up. Rows: 0; errors: 1; max bad: 0; error percent: 0
You are loading data without specifying data format, data will be treated as CSV format by default. If this is not what you mean, please specify data format by --source_format.
How do I fix this error?
Ok, so it looks like there was an issue in how it was stringified. I had to stringify first each nested array and then stringify the whole thing.
So in the end, the code looks like this:
const strData = arrData.map(values => values.map(value => JSON.stringify(value).replace(/\\"/g, '""')));
const csvData = strData.map(values => values.join(',')).join('\n');
const data = Utilities.newBlob(csvData, "application/octet-stream");
Also, all date values need to be in yyyy-MM-dd format not dd-MM-yyy
I am using an installed edit trigger of Google Sheets to create Google Tasks. However, when a row containing a task that has already been created as a Task is edited, a duplicate Task is created for the same day.
I'd like to find all the Tasks in a given list with a particular due date. Then I will be able to check their titles, to compare with the title of the task that would be created, so the script may decide if it should create a new task or update the existing one.
Here's my current triggered code:
function addTask(event){
if (spreadsheet.getActiveSheet().getName() === "Task List") {
var RowNum = event.range.getRow();
var taskproperties = spreadsheet.getActiveSheet().getRange(RowNum, 1, 1, 5).getValues();
var Title = taskproperties[0][1];
var Frequency = taskproperties[0][2];
var StartDate = taskproperties[0][3];
var Recurrence = taskproperties[0][4];
if (Title.trim().length !== 0 && Frequency.trim().length !== 0 &&
StartDate.toString().trim().length !== 0 && Recurrence.toString().trim().length !== 0)
{
//Code to Create a new task
//Code Get the task date
//Some codes to set Date parameters for use in script functions
//Some codes to set Date parameters for use in sheet functions
//Set the task parameters
//add task to list
//--------------------------------------------------------------
//Assign a cell in the spreadsheet for calculation of new dates for recurring task
var tempdatecell= spreadsheet.getSheetByName("Task List").getRange("F1")
//Insert new tasks based on the number of recurrence
for (i = 1; i < Recurrence; i++) {
//Insert a formula in a cell the spreadsheet to calculate the new task date
tempdatecell.setFormula('=WORKDAY.INTL("' + shTaskStartDate + '",' + i + '*VLOOKUP("' + Frequency + '",tasktype,2,false),"1000011")')
//Get task date from the cell
TaskDate = tempdatecell.getValue()
//Date parameters for use in script functions
var TaskDate = new Date(TaskDate);
var taskmonth = Number(TaskDate.getMonth())
var taskDay = TaskDate.getDate() + 1
var taskyear = TaskDate.getYear()
//Create a new task
var task = Tasks.newTask();
//Set the task parameters
task.title = Title;
task.due = new Date(taskyear, taskmonth, taskDay).toISOString()
//add task to list
task = Tasks.Tasks.insert(task, tasklistID);
}
tempdatecell.clearContent()
}
}
}
You might consider having your script write to another cell (probably in another column) that indicates the status of the task, such as added or updated and then write in a conditional statement that checks that cell to determine what to do with it. This is a really vague answer, but as Tanaike stated in their comment "provide your current script" or a generic version of it and we can be of greater help.
I have managed to find a work around which involves filtering the entire tasklist. It seems to work find with the few task that I have now. I am not sure how it will perform with a large volume of tasks. Any further contribution welcomes.
The code that I am using in the work around is as follows and replaces the line below //Create a new task in my original code:-
//Check if the task exist for the task date
var listoftasks = Tasks.Tasks.list(tasklistID)
var filtermonth="0" + shTaskStartMonth
var filterdate=scTaskStartYear + "-" + filtermonth.substr(filtermonth.length-2) + "-" + shTaskStartDay + "T00:00:00.000Z"
var filteredtask=listoftasks["items"].filter(function(item){
return item["due"]== filterdate && item["title"]===Title
})
if(filteredtask.length==0){
//Create a new task
var task = Tasks.newTask()
//Set the task parameters
task.title = Title;
task.due=new Date(scTaskStartYear,scTaskStartMonth,scTaskStartDay).toISOString()
//add task to list
task = Tasks.Tasks.insert(task, tasklistID)
}
else{
//Get the existing task
task = Tasks.Tasks.get(tasklistID, filteredtask[0].id)
task.setStatus("completed")
}
NB:- The setStatus does not work as expected but I will post a separate question for that.
Prior Research
Please do not close this question as a duplicate because my question deals with how to resolve the specific error message I am receiving and not the general question of whether my objective is achievable or not — as some other related questions, yielded by my research and below detailed, have asked.
Related questions and why they do not apply here
This question, asked 7/27/2012, does not apply because it: (1) is too old (after 10 months, new solutions/methods might exist) and (2) does not deal with the specific error message I am experiencing.
This question, asked 10/12/2012, fails to apply for similar reasons.
My below code was copied from here which was forked from here. These are presumably, working solutions because they have been referenced as such from other question/answer exchanges here on Stack Overflow.
Objective
Programmatically, I am trying to:
Search my email inbox.
Find Excel (.xls) file attachments.
Upload those .xls file attachments to Google Drive.
While uploading, convert the .xls files into a Google Spreadsheet file format.
Problem
When I execute processInbox() (code shown at the bottom of this question), it fails and I get the error message shown below.
Error Message
Request failed for returned code 403.
Server response:
{
"error":{
"errors":[
{
"domain":"usageLimits",
"reason":"accessNotConfigured",
"message":"AccessNotConfigured"
}
],
"code":403,
"message":"AccessNotConfigured"
}
}
(line 13, file "DriveUpload")
Question
What am I doing wrong? And how can I fix it?
For example, do I need to do something special in my API console relative to setting up my project to, say, access Google Drive or something? What am I missing?
Note: I have not yet successfully implemented oAuth in any of my applications, yet.
Error Source
Line 13
(This is the code line referenced by the error message.)
var uploadRequest = UrlFetchApp.fetch("https://www.googleapis.com/upload/drive/v2/files/?uploadType=media&convert=true&key="+key, params); // convert=true convert xls to google spreadsheet
Code
The complete body of code I am working with is shown below for your reference. I extracted the error-triggering, “line 13,” and highlighted it above to help us focus on the proximate cause of the problem.
DriveUpload.js
function uploadXls(file) {
authorize();
var key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; // <-- developer key
var metadata = { title: file.getName() }
var params = {method:"post",
oAuthServiceName: "drive",
oAuthUseToken: "always",
contentType: "application/vnd.ms-excel",
contentLength: file.getBytes().length,
payload: file.getBytes()
};
// convert=true convert xls to google spreadsheet
var uploadRequest = UrlFetchApp.fetch("https://www.googleapis.com/upload/drive/v2/files/?uploadType=media&convert=true&key="+key, params);
var uploadResponse = Utilities.jsonParse(uploadRequest.getContentText());
var params = {method:"put",
oAuthServiceName: "drive",
oAuthUseToken: "always",
contentType: "application/json",
payload: Utilities.jsonStringify(metadata)
};
var metaRequest = UrlFetchApp.fetch("https://www.googleapis.com/drive/v2/files/"+uploadResponse.id+"?key="+key, params)
return DocsList.getFileById(uploadResponse.id);
}
function authorize() {
var oauthConfig = UrlFetchApp.addOAuthService("drive");
var scope = "https://www.googleapis.com/auth/drive";
oauthConfig.setConsumerKey("anonymous");
oauthConfig.setConsumerSecret("anonymous");
oauthConfig.setRequestTokenUrl("https://www.google.com/accounts/OAuthGetRequestToken?scope="+scope);
oauthConfig.setAuthorizationUrl("https://accounts.google.com/OAuthAuthorizeToken");
oauthConfig.setAccessTokenUrl("https://www.google.com/accounts/OAuthGetAccessToken");
}
function processInbox() {
// get all threads in inbox
var threads = GmailApp.getInboxThreads();
for (var i = 0; i < threads.length; i++) {
// get all messages in a given thread
var messages = threads[i].getMessages();
// iterate over each message
for (var j = 0; j < messages.length; j++) {
// log message subject
var subject = messages[j].getSubject()
//Logger.log(subject);
if ( subject == "with xls attach" ){
Logger.log(messages[j].getSubject());
var attach = messages[j].getAttachments()[0];
var name = attach.getName();
var type = attach.getContentType();
//var data = attach.getDataAsString();
Logger.log( name + " " + type + " " );
var file = uploadXls(attach);
SpreadsheetApp.open(file);
}
}
}
};
Drive API is already built in GAS: https://developers.google.com/apps-script/reference/drive/
Use DriveApp and your problems go away ;-)
This maybe a temp solution
Step 1: Use a Google Form to Collect Data to a Google spreadsheet
Step 2: Add the Zoho Sheet App to your Google Drive
In a Zoho Sheet
Goto
Data Menu
»Link External Data
Select either
CSV
RSS/Atom Feed
or HTML Page
You can schedule it to update at specific time intervals
What I like is the VBA and Macros in Zoho
You can also do Pivot Charts and Tables
You can copy and paste Excel VBA into Zoho !
I have an Unpivot VBA that I will run on my Tabular dataset
before I can do a PivotChart
It is hard to beat all the functionality of Excel and I often fall back on familiar tools !
If I hear of anything I will post it
Good luck
I recently started getting this exception:
driveWriteVolume rateMax. Try Utilities.sleep(1000) between calls
Usually this means that I'm trying to access a service to quickly in repetition. However, I have never seen this message before and any searches I have made don't return any useful information about it.
Here is the best mockup I could do without pasting 100 lines of code:
function compileLabel(){
var doc = DocsList.createFile('My Label', '', 'text/html');
var threads = GmailApp.getUserLabelByName('My Label');
var startTime = Date.now();
while((Date.now() - startTime) < 240000){
for(t in threads){
var messages = threads[t].getMessages();
var threadHeader = createThreadHeader(); //Builds HTML representation of thread info as string
doc.append(threadHeader);
for(m in messages){
var msgHeader = createMessageHeader(); //Builds HTML representation from header info as string
doc.append(msgHeader);
doc.append(messages[i].getBody());
var attachments = messages[m].getAttachments();
if(attachments.length > 0){
var attachmentFolder = parentFolder.createFolder(messages[m].getSubject());
}
for(a in attachments){
attachmentFolder.createFile(attachments[a]);
}
}
}
}
parentFolder.createFile(doc.getAs('application/pdf')); //this is intermittantly throwing an exception about serialization now, but that's probably a different issue.
//After time based loop, do more things that don't have any DocsList based functions
}
As of 2 days ago, this worked very well. Now, unless I sleep(1000) before every doc.append(), createFile() or createFolder(), it produces this error. I can provide a project key if necessary.
I discovered the same error mails today, apparently I exceeded the allowed upload rate to Google Drive. To fix it I had to increase the sleep time to 3000 (I was already using 1000). This is the piece of code that caused the error:
var att = messages[y].getAttachments();
var attlinks = [];
for (var z=0; z<att.length; z++) {
try {
var file = folder.createFile(att[z]); //create file in gdrive
attlinks.push(''+file.getName()+''); //push link to array
Utilities.sleep(3000); //increased from 1000 to 3000
}
update
I had to increase the sleep time once more (to 10s) because the issue still occurred when there were more than 5 attachments.