I have a table and need to add more records based on new data from Google Sheets.
I see how I can do it with union, meaning running
Select * from (SELECT * from table),(select * from temp_table_from_sheets)
I.e: querying old table, new table. Delete old table and save the result of the query as old table.
BUT IT MUST be possible to append instead of BigQuery.Jobs.insert only.
Can you help me please?
EDIT - solution
After getting the answer below, I googled a lot and eventually came up with the following solution in Apps Script:
var sql = 'select ...'
var projectId = '...'
var datasetId = '...'
var tableId = '...'
var job = {
configuration: {
query: {
query: sql,
writeDisposition:'WRITE_APPEND',
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
}
}
}
};
var queryResults = BigQuery.Jobs.insert(job, projectId)
From BigQuery API Basics - Managing Tables:
Appending data
You can load additional data into a table either from source files or
by appending query results. Note that the schema of the loaded data
must match the schema of the existing table, but you can update the
schema before appending.
...
To append data from a query result:
Run an asynchronous query, pass in the name of your existing table,
and set writeDisposition=WRITE_APPEND.
Pushing Google Sheets content to BigQuery
I found it here
The tricky is sheet data to CSV.
var file = SpreadsheetApp.openByUrl(url).getSheetByName(sheetName);
// This represents ALL the data
var rows = file.getDataRange().getValues();
var rowsCSV = rows.join("\n");
var blob = Utilities.newBlob(rowsCSV, "text/csv");
var data = blob.setContentType('application/octet-stream');
// Create the data upload job.
var job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1,
writeDisposition: writeDispositionSetting
}
}
};
// send the job to BigQuery so it will run your query
var runJob = BigQuery.Jobs.insert(job, projectId, data);
Logger.log(runJob.status);
var jobId = runJob.jobReference.jobId
Logger.log('jobId: ' + jobId);
var status = BigQuery.Jobs.get(projectId, jobId);
// wait for the query to finish running before you move on
while (status.status.state === 'RUNNING') {
Utilities.sleep(500);
status = BigQuery.Jobs.get(projectId, jobId);
Logger.log('Status: ' + status);
}
Logger.log('FINNISHED!');
}
Related
I have build a google sheet that will be replicated 80 times for stores to enter operational data into.
This user generated data is then pushed to BigQuery with an Appscript via a button in the relevant sheet.
The script works fine, but I don't want to give each of the 80 users access to BigQuery, I would like to use a service account. As a bit of a newbie, I am not sure how to do this. I have some questions
Do I set this service account up in GCP IAM?
Do I assign permissions in IAM?
If I add service account to the GoogleSheet with editor permission- I assume that I would need to modify my appscript to use the service account. Otherwise I will get the error as per screenshot enter image description here
/**
* Loads the content of a Google Drive Spreadsheet into BigQuery
*/
function loadCogsPlayupHistory() {
// Enter BigQuery Details as variable.
var projectId = 'myproject';
// Dataset
var datasetId = 'my_Dataset';
// Table
var tableId = 'my table';
// WRITE_APPEND: If the table already exists, BigQuery appends the data to the table.
var writeDispositionSetting = 'WRITE_APPEND';
// The name of the sheet in the Google Spreadsheet to export to BigQuery:
var sheetName = 'src_cogs_playup_current';
Logger.log(sheetName)
var file = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('src_cogs_playup_current');
Logger.log(file)
// This represents ALL the data
var rows = file.getDataRange().getValues();
var rowsCSV = rows.join("\n");
var blob = Utilities.newBlob(rowsCSV, "text/csv");
var data = blob.setContentType('application/octet-stream');
Logger.log(rowsCSV)
// Create the data upload job.
var job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1,
writeDisposition: writeDispositionSetting
}
}
};
Logger.log(job)
// send the job to BigQuery so it will run your query
var runJob = BigQuery.Jobs.insert(job, projectId, data);
//Logger.log('row 61 '+ runJob.status);
var jobId = runJob.jobReference.jobId
Logger.log('jobId: ' + jobId);
Logger.log('row 61 '+ runJob.status);
Logger.log('FINISHED!');
// }
}
I have written an app script for Google Sheet which updates a table in BQ. The script should return among other information the total rows of the table. The script returns the job status and totalbytes but does not return the totalRows. I do not see a difference between the different values: https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
What do I have to change in order to get also TotalRows?
// Need to provoke a drive dialog
// DriveApp.getFiles()
// Replace this value with your project ID and the name of the sheet to update.
var projectId = 'my-project';
var sheetName = 'my-sheet';
// Use standard SQL to query BigQuery
var request = {
query: 'DROP TABLE `my-project.xyz.targetgroup_5_tbl`; CREATE TABLE `my-project.xyz.targetgroup_5_tbl` AS SELECT * FROM `waipu-app-prod.Access.targetgroup_5_std`;',
useLegacySql: false
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
if (queryResults.jobComplete=true) {
// Append the results.
var status= queryResults.jobComplete;
var dayc = new Date();
var totalbytes = queryResults.totalBytesProcessed;
var totalRows = queryResults.totalRows;
var rows = [
['target group',status,dayc,totalbytes,totalRows],
]
var status = queryResults.jobComplete;
var ss = SpreadsheetApp.getActiveSpreadsheet();
var currentSheet = ss.getSheetByName(sheetName);
currentSheet.getRange(23,1, 1, 5).setValues(rows);
console.info('%d rows inserted.', queryResults.totalRows);
} else {
console.info('No results found in BigQuery');
}
}
```
The code just have a small error, in one of the comparisons, you used:
if (queryResults.jobComplete=true) {
Which always will evaluate to true because this is an assignation, not a comparison. To compare if jobComplete is true, you should use '==' instead, i.e.,
if (queryResults.jobComplete==true) {
Or even better, since jobComplete is already a boolean variable, you can simply do:
if (queryResults.jobComplete) {
Now, the reason why you're not getting totalRows is because you are executing a sql script, which doesn't return any rows. Even if you consider only the last part of the script, you are executing a CREATE TABLE AS statement, which doesn't return any rows.
I understand that you want to know how many rows were inserted on this table. There are many options to do so, which includes getting the table details, or execute a SELECT COUNT query; however, based on what you're doing in the sql script: deleting and re-creating a table I recommend you a different approach: Use the API method Jobs.insert to specify a destination table so you can execute your query as a SELECT statement. Finally, use the writeDisposition:"WRITE_TRUNCATE" to delete the previous data, as in your script.
Here's a code that you can use as a reference:
var projectId = 'my-project';
// The request object changed to adapt to a Jobs.insert request
var request = {
configuration: {
query: {
destinationTable: {
projectId: projectId,
datasetId: 'my-dataset',
tableId: 'my-table'
},
query: 'SELECT * FROM ...', // Here goes the query used to create the table
useLegacySql: false,
writeDisposition: 'WRITE_TRUNCATE' // Truncate the data before writing it again
}
}
};
var queryResults = BigQuery.Jobs.insert(request, projectId); // Use Jobs.insert instead of Jobs.query
var jobId = queryResults.jobReference.jobId;
I am trying to create a view in Big query from an apps script but my code is giving me this error:
Error : API call to bigquery.tables.insert failed with error: Required parameter is missing (line 21...
I am not sure where i am going wrong. Please help.
Note: I am creating views using an app script because i have to copy views from one project to another project in big query but Transfer-service copy only tables not views. I have SQL query for all of my views but dont want to run all this SQLs manually in destination project.
Script:
function create_view() {
var projectId='MyProjectId';
var datasetId= 'MyDatasetName';
var tableID='MyViewName';
var table={
view: {
query:
'My query here',
useLegacySql: false
},
tableReference:{
projectId: projectId,
datasetId: datasetId,
tableID: tableID,
},
};
var queryResults = BigQuery.Tables.insert(table,projectId,datasetId);
Logger.log(queryResults.status);
}
The request you are issuing is correct, except for one little detail: the 'd' in tableID must be lowercase as in tableId.
Your final code would look like the following:
function create_view() {
var projectId = 'MyProjectId';
var datasetId = 'MyDatasetName';
var tableId = 'MyViewName';
var table={
view: {
query:
'My query here',
useLegacySql: false
},
tableReference:{
projectId: projectId,
datasetId: datasetId,
tableId: tableId,
},
};
var queryResults = BigQuery.Tables.insert(table,projectId,datasetId);
Logger.log(queryResults.status);
}
I am trying to extract an existing BigQuery table to a Cloud Storage bucket using Apps Script because I need to schedule it for frequent runs. The Cloud set up (billing, api enabled) is complete and when I extract the table via the Web UI it all works fine.
Please help me with the Apps Script code. Below you can see what I am writing. It works with no error messages but it does not create the json file in Storage.
function SQLTest() {
var projectId = 'xx_my_project';
var datasetId = 'xx_my_dataset';
var tableId = 'xx_my_table';
var destinationUris = 'gs://my_bucket/my_file.json';
var destinationFormat = 'NEWLINE_DELIMITED_JSON';
var job = {
configuration: {
extract: {
sourceTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
}
,
destinationUris: [destinationUris],
destinationFormat: destinationFormat
}
}
}
} ;
Unless I'm missing something, you didn't invoke any API here.
Please follow this guide on how to call a Bigquery API from Apps Script:
https://developers.google.com/apps-script/advanced/bigquery
Please also follow this guide to extract table:
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
Full corrected script
function SQLTest() {
var projectId = 'xx_my_project';
var datasetId = 'xx_my_dataset';
var tableId = 'xx_my_table';
var destinationUris = 'gs://my_bucket/my_file.json';
var destinationFormat = 'NEWLINE_DELIMITED_JSON';
var job = {
configuration: {
extract: {
sourceTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
}
,
destinationUris: [destinationUris],
destinationFormat: destinationFormat
}
}
};
var request = BigQuery.Jobs.insert(job,projectId);
}
I'm trying to write a GAS that simply runs a saved BigQuery query I have. The query itself is very straightforward - it's just an UPDATE query:
UPDATE `project.dataset.table`
SET field1= '72142',
field2= 'Yes'
WHERE field3 like '%72142%'
AND field1 IS NULL
I found this previous question
How to use App script to run a google big query and save it as a new table which can we reused?
That was quite helpful, but it includes creating a table after the query is run, and I don't want to do that. I tried to adapt the script to ignore the destination table and writeDisposition values.
But it's not working.
Update: this is the script
function runQuery() {
var projectId = 'project';
var datasetId = 'dataset';
var tableId = 'table';
var job = {
configuration: {
query: {
query: 'UPDATE `project.dataset.table` SET field1 = '72142', field2 = 'Yes' WHERE field3 like '%72142%'AND field1 IS NULL,
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
}
}
}
};
var queryResults = BigQuery.Jobs.insert(job, projectId);
Logger.log(queryResults.status);
}
And the error message is
"Syntax error.(line 21, file "xxx")
You need to remove all references to any table write flags, and you also have some syntax problems in your JSON:
function runQuery() {
var configuration = {
"query": {
"useQueryCache": false,
"useLegacySql": false,
"query": "UPDATE `<your_projectId>.<your_dataset>.<your_table>` SET field1 = '72142',field2 = 'Yes' WHERE field3 like '%72142%' AND field1 IS NULL"
}
};
var job = {
"configuration": configuration
};
var jobResult = BigQuery.Jobs.insert(job, "<your_projectId>");
Logger.log(jobResult);
}