Cloud Functions and Storage: Is deleting file asynchronous? - google-cloud-functions

In my Google Cloud Functions script, I want to delete a Google Cloud Storage file by using the following code:
const gcs = require('#google-cloud/storage')()
exports.deletePost = functions.https.onRequest((request, response) => {
if(!context.auth) {
throw new functions.https.HttpsError('failed-precondition', 'The function must be called while authenticated.');
}
const the_post = request.query.the_post;
const filePath = context.auth.uid + '/posts/' + the_post;
const bucket = gcs.bucket('android-com')
const file = bucket.file(filePath)
const pr = file.delete()
});
The problem is that I also need to delete a Google Firebase Firestore database entry after having deleted the Storage file. So I would want to know if I could do it within, for example, a promise that would be returned by delete?
PS : I don't find the doc

The code file.delete() is asynchronous and returns a Promise, as defined in the Google Cloud Storage: Deleting objects documentation.
To delete an object from one of your Cloud Storage buckets:
// Imports the Google Cloud client library
const {Storage} = require('#google-cloud/storage');
// Creates a client
const storage = new Storage();
/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// const bucketName = 'Name of a bucket, e.g. my-bucket';
// const filename = 'File to delete, e.g. file.txt';
// Deletes the file from the bucket
await storage
.bucket(bucketName)
.file(filename)
.delete();
console.log(`gs://${bucketName}/${filename} deleted.`);
It's not overly clear, but because the await syntax is used, this implies that the result of the expression to it's right is a Promise.
Note: Most of the useful Google Cloud Storage documentation can be found here.

Related

Google Sheets AddOn - AppScript & BigQuery Integration via Service Account

I have a Google Sheets workspace addon and recently did some work to integrate BigQuery. Essentially BigQuery hold a record of books each of which has an author, title etc and my Addon allows people to pull the books that they have read into their sheet. The first column in the sheet allows people to choose from all the authors in the DB, based on that selection the second column is populated with data from BigQuery with all books by that author etc etc. There is no need for my AddOn to access a user's BigQuery, they only access 'my' BgQuery.
This all works fine, but when I submitted my addon for approval I was told
Unfortunately, we cannot approve your request for the use of the following scopes
https://www.googleapis.com/auth/bigquery
We recommend using service accounts for this type of information exchange.
This seems fair and reading up on Service Accounts it seems a much better fit for my use case. I've gone through the process of creating the service accounts and downloaded my security details json file, however I just can't figure out how to actually query BigQuery from AppScript.
In my non-service account method I have the BigQuery Library installed in AppScript and basically run
var queryResults = BigQuery.Jobs.query(request, projectId);
I've been trying to work from an example at https://developers.google.com/datastudio/solution/blocks/using-service-accounts
function getOauthService() {
var serviceAccountKey = getServiceAccountCreds('SERVICE_ACCOUNT_KEY');// from private_key not private_key_id of JSON file
var serviceAccountEmail = getServiceAccountCreds('SERVICE_ACCOUNT_EMAIL');
return OAuth2.createService('RowLevelSecurity')
.setAuthorizationBaseUrl('https://accounts.google.com/o/oauth2/auth')
.setTokenUrl('https://accounts.google.com/o/oauth2/token')
.setPrivateKey(serviceAccountKey)
.setIssuer(serviceAccountEmail)
.setPropertyStore(scriptProperties)
.setCache(CacheService.getScriptCache())
.setScope(['https://www.googleapis.com/auth/bigquery.readonly']);
}
function getData(request) {
var accessToken = getOauthService().getAccessToken();
var billingProjectId = getServiceAccountCreds('BILLING_PROJECT_ID');
// var email = Session.getEffectiveUser().getEmail();
// return cc
// .newBigQueryConfig()
// .setAccessToken(accessToken)
// .setBillingProjectId(billingProjectId)
// .setUseStandardSql(true)
// .setQuery(BASE_SQL)
// .addQueryParameter('email', bqTypes.STRING, email)
// .build();
}
I've commented out the code in the above which relates to
var cc = DataStudioApp.createCommunityConnector();
in the above tutorial since I'm not using DataStudio but I'm really not sure what to replace it with so I can query BigQuery with AppScript via a Service Account. Can anyone offer any advice?
Based on the advice from #TheAddonDepot in the comments above my revised code now looks like:
function getBigQueryService() {
return (
OAuth2.createService('BigQuery')
// Set the endpoint URL.
.setTokenUrl('https://accounts.google.com/o/oauth2/token')
// Set the private key and issuer.
.setPrivateKey(JSON_CREDS.private_key) // from the json file downloaded when you create service account
.setIssuer(JSON_CREDS.client_email). // from the json file downloaded when you create service account
// Set the property store where authorized tokens should be persisted.
.setPropertyStore(PropertiesService.getScriptProperties())
// Caching
.setCache(CacheService.getUserCache())
// Locking
.setLock(LockService.getUserLock())
// Set the scopes.
.setScope(['https://www.googleapis.com/auth/bigquery.readonly'])
// .setScope('https://www.googleapis.com/auth/bigquery')
)
}
function queryData(){
const bigQueryService = getBigQueryService()
if (!bigQueryService.hasAccess()) {
Logger.log("BQ ERROR IS "+ bigQueryService.getLastError())
}
//const projectId = bigqueryCredentials.project_id
var projectId = "<yourprojectid>"
let url = 'https://bigquery.googleapis.com/bigquery/v2/projects/<yourprojectid>/queries'; //projectID is taken from the security json file for the service account, although it doesn't seem to matter if you use the project code
const headers = {
Authorization: `Bearer ${bigQueryService.getAccessToken()}`,
'Content-Type': 'application/json',
}
var data = {query:"<your query>",useLegacySql:false};
const options = {
method: 'post',
headers,
//contentType: 'application/json',
payload: JSON.stringify(data),
muteHttpExceptions: true // on for debugging
}
try {
const response = UrlFetchApp.fetch(url, options)
const result = JSON.parse(response.getContentText())
Logger.log("here is result "+ JSON.stringify(result))
} catch (err) {
console.error(err)
}
}

How to convert PDF to image using Google Apps Script? [duplicate]

is there a way to create an image (e.g. a png) from a google document?
I really mean an image, not just a pdf. GetAS only creates pdf, but returns an error if contentType is set to image/png or other equivalent formats.
My (actually trivial) code is
function convertFile() {
var SOURCE_TEMPLATE = "1HvqYidpUpihzo_HDAQ3zE5ScMVsHG9NNlwPkN80GHK0";
var TARGET_FOLDER = "1Eue-3tJpE8sBML0qo6Z25G0D_uuXZjHZ";
var source = DriveApp.getFileById(SOURCE_TEMPLATE);
var targetFolder = DriveApp.getFolderById(TARGET_FOLDER);
var target = source.makeCopy(source,targetFolder);
var newFile = DriveApp.createFile(target.getAs('image/png'));
}
When I run this code, I get the following error (my translation):
The conversion from application/vnd.google-apps.document to image/png is not supported.
Ty
How about this answer?
Reason of error:
makeCopy() returns File object. getAs() cannot be used for this. By this, the error occurs.
Workaround:
Unfortunately, in the current stage, Google Document cannot be directly exported as PNG images. So it is required to think of workarounds. Google Document can be converted to PDF. This answer uses this. As a workaround, I would like to propose to use an external API which is ConvertAPI. I thought that using the external API, the script becomes simple. This a method (PDF to PNG API) of API can be converted from PDF data to PNG data.
When you try this, for example, you can also test this using "Free Package". When you try using "Free Package", please Sign Up at "Free Package" and retrieve your Secret key.
Sample script:
Before you run this script, please retrieve your Secret key and set it.
var secretkey = "###"; // Please set your secret key.
var SOURCE_TEMPLATE = "1HvqYidpUpihzo_HDAQ3zE5ScMVsHG9NNlwPkN80GHK0";
var TARGET_FOLDER = "1Eue-3tJpE8sBML0qo6Z25G0D_uuXZjHZ";
var url = "https://v2.convertapi.com/convert/pdf/to/png?Secret=" + secretkey;
var options = {
method: "post",
payload: {File: DriveApp.getFileById(SOURCE_TEMPLATE).getBlob()},
}
var res = UrlFetchApp.fetch(url, options);
res = JSON.parse(res.getContentText());
res.Files.forEach(function(e) {
var blob = Utilities.newBlob(Utilities.base64Decode(e.FileData), "image/png", e.FileName);
DriveApp.getFolderById(TARGET_FOLDER).createFile(blob);
});
References:
makeCopy()
getAs()
ConvertAPI
PDF to PNG API of ConvertAPI
Updated on January 11, 2023:
In the current stage, Google Apps Script can use V8 runtime. By this, there are some Javascript libraries that can be used with Google Apps Script. Ref1, Ref2 In this question, in the current stage, by using pdf-lib, all pages in a PDF file can be converted to PNG images using Google Apps Script. The sample script is as follows.
Sample script:
This method uses Drive API. Please enable Drive API at Advanced Google services.
Please set SOURCE_TEMPLATE and TARGET_FOLDER, and run main().
/**
* This is a method for converting all pages in a PDF file to PNG images.
* PNG images are returned as BlobSource[].
* IMPORTANT: This method uses Drive API. Please enable Drive API at Advanced Google services.
*
* #param {Blob} blob Blob of PDF file.
* #return {BlobSource[]} PNG blobs.
*/
async function convertPDFToPNG_(blob) {
// Convert PDF to PNG images.
const cdnjs = "https://cdn.jsdelivr.net/npm/pdf-lib/dist/pdf-lib.min.js";
eval(UrlFetchApp.fetch(cdnjs).getContentText()); // Load pdf-lib
const setTimeout = function (f, t) { // Overwrite setTimeout with Google Apps Script.
Utilities.sleep(t);
return f();
}
const data = new Uint8Array(blob.getBytes());
const pdfData = await PDFLib.PDFDocument.load(data);
const pageLength = pdfData.getPageCount();
console.log(`Total pages: ${pageLength}`);
const obj = { imageBlobs: [], fileIds: [] };
for (let i = 0; i < pageLength; i++) {
console.log(`Processing page: ${i + 1}`);
const pdfDoc = await PDFLib.PDFDocument.create();
const [page] = await pdfDoc.copyPages(pdfData, [i]);
pdfDoc.addPage(page);
const bytes = await pdfDoc.save();
const blob = Utilities.newBlob([...new Int8Array(bytes)], MimeType.PDF, `sample${i + 1}.pdf`);
const id = DriveApp.createFile(blob).getId();
Utilities.sleep(3000); // This is used for preparing the thumbnail of the created file.
const link = Drive.Files.get(id, { fields: "thumbnailLink" }).thumbnailLink;
if (!link) {
throw new Error("In this case, please increase the value of 3000 in Utilities.sleep(3000), and test it again.");
}
const imageBlob = UrlFetchApp.fetch(link.replace(/\=s\d*/, "=s1000")).getBlob().setName(`page${i + 1}.png`);
obj.imageBlobs.push(imageBlob);
obj.fileIds.push(id);
}
obj.fileIds.forEach(id => DriveApp.getFileById(id).setTrashed(true));
return obj.imageBlobs;
}
// Please run this function.
async function myFunction() {
const SOURCE_TEMPLATE = "1HvqYidpUpihzo_HDAQ3zE5ScMVsHG9NNlwPkN80GHK0";
const TARGET_FOLDER = "1Eue-3tJpE8sBML0qo6Z25G0D_uuXZjHZ";
// Use a method for converting all pages in a PDF file to PNG images.
const blob = DriveApp.getFileById(SOURCE_TEMPLATE).getBlob();
const imageBlobs = await convertPDFToPNG_(blob);
// As a sample, create PNG images as PNG files.
const folder = DriveApp.getFolderById(TARGET_FOLDER);
imageBlobs.forEach(b => folder.createFile(b));
}
When this script is run, all pages of the inputted PDF file are converted to PNG images, and those images are created in the destination folder.
Note:
I think that the above script works. But, in this case, when you directly copy and paste the Javascript retrieved from https://cdn.jsdelivr.net/npm/pdf-lib/dist/pdf-lib.min.js to your Google Apps Script project, the process cost for loading it can be reduced.
References:
pdf-lib
copyPages of pdf-lib
addPage of pdf-lib
I know this is an older question, but I thought I'd answer, since I believe I've found a solution that doesn't involve paying for a third-party subscription.
This can be accomplished by accessing the thumbnail of the Doc and creating a new PNG file from that thumbnail. Try this:
function convertFile() {
var SOURCE_TEMPLATE = "1HvqYidpUpihzo_HDAQ3zE5ScMVsHG9NNlwPkN80GHK0";
var TARGET_FOLDER = "1Eue-3tJpE8sBML0qo6Z25G0D_uuXZjHZ";
var source = DriveApp.getFileById(SOURCE_TEMPLATE).getThumbnail().getAs('image/png');
var targetFolder = DriveApp.getFolderById(TARGET_FOLDER);
TARGET_FOLDER.createFile(source);
}
However, I've found that getting the thumbnail of the Doc is not as high quality as getting the thumbnail of a PDF created from the Doc. You can try the code below to compare which version of the new PNG you prefer.
To do this, you will also need to enable Advanced Services on your project, specifically the Drive API service. To do this, follow these instructions to add a new Service to your Google Apps Script project:
Open the Apps Script project.
At the left, click Editor < >.
At the left, next to Services, click Add a service +.
Select Drive API and click Add.
Once you do that, you'll be able to use the Drive command in your script, which is different than DriveApp. Note also the update to source.makeCopy() to only include the TARGET_FOLDER:
function convertFile() {
var SOURCE_TEMPLATE = "1HvqYidpUpihzo_HDAQ3zE5ScMVsHG9NNlwPkN80GHK0";
var TARGET_FOLDER = "1Eue-3tJpE8sBML0qo6Z25G0D_uuXZjHZ";
var source = DriveApp.getFileById(SOURCE_TEMPLATE);
var targetFolder = DriveApp.getFolderById(TARGET_FOLDER);
var target = source.makeCopy(targetFolder);
var pdfBlob = target.getAs(MimeType.PDF);
var newPDF = TARGET_FOLDER.createFile(pdfBlob).setName('Some Name.pdf');
var newId = newPDF.getId();
Drive.Files.update({
title: newPDF.getName(), mimeType: MimeType.PDF
}, newId, pdfBlob);
var newFile = DriveApp.getFileById(newId).getThumbnail().getAs('image/png');
TARGET_FOLDER.createFile(newFile);
target.setTrashed(true);
newPDF.setTrashed(true);
}
This code will create a copy of your Google Doc file, convert it to a PDF, then grab the thumbnail of the PDF as a PNG, and then delete the copy of the Doc file and the PDF that were created.
The Drive.Files.update() function is the critical part of this code, as it finalizes the creation of the PDF file in your Drive. Trying to run the code without that portion will just return the new PDF file as null since the new PDF hasn't completely finished being created at that point.
Hope this helps!

Delete multiple google drive files that are unorganized / orphaned

I wanted to remove several thousand files in a Google Drive folder. I deleted the folder, which many in the community know (but I didn't) leaves the files present but orphaned. I can list the files via "is:unorganized owner:me" but can't select more than a few hundred at a time to (slowly) delete.
Is there a script that will search for and delete these, and only these, files?
Thanks
I believe your goal as follows.
You want to retrieve the files searched with is:unorganized owner:me and want to delete the files.
You want to reduce the process cost of this process.
Issue and workaround:
In the current stage, unfortunately, the files cannot be directly retrieved by searching is:unorganized owner:me with Drive API and Drive service. So as the current workaround, I would like to suggest the following flow.
Retrieve the file list of all files in the Google Drive. In this case, 'me' in owners and trashed = false is used as the search query. The method of "Files: list" of Drive API is used for this.
Retrieve the file list of files which have no parent folder using a script.
By above flow 1 and 2, the search of is:unorganized owner:me can be achieved using a script.
Move the files to the trash box using the retrieved file list using the method of "Files: update" of Drive API.
In this case, the files are not directly deleted. Because when the files are directly deleted, I thought that this situation is very dangerous. Of course, the files can be also directly deleted using the method of "Files: delete" of Drive API. Ref
From your question, I thought that there might be a lot of files you want to delete. So in this answer, I would like to suggest to use the batch requests.
When above flow is reflected to the script, it becomes as follows.
IMPORTANT: PLEASE BE CAREFUL!
When you use this sample script, please be careful this. I would like to recommend the following flow for using this script.
Retrieve the file list of is:unorganized owner:me and check whether all files of the file list are the files you want to delete.
You can retrieve the file list using const fileList = getFileList(token); in main() function.
When you could completely confirm that all files of the file list are the files you want to delete, please use const res = moveFilesToTrashBox(token, fileList);. By this, the files of file list are moved to the trash box.
Please confirm the trash box. When the files you don't want to delete are included, please restore them.
Sample script:
Before you use this script, please enable Drive API at Advanced Google services. And please run main() function.
// Move the files in the file list to the trash box.
function moveFilesToTrashBox(token, fileList) {
const limit = 100;
const split = Math.ceil(fileList.length / limit);
const res = [];
for (let i = 0; i < split; i++) {
const boundary = "xxxxxxxxxx";
const payload = fileList.splice(0, limit).reduce((s, {id}, i, a) => s += "Content-Type: application/http\r\n" +
"Content-ID: " + i + "\r\n\r\n" +
"PATCH https://www.googleapis.com/drive/v3/files/" + id + "\r\n" +
"Content-Type: application/json\r\n\r\n" +
JSON.stringify({trashed: true}) + "\r\n" +
"--" + boundary + (i == a.length - 1 ? "--" : "") + "\r\n" , "--" + boundary + "\r\n");
const params = {
method: "post",
contentType: "multipart/mixed; boundary=" + boundary,
payload: payload,
headers: {Authorization: "Bearer " + token},
muteHttpExceptions: true,
};
const r = UrlFetchApp.fetch("https://www.googleapis.com/batch/drive/v3", params);
res.push(r.getContentText());
}
return res;
}
// Retrieve the file list by searching with "is:unorganized owner:me".
function getFileList(token) {
const fields = decodeURIComponent("nextPageToken,files(name,id,mimeType,parents)");
const q = decodeURIComponent("'me' in owners and trashed = false");
let allFiles = [];
let pageToken = "";
do {
const res = UrlFetchApp.fetch(
`https://www.googleapis.com/drive/v3/files?pageSize=1000&fields=${fields}&q=${q}&pageToken=${pageToken}`,
{ headers: { authorization: `Bearer ${token}` } }
);
const obj = JSON.parse(res);
allFiles = allFiles.concat(obj.files);
pageToken = obj.nextPageToken;
} while (pageToken);
return allFiles.filter(({ parents }) => !parents);
}
// Please run this function.
function main() {
const token = ScriptApp.getOAuthToken();
// Retrieve the file list of all files in the Google Drive.
const fileList = getFileList(token);
console.log(fileList.length);
console.log(fileList);
// Move the files to the trash box using the retrieved file list.
// When you could completely confirm that all files of the file list are the files you want to delete, please use the below script.
// const res = moveFilesToTrashBox(token, fileList);
// console.log(res);
// DriveApp.createFile(); // This is used for automatically adding a scope of "https://www.googleapis.com/auth/drive".
}
References:
Files: list
Files: update
Batch request

Google Cloud Functions: require(...) is not a function

I am trying to deploy a Google Cloud function, I started by just adding the initial requirements to my index.js file:
// Import the Google Cloud client libraries
const nl = require('#google-cloud/language')();
const speech = require('#google-cloud/speech')();
const storage = require('#google-cloud/storage')();
But I get the following message when deploying:
Detailed stack trace: TypeError: require(...) is not a function
This only happens with the #google-cloud/speech and #google-cloud/language modules, the #google-cloud/storage module is loaded fine as a function (I tested by commenting the first two).
Any advise will be greatly appreciated.
Borrigan
With reference to this Github comment, there was some changes in google-cloud v2 package
so you import packages such as:
const {Storage} = require('#google-cloud/storage');
const storage = new Storage({
// config...
});
Google cloud function are nodejs modules so the syntax is same as nodejs syntax.
Regarding your problem:
you have to write
const storage = require('#google-cloud/storage');
(without () at the end of each statement)
So the correct declaration will be:
// Import the Google Cloud client libraries
const nl = require('#google-cloud/language');
const speech = require('#google-cloud/speech');
const storage = require('#google-cloud/storage');
I hope this helps.
It tells you that whatever you required is not a function and therefore can't be invoked with ()
if you look here : https://www.npmjs.com/package/#google-cloud/language#using-the-client-library
you see a service object with multiple class returning functions is being returned, so you should set it up like this:
const nl = require('#google-cloud/language');
const language = new nl.LanguageServiceClient();
Follow the new syntax below:
const {Storage} = require('#google-cloud/storage');
const googleCloudStorage = new Storage({
projectId: 'your-project-id',
keyFilename: 'path-to-json-config-file'
});

Google Apps Script Execution API - Calling app is Apps Script - Apps Script to Apps Script

I have two scripts in different domains, I would like to fetch data from one to the other.
I've found Script Execution API to be useful, however they don't provide any example GAS to GAS. https://developers.google.com/apps-script/guides/rest/quickstart/target-script
Any suggestion/guidance on how to accomplish such thing?
This answer might be helpful. what i have done is that created a cloud platform project and created two standalone project, one contains the calling script and other has the method you want to call and must be deployed as a API Executable.
IMPORTANT: Both these project must be associated with the Cloud Platform Project.
For associating the projects to Cloud Platform's project , in script editor do this, Resources --> Cloud Platform Project. You'll see the dialog box which has input box having placeholder Enter Project Number Here, here put the Cloud Platform's project number and do the same for other project as well.
Now as i have followed google's quick start tutorial for EXECUTION API. i have a target script (Script containing your method) as follows:
/**
* The function in this script will be called by the Apps Script Execution API.
*/
/**
* Return the set of folder names contained in the user's root folder as an
* object (with folder IDs as keys).
* #return {Object} A set of folder names keyed by folder ID.
*/
function getFoldersUnderRoot() {
var root = DriveApp.getRootFolder();
var folders = root.getFolders();
var folderSet = {};
while (folders.hasNext()) {
var folder = folders.next();
folderSet[folder.getId()] = folder.getName();
}
return folderSet;
}
NOTE: This script must be deployed as a API Executable. In project you must have enabled the Google Execution API.
Now the calling script,
function makeRequest(){
// DriveApp.getRootFolder();
var access_token=ScriptApp.getOAuthToken();
var url = "https://script.googleapis.com/v1/scripts/{YourScriptId}:run";
var headers = {
"Authorization": "Bearer "+access_token,
"Content-Type": "application/json"
};
var payload = {
'function': 'getFoldersUnderRoot',
devMode: true
};
var res = UrlFetchApp.fetch(url, {
method: "post",
headers: headers,
payload: JSON.stringify(payload),
muteHttpExceptions: true
});
Logger.log(res);
}
On first line inside function you can see i have commented out the Drive API call. I did that because the calling script and target script must have the same scope.
Yes you can use Execution API to fetch data from one domain to other. From the documentation, it consists of a single scripts resource, which has a single method, run, that makes calls to specific Apps Script functions.
The run method must be given the following information when called:
The ID of the script being called.
The name of the function within the script to execute.
The list of parameters the function requires (if any).
Here is an example from GitHub:
var script = google.script('v1');
var key = require('./creds.json');
var jwtClient = new google.auth.JWT(key.client_email, null, key.private_key, ['https://www.googleapis.com/auth/spreadsheets'], null);
jwtClient.authorize(function(err, tokens) {
if (err) {
console.log(err);
return;
}
script.scripts.run({
auth: jwtClient,
scriptId: 'script-id',
resource: {
function: 'test',
parameters: []
}
}, function(err, response) {
if (err) {
throw err;
}
console.log('response', response);
})
});
You can also check this related SO question on how to use Google Apps Script Execution API and Javascript to make the interaction between website and Google Drive.