Google Apps Script: Downloading files from Drive (same user) - google-apps-script

I'm trying to write a Google Apps Script to download all files in a particular Drive folder (likely .csv files). I have found the getDownloadUrl() method but I haven't been able to figure out what to do with it. I'm currently trying the following code, where files is the list of the files in the folder:
while(files.hasNext()) {
var response = UrlFetchApp.fetch(files.next().getDownloadUrl());
Logger.log(response.getContentText());
}
When I try to run the code, however, I get a 401 error which I guess means I lack the proper authorization? But I was under the impression that I wouldn't need to go through all of the OAuth2 steps if everything was taking place within my one Google account. The Google guide to connecting to external APIs makes it look like I should be able to just fetch the url. I've already gotten access to my Drive files, because the download URL does exist when I run that method. What am I missing here? I'm really new to all of this so maybe it's something basic.
Thanks!
EDIT:
I managed to fix the 401 error by modifying the code as follows:
while(files.hasNext()) {
var response = UrlFetchApp.fetch(files.next().getDownloadUrl(),{headers: {Authorization: "Bearer " + ScriptApp.getOAuthToken()}});
Logger.log(response.getContentText());
}
But the issue remains that this only returns the contents to me, rather than downloading the file. How can I initiate a download from the results of this fetch call?

Besides listing all download links, I guess original poster also wants to download files to user's computer (according to earlier discussion).
To do this, encode blob with base 64 in server side (e.g. Google App Script) and download with data URI in client's browser. Below are code for this, with help of this answer.
Google App Script
...
function getBlobInBase64(fileId){
// omit authorization code if any
var file = DriveApp.getFileById(fileId);
var blob = file .getBlob();
return {
file_name: file.getName(),
mime: file.getMimeType(),
b64: Utilities.base64Encode(blob.getBytes());
}
...
Javascript that serve with index.html
...
function getFile(fileId){
google.script.run.withSuccessHandler((data) => {
var uri = 'data:' + data.mime + ';charset=ISO-8859-1;base64,' + encodeURIComponent(data.b64);
downloadURI(uri, data.file_name);
}).withFailureHandler((err) => {
console.log(err);
}).getBlobInBase64();
}
...
NOTE: I haven't run this code but the method should work as used in my other project.

This will log the file names & URLS for any files available for downloading (first 100 in root drive):
function myFunction() {
var files = DriveApp.getFiles();
var c = 0;
while (files.hasNext() && c<100) {
var file = files.next();
Logger.log("File Name: " + file.getName());
Logger.log(" Download URL: " + file.getDownloadUrl());
c++;
}
}

My answer might be a bit off but I think you have a better chance downloading files from Google Drive using the webContentLink as it is the method I commonly use. I obtain webContentLink by using Files.list and ask for webContentLink in the fields parameter. I run that link through the browser and it downloads the file.

If you are trying to download Google Drive files to local computer using Google Apps Script, Then please understand that Google Apps Script is a server side scripting language. It can't download and save files to your local drive.

Here is a webapp that may be helpful for you. It does not do exactly what you are looking for but you may be able to edit it and get a result. Hope it helps!
CODE:
function doGet(e) { // main function
var template = HtmlService.createTemplateFromFile('index.html'); // filename always!
return template.evaluate().setTitle('Search Drive').setSandboxMode(HtmlService.SandboxMode.IFRAME);
}
// Process the form
function processForm(searchTerm) {
var resultToReturn;
Logger.log('processForm was called! ' + searchTerm);
resultToReturn = SearchFiles(searchTerm); // Call to the search files function to search files on google drive
Logger.log('resultToReturn: ' + resultToReturn);
return resultToReturn; // return the results
}
function SearchFiles(searchTerm) {
var searchFor ="title contains '" + searchTerm + "'"; //single quotes are needed around searchterm
var owneris ="and 'YOUREmail#email.com' in Owners"; //email address to search for
var names = [];
Logger.log(searchFor + " " + owneris);
var files = DriveApp.searchFiles(searchFor + " " + owneris);
while (files.hasNext()) {
var file = files.next();
var fileId = file.getId();// To get FileId of the file
var lm = file.getLastUpdated();
var name = file.getName()+"|~|"+fileId+"|~|"+lm; // Im concatenating the filename with file id separated by |~|
names.push(name); // adding to the array
}
return names; // return results
}
INDEX.html
<html>
<head>
<base target="_top">
<script>
function displayMessage() {
var searchTerm;
searchTerm = document.getElementById('idSrchTerm').value;
console.log('searchTerm: ' + searchTerm );
// Below call means: call to processForm passing the searchTerm value (previously escaped) and after finish call the handleResults function
google.script.run.withSuccessHandler(handleResults).processForm(searchTerm.replace("'","\'"));
}
function handleResults(results){
console.log('Handle Results was called! ');
document.writeln('BACK<br/><br/>');
var length=results.length; // total elements of results
for(var i=0;i<length;i++)
{
var item=results[i];
item=item.split("|~|"); // split the line |~|, position 0 has the filename and 1 the file id
document.writeln("<b><a href='https://docs.google.com/document/d/"+item[1]+"' target='_blank'>"+item[0]+"</b></a> (Last modified: "+item[2]+")<br/><br/>"); // write result
}
document.writeln("End of results...");
}
</script>
</head>
<body><center><br/>
Search: <input type="text" id="idSrchTerm" name="search">
<input type="button" value="search files on Google Drive" name="submitButton" onclick="displayMessage()"/>
</center>
</body>
</html>

Related

unable to copy the gmail file to google sheets. It throws the error as:API call to drive.files.insert failed with error: Invalid mime type provided [duplicate]

This is a continuation from How to Use Advanced Drive Service to Upload Files.
My Webapp consists of an upload form for data files, which are then stored on Google Drive. (Full code in snippet below.)
I'm having a problem with the following line of code:
var file = Drive.Files.insert(resource, mediaData); // create file using Drive API
try {
//Get root folder and pull all existing folders, plus setup variables pulled from form
var dropbox = form.Country;
var timeStamp = new Date();
//Set file name slightly differently for Weekly Member Report (do not want to overright based on name just keep each extract so add timestamp to name)
if (form.reportType == "Member Weekly"){
var filename = form.reportType + timeStamp + ".xls";
}
else
{
var filename = form.reportType+".xls";
}
var rootfolder = DriveApp.getFolderById("0Byvtwn42HsoxfnVoSjB2NWprYnRiQ2VWUDZEendNOWwwM1FOZk1EVnJOU3BxQXhwU0pDSE0");
//Note root folder is Live Uploads Folder in Flatworld App folder structure
var folder, folders = rootfolder.getFoldersByName(dropbox);
//Check if folder exists and if not create
if (folders.hasNext()) {
folder = folders.next();
} else {
folder = rootfolder.createFolder(dropbox);
}
var FolderURL = folder.getUrl(); // Retain URL of folder for final end message to user
//Check if file already exists and delete if it does
var file, files = folder.getFilesByName(filename);
while( files.hasNext()){
file = files.next();
file.setTrashed(true);
}
//New Code from Stackover Flow:
//Upload file and set various properties
var mediaData = form.myFile;
var resource = {
title: filename,
mimetype: 'application/vnd.ms-excel',
description: "Uploaded via BNI Upload Form by " + form.myName + " on: " + timeStamp
};
var file = Drive.Files.insert(resource, mediaData); // create file using Drive API
var fileId = file.id;
var DriveAppFile = DriveApp.getFileById(fileId); // retrieve file in DriveApp scope.
var FileURL = DriveAppFile.getUrl(); //Retain URL of file for final end message to user
DriveApp.removeFile(DriveAppFile); // remove new file from Users root My Drive
folder.addFile(DriveAppFile); // puts file in selected folder
//End of New code from Stackover Flow
//Success message displayed to user
return "Thanks! File uploaded successfully to: <br><br><b>Folder Location:</b> " + FolderURL + "<br>" + "<b>File:</b> " + FileURL + ". <br><br>For any queries please email user#example.com copying the URLs displayed here in your email. You can close this browser window now or use the link below to upload another file.<br><br>";
} catch (error) {
//Catch error return it to user and email with error details
Its throwing the error message "Empty Response" on the line of code above when we try and upload a large file (15MB) Do you have any suggestions. This is well inside the Files insert limit of 5120GB and the code works fine on smaller files.
I've now tried to add in a loop in function to try the upload a couple of times, still throwing the same error sadly:
//setup function that will return null if file is not uploaded correctly
function createDriveFile(resource_f, mediaData_f){
try{
var file = Drive.Files.insert(resource_f, mediaData_f); // create file using Drive API
return file;
} catch (e) {return null;}
}
//try upload and loop if null
var maxTries = 3;
var tries = 0;
do {
tries = tries + 1;
if (tries > 0) {
var file = createDriveFile(resource, mediaData);
Logger.log("I'm trying to upload, try number: " + tries);
}
} while ((file == null) && (tries < maxTries));
if (file == null) {
var file = Drive.Files.insert(resource, mediaData); // try one laste time to create file using Drive API - outside loop function so if error is thrown script stops
}
The error only seems to occur on a larger file, even if we reduce the size of the same file that solves error so do we need to adjust the upload process to account for a larger file. Is there a Google Apps Script equivalent of making the API upload request resumable?
Your file size is the determinant factor here. Referencing the documentation suggests the simple upload method used here is good for up to 5MB only.
https://developers.google.com/drive/web/manage-uploads
Your comments seem to confirm this is what is happening for you.
As you hinted, use the resumable upload method. Use the uploadType: resumable parameter flag – API docs on the insert method describes how.
You can also check the naming of the file, for me I had a slash in the name which is why it would not upload. So take away any special characters before uploading and it should work.

Delete multiple google drive files that are unorganized / orphaned

I wanted to remove several thousand files in a Google Drive folder. I deleted the folder, which many in the community know (but I didn't) leaves the files present but orphaned. I can list the files via "is:unorganized owner:me" but can't select more than a few hundred at a time to (slowly) delete.
Is there a script that will search for and delete these, and only these, files?
Thanks
I believe your goal as follows.
You want to retrieve the files searched with is:unorganized owner:me and want to delete the files.
You want to reduce the process cost of this process.
Issue and workaround:
In the current stage, unfortunately, the files cannot be directly retrieved by searching is:unorganized owner:me with Drive API and Drive service. So as the current workaround, I would like to suggest the following flow.
Retrieve the file list of all files in the Google Drive. In this case, 'me' in owners and trashed = false is used as the search query. The method of "Files: list" of Drive API is used for this.
Retrieve the file list of files which have no parent folder using a script.
By above flow 1 and 2, the search of is:unorganized owner:me can be achieved using a script.
Move the files to the trash box using the retrieved file list using the method of "Files: update" of Drive API.
In this case, the files are not directly deleted. Because when the files are directly deleted, I thought that this situation is very dangerous. Of course, the files can be also directly deleted using the method of "Files: delete" of Drive API. Ref
From your question, I thought that there might be a lot of files you want to delete. So in this answer, I would like to suggest to use the batch requests.
When above flow is reflected to the script, it becomes as follows.
IMPORTANT: PLEASE BE CAREFUL!
When you use this sample script, please be careful this. I would like to recommend the following flow for using this script.
Retrieve the file list of is:unorganized owner:me and check whether all files of the file list are the files you want to delete.
You can retrieve the file list using const fileList = getFileList(token); in main() function.
When you could completely confirm that all files of the file list are the files you want to delete, please use const res = moveFilesToTrashBox(token, fileList);. By this, the files of file list are moved to the trash box.
Please confirm the trash box. When the files you don't want to delete are included, please restore them.
Sample script:
Before you use this script, please enable Drive API at Advanced Google services. And please run main() function.
// Move the files in the file list to the trash box.
function moveFilesToTrashBox(token, fileList) {
const limit = 100;
const split = Math.ceil(fileList.length / limit);
const res = [];
for (let i = 0; i < split; i++) {
const boundary = "xxxxxxxxxx";
const payload = fileList.splice(0, limit).reduce((s, {id}, i, a) => s += "Content-Type: application/http\r\n" +
"Content-ID: " + i + "\r\n\r\n" +
"PATCH https://www.googleapis.com/drive/v3/files/" + id + "\r\n" +
"Content-Type: application/json\r\n\r\n" +
JSON.stringify({trashed: true}) + "\r\n" +
"--" + boundary + (i == a.length - 1 ? "--" : "") + "\r\n" , "--" + boundary + "\r\n");
const params = {
method: "post",
contentType: "multipart/mixed; boundary=" + boundary,
payload: payload,
headers: {Authorization: "Bearer " + token},
muteHttpExceptions: true,
};
const r = UrlFetchApp.fetch("https://www.googleapis.com/batch/drive/v3", params);
res.push(r.getContentText());
}
return res;
}
// Retrieve the file list by searching with "is:unorganized owner:me".
function getFileList(token) {
const fields = decodeURIComponent("nextPageToken,files(name,id,mimeType,parents)");
const q = decodeURIComponent("'me' in owners and trashed = false");
let allFiles = [];
let pageToken = "";
do {
const res = UrlFetchApp.fetch(
`https://www.googleapis.com/drive/v3/files?pageSize=1000&fields=${fields}&q=${q}&pageToken=${pageToken}`,
{ headers: { authorization: `Bearer ${token}` } }
);
const obj = JSON.parse(res);
allFiles = allFiles.concat(obj.files);
pageToken = obj.nextPageToken;
} while (pageToken);
return allFiles.filter(({ parents }) => !parents);
}
// Please run this function.
function main() {
const token = ScriptApp.getOAuthToken();
// Retrieve the file list of all files in the Google Drive.
const fileList = getFileList(token);
console.log(fileList.length);
console.log(fileList);
// Move the files to the trash box using the retrieved file list.
// When you could completely confirm that all files of the file list are the files you want to delete, please use the below script.
// const res = moveFilesToTrashBox(token, fileList);
// console.log(res);
// DriveApp.createFile(); // This is used for automatically adding a scope of "https://www.googleapis.com/auth/drive".
}
References:
Files: list
Files: update
Batch request

Advanced Drive Service returning Empty Response Error when inserting file

This is a continuation from How to Use Advanced Drive Service to Upload Files.
My Webapp consists of an upload form for data files, which are then stored on Google Drive. (Full code in snippet below.)
I'm having a problem with the following line of code:
var file = Drive.Files.insert(resource, mediaData); // create file using Drive API
try {
//Get root folder and pull all existing folders, plus setup variables pulled from form
var dropbox = form.Country;
var timeStamp = new Date();
//Set file name slightly differently for Weekly Member Report (do not want to overright based on name just keep each extract so add timestamp to name)
if (form.reportType == "Member Weekly"){
var filename = form.reportType + timeStamp + ".xls";
}
else
{
var filename = form.reportType+".xls";
}
var rootfolder = DriveApp.getFolderById("0Byvtwn42HsoxfnVoSjB2NWprYnRiQ2VWUDZEendNOWwwM1FOZk1EVnJOU3BxQXhwU0pDSE0");
//Note root folder is Live Uploads Folder in Flatworld App folder structure
var folder, folders = rootfolder.getFoldersByName(dropbox);
//Check if folder exists and if not create
if (folders.hasNext()) {
folder = folders.next();
} else {
folder = rootfolder.createFolder(dropbox);
}
var FolderURL = folder.getUrl(); // Retain URL of folder for final end message to user
//Check if file already exists and delete if it does
var file, files = folder.getFilesByName(filename);
while( files.hasNext()){
file = files.next();
file.setTrashed(true);
}
//New Code from Stackover Flow:
//Upload file and set various properties
var mediaData = form.myFile;
var resource = {
title: filename,
mimetype: 'application/vnd.ms-excel',
description: "Uploaded via BNI Upload Form by " + form.myName + " on: " + timeStamp
};
var file = Drive.Files.insert(resource, mediaData); // create file using Drive API
var fileId = file.id;
var DriveAppFile = DriveApp.getFileById(fileId); // retrieve file in DriveApp scope.
var FileURL = DriveAppFile.getUrl(); //Retain URL of file for final end message to user
DriveApp.removeFile(DriveAppFile); // remove new file from Users root My Drive
folder.addFile(DriveAppFile); // puts file in selected folder
//End of New code from Stackover Flow
//Success message displayed to user
return "Thanks! File uploaded successfully to: <br><br><b>Folder Location:</b> " + FolderURL + "<br>" + "<b>File:</b> " + FileURL + ". <br><br>For any queries please email user#example.com copying the URLs displayed here in your email. You can close this browser window now or use the link below to upload another file.<br><br>";
} catch (error) {
//Catch error return it to user and email with error details
Its throwing the error message "Empty Response" on the line of code above when we try and upload a large file (15MB) Do you have any suggestions. This is well inside the Files insert limit of 5120GB and the code works fine on smaller files.
I've now tried to add in a loop in function to try the upload a couple of times, still throwing the same error sadly:
//setup function that will return null if file is not uploaded correctly
function createDriveFile(resource_f, mediaData_f){
try{
var file = Drive.Files.insert(resource_f, mediaData_f); // create file using Drive API
return file;
} catch (e) {return null;}
}
//try upload and loop if null
var maxTries = 3;
var tries = 0;
do {
tries = tries + 1;
if (tries > 0) {
var file = createDriveFile(resource, mediaData);
Logger.log("I'm trying to upload, try number: " + tries);
}
} while ((file == null) && (tries < maxTries));
if (file == null) {
var file = Drive.Files.insert(resource, mediaData); // try one laste time to create file using Drive API - outside loop function so if error is thrown script stops
}
The error only seems to occur on a larger file, even if we reduce the size of the same file that solves error so do we need to adjust the upload process to account for a larger file. Is there a Google Apps Script equivalent of making the API upload request resumable?
Your file size is the determinant factor here. Referencing the documentation suggests the simple upload method used here is good for up to 5MB only.
https://developers.google.com/drive/web/manage-uploads
Your comments seem to confirm this is what is happening for you.
As you hinted, use the resumable upload method. Use the uploadType: resumable parameter flag – API docs on the insert method describes how.
You can also check the naming of the file, for me I had a slash in the name which is why it would not upload. So take away any special characters before uploading and it should work.

Migrating Code from DocList to DriveApp

I am trying to port the code below to DriveApp but the "doc.append" function does not work when I migrate to "DriveApp.append".
function process(thread, threadStart, folder, pass){
var start = Date.now();
var label = folder.getName();
var html;
if(pass > 1){
var docID = folder.createFile(label + "(part " + pass + ")", '<html>', MimeType.HTML).getFolderById();
}
else{
var docID = folder.createFile(label + "(part 1)", "<html>", MimeType.HTML).getFolderById();
}
var doc = DocsList.getFolderById(docID);
try{
doc.append(globalTOC(total_messages(thread), thread.length, label));
}
catch(exception){
Utilities.sleep(5000);
doc.append(globalTOC(total_messages(thread), thread.length, label));
}
The code you have posted has a few issues.
Firstly, it seems to be confusing Folders, Files and File IDs. The first part creates a File in a Folder, but then tries to call getFolderById(). Files do not have a method of this name. It then tries to acquire a File from that ID. If you want a File and its ID, you should just use the original File and call getId() on that:
var myDoc = folder.createFile(myFileName,
myHTMLcontents, MimeType.HTML);
var myDocID = myDoc.getId();
The above will work if you are using DriveApp Files or DocsList Files (which are different objects and should not be used interchangeably).
Secondly, there is currently no append() function available through DriveApp.File. If you need append functionality, one way to do it is to extract the File's contents as string, append to that string, and reset the contents with the new string:
var blob = doc.getBlob();
var content = blob.getDataAsString();
content += ' NEW CONTENT\n';
doc.setContent(content);
Note that setContent() will throw an exception if the content exceeds 10MB.
An alternate approach would be to build your file as a Google Doc, appending paragraphs as needed, and eventually covert that Doc to the file type you need.
Ryan Rith wrote 'DriveApp Files or DocsList Files are different objects and should not be used interchangeably'
Maybe the andwer I provided at migrating from docslist to driveapp can help while trying to port the code from DocsList to DriveApp

downloading files from URL with ashx

Just started using google scripts and can't seem to find the solution for downloading a file. Here's the issue:
Running a script on 100+ URL's daily
Each URL is .ashx that downloads the file automatically
The script included downloads a file, but not the file type it should be (.xls)
The only workaround is to manually change the file extension in Google Drive to .xls in order to view it correctly
Question: is there anyway to save the file from the .ashx URL as a .xls while running the script? I'm totally fine doing something totally different but need to be able to download these files as .xls' on download.
Thanks for any sugestions,
Jason
function getFileFromURL(fileURL, folder) {
var rc = 404;
var fileName = "";
var fileSize = 0;
try {
var response = UrlFetchApp.fetch("http://www.test.com/InventoryXLS?viewid=450");
var rc = response.getResponseCode();
} catch (e)
{
// fetch() does not handle unresolved DNS or file not found errors
// We'll treat all unhandled errors as "404 Not Found"
// This catch block simply suppresses the error
debugger;
}
if (rc == 200) {
var fileBlob = response.getBlob()
var folder = DocsList.getFolder("MyFolderNameinDrive");
if (folder != null) {
var file = folder.createFile(fileBlob);
fileName = file.getName();
fileSize = file.getSize();
}
}
var retObj = { "rc":rc, "fileName":fileName, "fileSize":fileSize };
debugger; // Stop to observe if in debugger
return retObj
}
Without testing on a .ashx file itself, I have no idea if this will work, but a couple of weeks ago the ability to convert a file with apps script was implemented. See number 15 on this issue.
Then again, you may not wish to convert the file to a Google Sheet. If so, just leave out the optional parameter that contains {convert: true}. Hopefully, this will still be able to set the name and mimeType of your file correctly. Otherwise, I'll delete this answer and we'll start over!
So perhaps something like:
var file = {
title: 'Converted Spreadsheet',
mimeType: 'application/vnd.ms-excel'
};
file = Drive.Files.insert(file, fileBlob, {
convert: true
});
If you could share a .ashx file, I could do some more testing...