I am trying to upload a bunch of zip files to a Team Drive through Python API. The weird thing is that the program works if the zip file has extension "csv.gz" but fails if the extension is ".zip". Here is the code:
from __future__ import print_function
import uuid
from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
from apiclient import errors
from apiclient.http import MediaFileUpload
args = tools.argparser.parse_args()
args.noauth_local_webserver = True
SCOPES = 'https://www.googleapis.com/auth/drive'
store = file.Storage('storage.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
creds = tools.run_flow(flow, store, args)
DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))
def upload_file_to_td_folder(folder_id, fn, mimeType):
body = {'name': fn, 'mimeType': mimeType, 'parents': [folder_id]}
return DRIVE.files().create(body=body, media_body=fn,
supportsTeamDrives=True, fields='id').execute().get('id')
FILE_MIME = 'application/vnd.google-apps.file'
folder_id = "abcdefg"
# file = "data_2018-09-07.csv.gz" this one works!
file = "data_2018-09-07.zip"
upload_file_to_td_folder(folder_id, file, FILE_MIME)
Here is the error message:
Traceback (most recent call last):
File "test_td.py", line 103, in <module>
upload_file_to_td_folder(folder_id, file, FILE_MIME)
File "test_td.py", line 46, in upload_file_to_td_folder
supportsTeamDrives=True, fields='id').execute().get('id')
File "/home/bo/anaconda3/lib/python3.6/site-packages/oauth2client/_helpers.py", line 133, in positional_wrapper
return wrapped(*args, **kwargs)
File "/home/bo/anaconda3/lib/python3.6/site-packages/googleapiclient/http.py", line 840, in execute
raise HttpError(resp, content, uri=self.uri) googleapiclient.errors.HttpError:
<HttpError 400 when requesting https://www.googleapis.com/upload/drive/v3/files?supportsTeamDrives=true&fields=id&alt=json&uploadType=multipart
returned "Bad Request">
I think the problem is the type of file extension used here.
file = "data_2018-09-07.csv.gz"
Gzip is a file format for storing a single compressed file.
file = "data_2018-09-07.zip"
ZIP is a file format used for storing arbitrary number of files and folders together with lossless compression.
It means gzip is only capable to compress a single file while zip compresses multiple files one by one and archives them into a single file afterwards.
So if you need a single file inside a big archive, you have to decompress the whole zip/gzip file to get the file.
In your case, you have specified the file as a csv.gz, which is more effective as compressed as a single file.
You can try here the ZIP Extractor extension enables efficient management of zip archives within Drive.
Related
I am working on this last 3 days. i don't have any idea.. there are similar question but i didn't got my answer.
this is ERROR
File ID: 1U4XUrAhMk1WFAKE_IDqmQcteYqmIWPMFEd Traceback (most recent
File is Created I want to USE this ID and Edit Docs in Drive
call last): File "main.py", line 61, in
main() File "main.py", line 53, in main
service.documents() AttributeError: 'Resource' object has no attribute 'documents'
My goal
create Docs in GOOGLE Drive
insert Table in it
from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import sys
from gdoctableapppy import gdoctableapp
# If modifying these scopes, delete the file token.pickle.
SCOPES = ["https://www.googleapis.com/auth/drive"]
def main():
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists("token.pickle"):
with open("token.pickle", "rb") as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file("credentials.json", SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open("token.pickle", "wb") as token:
pickle.dump(creds, token)
service = build("drive", "v3", credentials=creds)
serviceDoc = build("docs", "v1", credentials=creds)
# Call the Drive v3 API
# Create Google Docs file in folder
file_metadata = {
"name": sys.argv[1],
"parents": ["Folder ID"],
}
file = service.files().create(body=file_metadata, fields="id").execute()
print("File ID: %s" % file.get("id"))
DOCUMENT_ID = file.get("id")
requests = [{"insertTable": {"rows": 2, "columns": 2, "location": {"index": 1}}}]
result = (
service.documents()
.batchUpdate(documentId=DOCUMENT_ID, body={"requests": requests})
.execute()
)
return
if __name__ == "__main__":
main()
The reason why you are encountering such error is because your service variable is for Drive API, it doesn't have a documents() method.
Use serviceDoc instead:
serviceDoc.documents()
.batchUpdate(documentId=DOCUMENT_ID, body={"requests": requests})
.execute()
In addition:
I noticed that when you create a Docs file mimeType is not part of your file_metadata. If you create files without a specific mimeType, your newly created file will be application/octet-stream. See Create Files
If you want to create a Google Docs using Drive API, please add a "mimeType"='application/vnd.google-apps.document' in your file_metadata
Sample:
file_metadata = {
"name": sys.argv[1],
"mimeType"='application/vnd.google-apps.document',
"parents": ["Folder ID"]
}
Reference:
Google Workspace and Drive MIME Types
My google drive is linked to my google colab notebook. Using the pytorch library torch.load($PATH) fails to load this 219 Mo file (pre-trained neural network) (https://drive.google.com/drive/folders/1-9m4aVg8Hze0IsZRyxvm5gLybuRLJHv-) which is in my google drive. However it works fine when I do it locally on my computer. The error i get on google collab is: (settings: Python 3.6, pytorch 1.3.1):
state_dict = torch.load(model_path)['state_dict']
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 303, in load
return _load(f, map_location, pickle_module)
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 454, in _load
return legacy_load(f)
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 380, in legacy_load
with closing(tarfile.open(fileobj=f, mode='r:', format=tarfile.PAX_FORMAT)) as tar,
File "/usr/lib/python3.6/tarfile.py", line 1589, in open
return func(name, filemode, fileobj, **kwargs)
File "/usr/lib/python3.6/tarfile.py", line 1619, in taropen
return cls(name, mode, fileobj, **kwargs)
File "/usr/lib/python3.6/tarfile.py", line 1482, in init
self.firstmember = self.next()
File "/usr/lib/python3.6/tarfile.py", line 2297, in next
tarinfo = self.tarinfo.fromtarfile(self)
File "/usr/lib/python3.6/tarfile.py", line 1092, in fromtarfile
buf = tarfile.fileobj.read(BLOCKSIZE)
OSError: [Errno 5] Input/output error```
Any help would be much appreciated!
Large sized files are automatically analyzed for virus on Drive, every time you attempt to download a large file you have to pass thru this scan, making it hard to reach the download link.
You could download the file directly using the Drive API and then pass it to the torch, it shouldn't be hard to implement on Python, I've made a sample on how to Download your file and pass it to Torch.
import torch
import pickle
import os.path
import io
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.http import MediaIoBaseDownload
from __future__ import print_function
url = "https://drive.google.com/file/d/1RwpuwNPt_r0M5mQGEw18w-bCfKVwnZrs/view?usp=sharing"
# If modifying these scopes, delete the file token.pickle.
SCOPES = (
'https://www.googleapis.com/auth/drive',
)
def main():
"""Shows basic usage of the Sheets API.
Prints values from a sample spreadsheet.
"""
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
drive_service = build('drive', 'v2', credentials=creds)
file_id = '1RwpuwNPt_r0M5mQGEw18w-bCfKVwnZrs'
request = drive_service.files().get_media(fileId=file_id)
# fh = io.BytesIO()
fh = open('file', 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
fh.close()
torch.load('file')
if __name__ == '__main__':
main()
To run it you'll have first to:
Enable the Drive API for your account
Install the Google Drive API libraries,
This takes no more than 3 minutes and is properly explained on the Quickstart Guide for Google Drive API, just follow steps 1 and 2 and run the provided sample code from above.
It worked by uploading directly the file to google colab instead of loading it from google drive using:
from google.colab import files
uploaded= files.upload()
I guess this solution is similar to the one proposed by #Yuri
I have the following flask app where I want to be able to upload a TXT or TSV file to a form. The problem is, when I try to upload a TXT file, it works, but when I try to upload a TSV file, I get the following error:
File "/Users/cdastmalchi/Desktop/author_script/main.py", line 89, in process_file
if not places_exist(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
File "/Users/cdastmalchi/Desktop/author_script/main.py", line 27, in places_exist
infile = open(filename, 'rU')
IOError: [Errno 2] No such file or directory: './Authors_Template.tsv'
Authors_Template.tsv is a template file that gets downloaded from the form and goes into the Downloads, and then I want users to be able to edit this template and then re-upload it. When I make the template Authors_Template.txt instead and then Download and re-upload it, it works. How can I solve this problem? I've even tried narrowing down the ALLOWED_EXTENSIONS list to just TSV and I still get the same issue.
app.py
from werkzeug.utils import secure_filename
import flask, string, random
import json
import subprocess
import os
import re
import time
UPLOAD_FOLDER = '.'
ALLOWED_EXTENSIONS = set(['txt','tsv'])
app = flask.Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.secret_key = ''.join(random.choice(string.ascii_letters) for _ in range(20)) #needed to use flask.session
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
def places_exist(filename):
infile = open(filename, 'rU')
placeDict = {}
addresses_temp = []
addresses = []
places_temp =[]
places = []
places_exist = True
for i in infile:
item = i.rstrip("\n").split("\t")
places_temp.append(item[0])
addresses_temp.append(item[1])
p_index = (places_temp.index('Place')) + 1
a_index = (addresses_temp.index('Address')) + 1
places = places_temp[p_index:]
addresses = addresses_temp[a_index:]
infile.close()
infile = open(filename, 'rU')
return places_exist
#app.route('/', methods=['GET'])
def home():
return flask.render_template('index.html')
#app.route('/process_file', methods=['POST'])
def process_file():
#here, you can run all the checks as before, but instead of flash, you can return jsonified results to read in the front-end
if 'file' not in flask.request.files or not flask.request.files['file'].filename:
return flask.jsonify({'result':'False', 'message':'no files selected'})
return flask.redirect(url_for('home'))
file = flask.request.files['file']
filename = secure_filename(file.filename)
if not allowed_file(file.filename):
return flask.jsonify({'result':'False', 'message':'Must be TXT file!'})
return flask.redirect(url_for('home'))
if not places_exist(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
return flask.jsonify({'result':'False', 'message':'There is an affiliation missing from your Place list. Please re-try.'})
return flask.redirect(url_for('home'))
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
flask.session['filename'] = filename
return flask.jsonify({'result':'True'})
UPDATE:
def process_file():
#here, you can run all the checks as before, but instead of flash, you can return jsonified results to read in the front-end
if 'file' not in flask.request.files or not flask.request.files['file'].filename:
return flask.jsonify({'result':'False', 'message':'no files selected'})
return flask.redirect(url_for('home'))
file = flask.request.files['file']
filename = secure_filename(file.filename)
if not allowed_file(file.filename):
return flask.jsonify({'result':'False', 'message':'Must be TXT file!'})
return flask.redirect(url_for('home'))
# Save the file in the temp folder
file.save(os.path.join(app.config['TEMP_FOLDER'], filename))
# Process the file
if not places_exist(os.path.join(app.config['TEMP_FOLDER'], filename)):
return flask.jsonify({'result':'False', 'message':'There is an affiliation missing from your Place list. Please re-try.'})
return flask.redirect(url_for('home'))
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
flask.session['filename'] = filename
return flask.jsonify({'result':'True'})
You are trying to read a file before its writing in your directory. First you need to save the file in your application upload directory then read it.
def process_file():
# here, you can run all the checks as before, but instead of flash, you can return jsonified results to read in the front-end
if 'file' not in flask.request.files or not flask.request.files['file'].filename:
return flask.jsonify({'result':'False', 'message':'no files selected'})
return flask.redirect(url_for('home'))
file = flask.request.files['file']
filename = secure_filename(file.filename)
if not allowed_file(file.filename):
return flask.jsonify({'result':'False', 'message':'Must be TXT file!'})
return flask.redirect(url_for('home'))
# Save the file in the correct Location
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
# Process your file already saved
if not places_exist(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
return flask.jsonify({'result':'False', 'message':'There is an affiliation missing from your Place list. Please re-try.'})
return flask.redirect(url_for('home'))
flask.session['filename'] = filename
return flask.jsonify({'result':'True'})
EDIT: You have to be careful if you need to check the file before saving, if you save right away you will override your old file, a good approach will be to save the file in a temp location, check that file and then save in the final directory, and obviously delete the file in the tmp folder.
PS: Also you have 2 return, if you need to respond as a HTML or JSON you have to check the headers of the request.
I have a CSV file, which contains a list of Google extension IDs.
I'm writing a code that will read the extension IDs, add the webstore url, then perform a basic get request:
import csv
import requests
with open('small.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
urls = "https://chrome.google.com/webstore/detail/" + row[0]
print urls
r = requests.get([urls])
Running this code results in the following Traceback:
Traceback (most recent call last):
File "C:\Users\tom\Dropbox\Python\panya\test.py", line 9, in <module>
r = requests.get([urls])
File "C:\Python27\lib\site-packages\requests\api.py", line 69, in get
return request('get', url, params=params, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 567, in send
adapter = self.get_adapter(url=request.url)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 641, in get_adapter
raise InvalidSchema("No connection adapters were found for '%s'" % url)
InvalidSchema: No connection adapters were found for '['https://chrome.google.com/webstore/detail/blpcfgokakmgnkcojhhkbfbldkacnbeo']'
How can revise the code, so that it would accept the urls in the list, and make the GET request?
requests.get expects a string, but you're creating and passing a list [urls]
r = requests.get([urls])
Change it to just
r = requests.get(urls)
and it should work.
I am trying to download file from sharepoint using python library. Code is connecting and getting content of sharepoint perfectly but not downloading particular file.
here is script:
import requests
from requests_ntlm import HttpNtlmAuth
headers = {'accept': 'application/json;odata=verbose'}
r = requests.get("https://abc.we.x/abd.doc", auth=HttpNtlmAuth('domain\\User','ppusers#123'),headers=headers)
print r.json()["d"]["CustomMasterUrl"]
if we print j.content then i can see all html content of this page but r.json()["d"]["CustomMasterUrl"] line giving error as below
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Python27\lib\site-packages\requests\models.py", line 819, in json
return json.loads(self.text, **kwargs)
File "C:\Python27\lib\json\__init__.py", line 338, in loads
return _default_decoder.decode(s)
File "C:\Python27\lib\json\decoder.py", line 366, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Python27\lib\json\decoder.py", line 384, in raw_decode
raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded
Please help in downloading file if someone has other way to do that.
Thanks in advance.
I think you can use cmislib to download files from sharepoint
client = CmisClient(serverurl, username, password)
repo = client.defaultRepository
doc = repo.getObjectbyPath("PathOfFile")
result = doc.getContentStream()
c = result.read()
docFile = open("/Destination/File/Path"+doc.properties['cmis:name'],"w")
docFile.write(c)
docFile.close()
This part of code works fine with an Alfresco Server, but sharepoint should also support cmis. https://msdn.microsoft.com/en-US/library/office/jj945829.aspx
Maybe this can help you.
Thanks for all response. Here is working solution. Basically we were missing stream writing to file.
import requests
from requests_ntlm import HttpNtlmAuth
headers = {'accept': 'application/json;odata=verbose'}
url = "https://docs.qmad.er.com/sites/cloud_BPAP/Shared%20Documents/Database%20Deliverables/20150319_Maintenance_BPAP15/20150319_Maintenance.zip"
username = 'abc\\ert' #domain\\username
password = 'User#123'
r = requests.get(url, auth=HttpNtlmAuth(username,password),stream=True)
fh = open("clientNet.zip", "wb")
fh.write(r.content)
fh.close()