import urllib2
import urllib
import json
url = "http://ajax/googleapis.com/ajax/services/search/web?v=1.0&"
query = raw_input ("What do you want to search for ? >> ")
query = urllib.urlencode({'q': query})
response = urllib2.urlopen (url + query).read()
data = json.loads (response)
results = data ['responseData'] ['results']
for result in results:
title = result['title']
url = result['url']
print (title + ';' + url)
ERROR
/System/Library/Frameworks/Python.framework/Versions/2.6/bin/python2.6 /Users/dragonleo/PycharmProjects/untitled2/googleapi
What do you want to search for ? >> apple
Traceback (most recent call last):
File "/Users/dragonleo/PycharmProjects/untitled2/googleapi", line 8, in
response = urllib2.urlopen (url + query).read()
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 1181, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 1156, in do_open
raise URLError(err)
urllib2.URLError:
Appreciate if expert can explain why I am getting the error
Two problems stand out immediately:
There are multiple typos in the code above. Specifically, there are no spaces between brackets and parens. Also, the URL should be ajax.googleapis.com.
The Google Web Search API is no longer available. You should migrate to the Google Custom Search API
Related
I'm trying to use marshmallow-sqlalchemy with aiohttp and I have followed their docs with the basic example and I'm getting an error.
I have this schema:
from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
from db.customer import Customer
class CustomerSchema(SQLAlchemyAutoSchema):
class Meta:
model = Customer
include_relationships = True
load_instance = True
And then the following code for the query:
from sqlalchemy import select
from db import db_conn
from db.customer import Customer
from queries.schema import CustomerSchema
customer_schema = CustomerSchema()
async def get_all_users():
async with db_conn.get_async_sa_session() as session:
statement = select(Customer)
results = await session.execute(statement)
_ = (results.scalars().all())
print(_)
response = customer_schema.dump(_, many=True)
print(response)
For the first print statement I'm getting
[<db.customer.Customer object at 0x10a183340>, <db.customer.Customer object at 0x10a183940>, <db.customer.Customer object at 0x10b0cd9d0>]
But then it fails with
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 60, in await_only
raise exc.MissingGreenlet(
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: http://sqlalche.me/e/14/xd2s)
So how can I use marshmallow-sqlalchemy to serialize the SqlAlchemy reponse?
Another options (packages, etc) or a generic custom solutions are OK too.
For the time being I'm using this:
statement = select(Customer)
results = await session.execute(statement)
_ = (results.scalars().all())
response = {}
for result in _:
value = {k: (v if not isinstance(v, sqlalchemy.orm.state.InstanceState) else '_') for k, v in result.__dict__.items()}
response[f'customer {value["id"]}'] = value
return response
Full traceback:
Traceback (most recent call last):
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_protocol.py", line 422, in _handle_request
resp = await self._request_handler(request)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_app.py", line 499, in _handle
resp = await handler(request)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_urldispatcher.py", line 948, in _iter
resp = await method()
File "/Users/ruslan/OneDrive/Home/Dev/projects/code/education/other/cft/views/user.py", line 24, in get
await get_all_users()
File "/Users/ruslan/OneDrive/Home/Dev/projects/code/education/other/cft/queries/user.py", line 18, in get_all_users
response = customer_schema.dump(_, many=True)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 547, in dump
result = self._serialize(processed_obj, many=many)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 509, in _serialize
return [
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 510, in <listcomp>
self._serialize(d, many=False)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 515, in _serialize
value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/fields.py", line 310, in serialize
value = self.get_value(obj, attr, accessor=accessor)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow_sqlalchemy/fields.py", line 27, in get_value
return super(fields.List, self).get_value(obj, attr, accessor=accessor)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/fields.py", line 239, in get_value
return accessor_func(obj, check_key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 472, in get_attribute
return get_value(obj, attr, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/utils.py", line 239, in get_value
return _get_value_for_key(obj, key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/utils.py", line 253, in _get_value_for_key
return getattr(obj, key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 480, in __get__
return self.impl.get(state, dict_)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 931, in get
value = self.callable_(state, passive)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 879, in _load_for_state
return self._emit_lazyload(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 1036, in _emit_lazyload
result = session.execute(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1689, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1582, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/sql/lambdas.py", line 481, in _execute_on_connection
return connection._execute_clauseelement(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1451, in _execute_clauseelement
ret = self._execute_context(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1813, in _execute_context
self._handle_dbapi_exception(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1998, in _handle_dbapi_exception
util.raise_(exc_info[1], with_traceback=exc_info[2])
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1770, in _execute_context
self.dialect.do_execute(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 449, in execute
self._adapt_connection.await_(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 60, in await_only
raise exc.MissingGreenlet(
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: http://sqlalche.me/e/14/xd2s)
The problem in this case is that the Marshmallow schema is configured to load related models (include_relationships=True). Since the initial query doesn't load them automatically, the schema triggers a query to fetch them, and this causes the error.
The simplest solution, demonstrated in the docs, is to eagerly load the related objects with their "parent":
async def get_all_users():
async with db_conn.get_async_sa_session() as session:
# Let's assume a Customer has a 1 to many relationship with an Order model
statement = select(Customer).options(orm.selectinload(Customer.orders))
results = await session.execute(statement)
_ = (results.scalars().all())
print(_)
response = customer_schema.dump(_, many=True)
print(response)
There is more discussion in the Preventing Implicit IO when Using AsyncSession section of the docs.
I am writing an API for communication between Jira and another server. I want to grab certain data from the Jira server and use it to populate some fields in the other software tool.
When i do this it works fine:
resp = s.get(url)
rec_data = resp.json()
data = json.dumps({"recordRevisionNumber": rev_id, "ELM_CSTM_TEXT1": "Open", "ELM_CSTM_TEXT2": "ARA-222"})
resp = s.put(url, data=data)
But I change out the hard coded "Open" and "ARA-222" with the variables "jira_id and "jira_status" I get an error.
data = json.dumps({"recordRevisionNumber": rev_id, "ELM_CSTM_TEXT1": jira_id, "ELM_CSTM_TEXT2": jira_status})
resp = s.put(url, data=data)
Here is the error:
Traceback (most recent call last):
File "C:/Users/byun/Desktop/Workspace/TopTeam_API.py", line 37, in <module>
data = json.dumps({"recordRevisionNumber": rev_id, "ELM_CSTM_TEXT1": jira_id, "ELM_CSTM_TEXT2": jira_status})
File "C:\Program Files (x86)\Python 3.5\lib\json\__init__.py", line 230, in dumps
return _default_encoder.encode(obj)
File "C:\Program Files (x86)\Python 3.5\lib\json\encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "C:\Program Files (x86)\Python 3.5\lib\json\encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "C:\Program Files (x86)\Python 3.5\lib\json\encoder.py", line 180, in default
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: <JIRA Status: name='Open', id='1'> is not JSON serializable
The variables are fine, they print out like this:
108 ARA-129 Open
108 ARA-128 Open
108 ARA-127 Open
108 ARA-126 Open
108 ARA-125 Open
Please Help.
I figured it out. The jira_id was working fine. The jira_status worked after I turned it into a string.
status = str(jira_status)
data = json.dumps({"recordRevisionNumber": rev_id, "ELM_CSTM_TEXT1": jira_id, "ELM_CSTM_TEXT2": status})
I am trying to do a simple authentication using Python3 and urllib on an API that should return account balances.
The code I have is the following:
import urllib
import urllib.request
import json
id = "nkkhuz6" # fake
secret = "s9MeR0J9yxtndLBPVA" # fake
auth_str = id + ":" + secret
def getBalances():
values = {'u' : auth_str}
data = urllib.parse.urlencode(values)
data = data.encode('utf-8') # data should be bytes
request = urllib.request.Request(url = "https://api.com", data = data)
with urllib.request.urlopen(request) as f:
print(json.loads(f.read().decode('utf-8')))
However when I run getBalances() I get the following errors:
Adriaans-MacBook-Pro:Documents adriaanjoubert$ python3 main.py
Traceback (most recent call last):
File "main.py", line 96, in <module>
getBalances()
File "main.py", line 19, in getBalances
with urllib.request.urlopen(request) as f:
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 469, in open
response = meth(req, response)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 579, in http_response
'http', request, response, code, msg, hdrs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 507, in error
return self._call_chain(*args)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 441, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 587, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 405: Method Not Allowed
I am sure the URL is correct and if I append a trailing / I get the error urllib.error.HTTPError: HTTP Error 404: Not Found
When I run the following code I do get my account balances:
cmd = """curl -u """ + auth_str + """ https://api.com/"""
os.system(cmd)
What am I doing wrong? I would like to use urllib so that I can store the stuff I get back from the API in a variable.
I have a CSV file, which contains a list of Google extension IDs.
I'm writing a code that will read the extension IDs, add the webstore url, then perform a basic get request:
import csv
import requests
with open('small.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
urls = "https://chrome.google.com/webstore/detail/" + row[0]
print urls
r = requests.get([urls])
Running this code results in the following Traceback:
Traceback (most recent call last):
File "C:\Users\tom\Dropbox\Python\panya\test.py", line 9, in <module>
r = requests.get([urls])
File "C:\Python27\lib\site-packages\requests\api.py", line 69, in get
return request('get', url, params=params, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 567, in send
adapter = self.get_adapter(url=request.url)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 641, in get_adapter
raise InvalidSchema("No connection adapters were found for '%s'" % url)
InvalidSchema: No connection adapters were found for '['https://chrome.google.com/webstore/detail/blpcfgokakmgnkcojhhkbfbldkacnbeo']'
How can revise the code, so that it would accept the urls in the list, and make the GET request?
requests.get expects a string, but you're creating and passing a list [urls]
r = requests.get([urls])
Change it to just
r = requests.get(urls)
and it should work.
I am trying to download file from sharepoint using python library. Code is connecting and getting content of sharepoint perfectly but not downloading particular file.
here is script:
import requests
from requests_ntlm import HttpNtlmAuth
headers = {'accept': 'application/json;odata=verbose'}
r = requests.get("https://abc.we.x/abd.doc", auth=HttpNtlmAuth('domain\\User','ppusers#123'),headers=headers)
print r.json()["d"]["CustomMasterUrl"]
if we print j.content then i can see all html content of this page but r.json()["d"]["CustomMasterUrl"] line giving error as below
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Python27\lib\site-packages\requests\models.py", line 819, in json
return json.loads(self.text, **kwargs)
File "C:\Python27\lib\json\__init__.py", line 338, in loads
return _default_decoder.decode(s)
File "C:\Python27\lib\json\decoder.py", line 366, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Python27\lib\json\decoder.py", line 384, in raw_decode
raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded
Please help in downloading file if someone has other way to do that.
Thanks in advance.
I think you can use cmislib to download files from sharepoint
client = CmisClient(serverurl, username, password)
repo = client.defaultRepository
doc = repo.getObjectbyPath("PathOfFile")
result = doc.getContentStream()
c = result.read()
docFile = open("/Destination/File/Path"+doc.properties['cmis:name'],"w")
docFile.write(c)
docFile.close()
This part of code works fine with an Alfresco Server, but sharepoint should also support cmis. https://msdn.microsoft.com/en-US/library/office/jj945829.aspx
Maybe this can help you.
Thanks for all response. Here is working solution. Basically we were missing stream writing to file.
import requests
from requests_ntlm import HttpNtlmAuth
headers = {'accept': 'application/json;odata=verbose'}
url = "https://docs.qmad.er.com/sites/cloud_BPAP/Shared%20Documents/Database%20Deliverables/20150319_Maintenance_BPAP15/20150319_Maintenance.zip"
username = 'abc\\ert' #domain\\username
password = 'User#123'
r = requests.get(url, auth=HttpNtlmAuth(username,password),stream=True)
fh = open("clientNet.zip", "wb")
fh.write(r.content)
fh.close()