Please help, I have developed this scraper API and it works with internal commands in Pythionanywhere, but when I try to access it using my use account URL, it gives the error: [errno 11] and I have searched for solutions, I co[enter image description here](https://i.stack.imgur.com/cs9T6.png)uldn't find any.
I was expecting a JSON output as it did on the internal server.
#from requests_html import HTMLSession
import json
import requests
class Scraper():
def scrapedata(self, tag):
url = "https://www.etenders.gov.za/Home/TenderOpportunities/?status=1"
headers = {'user-agent': 'Mozilla/5.0'}
response = requests.get(url, headers=headers)
data = response.json()
file_urs = []
for e in data:
item = {
'province': (f"{e['province']}"),
'id' : (f"{e['tender_No']}"),
}
print(file_urs)
file_urs.append(item)
return file_urs
quotes = Scraper()
quotes.scrapedata('cat')
from flask import Flask
import json
from scrape import Scraper
app = Flask(name)
quotes = Scraper()
#app.route('/', methods =['GET', 'POST'])
async def read_item(cat):
json_dumps = json.dumps.scrapedata(cat)
return json_dumps
#return quotes.scrapedata(cat).jon()
from flask import Flask
import json
from scrape import Scraper
app = Flask(name)
quotes = Scraper()
#app.route('/', methods =['GET', 'POST'])
async def read_item(cat):
json_dumps = json.dumps.scrapedata(cat)
return json_dumps
#return quotes.scrapedata(cat).jon()
I was expecting a JSON output as it did on the internal server.
Related
I installed Django and Django Rest Api. I want to send some data to rest api. Rest api will take the data and run a script with this data and get a result. Then send this result back to me.
There won't be database usage.
Like this, request : http://testerapi.com:8000/search?q=title:xfaster564CertVal9body:A%22&fl=id
Response : {validation : true}
Is it possible?
Yes it is possible ! But i will try to respond with api function based view.
Let's suppose that our worker function to call when call the API (GET or POST) is in the utilities.py file, the models.py, serializers.py and views.py.
utilities.py
def my_worker(a, b=0, c=0):
# do something with a, b, c
return a + b + c > 10
models.py
from datetime import datetime
class User(object):
def __init__(self, email, name, created = None):
self.email = email
self.name = name
self.created = created or datetime.now()
serializers.py
I use simple Serializer but ModelSerializer is better i think
from rest_framework import serializers
class UserSerializer(serializers.Serializer):
# initialize fields
email = serializers.EmailField()
name = serializers.CharField(max_length = 200)
created = serializers.DateTimeField()
views.py
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt # Allow request without csrf_token set
from rest_framework.decorators import api_view
from .models import User
from .serializers import UserSerializer
# Import my_worker from .utilities
from .utilities import my_worker
#csrf_exempt
#api_view('GET') # Only get request is allowed
def user_worker(request, a, b, c):
"""
Do something with
"""
if request.method == 'GET':
# Do some stuff
users = User.objects.all()
serializer = UserSerializer(users, many=True)
# Call the utilities script here
result = my_worker(a, b, c)
if result: # a+b+c > 10
return JsonResponse({"validation": "true"}, safe=False)
else:
return JsonResponse({"validation": "false"}, safe=False)
Note that i dont use the UserSerializer but show it at example.
You can then execute a more complex function (here the my_worker).
Adapt it according to your needs.
I get this json string from a page:
{"lprice":"8330.1","curr1":"BTC","curr2":"EUR"}
I tried to access the lprice with this code:
import requests
def get_latest_price(api, currencie, real):
CEXIO_API_URL = "https://cex.io/api/last_price/%s/%s" % (currencie.upper(), real.upper())
response = requests.get(CEXIO_API_URL)
response_json = response.json()
return float(response_json['lprice'])
But if I do it like this, I get this error:
File
"/home/malte/Entwicklung/cryptoprice/build/all/app/install/qml/cryptoprice.py",
line 16, in get_latest_price
return float(response_json['lprice'])
KeyError: 'lprice'
I assume that your
response_json is your json-string {"lprice":"8330.1","curr1":"BTC","curr2":"EUR"}
Then it should work if you convert the json string into a dictionary with the loads function
import requests
import json
def get_latest_price(api, currencie, real):
CEXIO_API_URL = "https://cex.io/api/last_price/%s/%s" % (currencie.upper(), real.upper())
response = requests.get(CEXIO_API_URL)
response_json = response.json()
response_json = json.loads(response_json)
return float(response_json['lprice'])
This is the error:
In line 7 and line 9;
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
This is my code:
import requests
BASE_URL = 'http://127.0.0.1:8000/'
ENDPOINT = 'api/'
def get_resource(id):
resp = requests.get(f"{BASE_URL}{ENDPOINT}{id}/")
print(resp.status_code)
print(resp.json())
id = input("enter some id:")
get_resource(id)
Response returns a big data from this site with headers, status code and other...
You should to dump also content of this response. Read there
Try next:
resp.text.json()
Or:
resp.content.json()
content gives you access to the raw bytes of the response payload, you will often want to convert them into a string using a character encoding such as UTF-8
This is my view.py:
from django.shortcuts import render
from django.views.generic import View
from withoutrest.models import employee
from django.http import HttpResponse
import json
from django.core.serializers import serialize
class EmployeeDetails(View):
def get(self, request, id, *args, **kwargs):
emp = employee.objects.get(id=id)
json_data = serialize('json', [emp,])
return HttpResponse(json_data, content_type='application/json')
from flask import Flask, abort, request
import json
app = Flask(__name__)
#app.route('/data', methods=['POST'])
def data():
if not request.json:
abort(400)
print(request.json)
return json.dumps(request.json)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5001, debug=True)
i want to send all my post request to another endpoint at localhost:8000.. please suggest the correctionsenter code here
I tried run your code, it works fine, here is my exmaple:
Server same as yours. One suggestion about your code: .json attribute has been deprecated, you should use the request.get_json() method.
from flask import Flask, abort, request
import json
app = Flask(__name__)
#app.route('/data', methods=['POST'])
def data():
if not request.json:
abort(400)
print(request.json)
return json.dumps(request.json)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5001, debug=True)
client:
import requests
res = requests.post('http://localhost:5001/data', json={"mytext":"test"})
if res.ok:
print(res.json())
I guessed you don't fire the right HTTP request. for exmaple, do you add attribute Content-Type: application/json in your http header?
I want to create a REST API without using Flask. I have created once using Flask as shown below but now I want to try without Flask. I came to know that urllib is one of the packages for doing it but not sure how to do. Even if there is some way other than urllib then that is also fine.
from werkzeug.wrappers import Request, Response
import json
from flask import Flask, request, jsonify
app = Flask(__name__)
with open ("jsonfile.json") as f:
data = json.load(f)
#data=f.read()
#app.route('/', methods=['GET', 'POST'])
def hello():
return jsonify(data)
if __name__ == '__main__':
from werkzeug.serving import run_simple
run_simple('localhost', 9000, app)
You can try something like this
import json
import http.server
import socketserver
from typing import Tuple
from http import HTTPStatus
class Handler(http.server.SimpleHTTPRequestHandler):
def __init__(self, request: bytes, client_address: Tuple[str, int], server: socketserver.BaseServer):
super().__init__(request, client_address, server)
#property
def api_response(self):
return json.dumps({"message": "Hello world"}).encode()
def do_GET(self):
if self.path == '/':
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(bytes(self.api_response))
if __name__ == "__main__":
PORT = 8000
# Create an object of the above class
my_server = socketserver.TCPServer(("0.0.0.0", PORT), Handler)
# Star the server
print(f"Server started at {PORT}")
my_server.serve_forever()
And testing like this
→ curl http://localhost:8000
{"message": "Hello world"}%
but keep in mind that this code is not stable and just sample
You shall take an existing web server and use WSGI compatible app, for example, for Apache HTTP 2.2
Install mod_wsgi (just search how to install mod_wsgi in your operating system)
Configure mod_wsgi in Apache httpd.conf
LoadModule wsgi_module modules/mod_wsgi.so
WSGIScriptAlias /wsgi /var/www/wsgi/myapp.wsgi
Write myapp.wsgi
The code for myapp.wsgi must call the second argument once in this way:
def application(environ, start_response):
status = '200 OK'
output = b'{"message": "Hello world"}'
response_headers = [('Content-type', 'application/json'),
('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output]