I am trying to Extract JSON data from URL with parameters to it and code is as follows:
import json
from flask import Flask, render_template, request, jsonify
import requests
app = Flask(__name__)
import urllib2
#app.route("/",methods=['GET','POST'])
def home():
if request.method == 'POST':
#user inputs
valueone= request.form.get('first')
valueTwo = request.form.get('second')
valueThree = request.form.get('third')
#api call
data = {"perfid" : {0}, "section" : {"hostname" : {1}, "iteration" : {2}, "sectionname" : "sysstat_M"}}
req = urllib2.Request('http://api-latx-dev.corp.netapp.com/ws/spm/spm-general', json.dumps(data))
response = urllib2.urlopen(req)
the_page = response.read()
url=response.getcode()
returnData = {}
if url.status_code == 200:
returnData["status"] = "SUCCESS"
returnData["result"] = the_page
return jsonify(returnData)
else:
returnData["status"] = "ERROR"
return jsonify(returnData)
#jsonify(response.json())
return render_template('index.html')
I need to fetch data But getcode() gives int which is not working with "if url.status_code == 200".Can anyone please suggest how to deal with it.
Related
I have two python classes:- One class(CloudLink) is responsible for sending JSON events to the app and another(ReadData) is responsible for building the JSON data.
The ReadData class will be using the CloudLink methods to send the JSON data to the App. But I'm getting error _buildJSONdata() missing 1 required positional argument: 'Data'.
ReadData class
from pyspark.sql import SparkSession
import functools
from pyspark.sql import DataFrame
from pyspark.sql.functions import explode
from cosmosconnect import azurecosmos
class ReadData:
#exception(logger)
def __init__(self):
self.spark_session = (
SparkSession.builder
.appName("readData")
.getOrCreate()
)
mssparkutils.fs.unmount('/mnt/test')
logger.info("Drive unmounted")
mssparkutils.fs.mount(
'abfss://abc#transl.dfs.core.windows.net/',
'/mnt/test',
{'linkedService': "linkCosmos"}
)
logger.info("Mounted Successfully")
self.input_directory = (f"synfs:/{mssparkutils.env.getJobId()}/mnt/test/input_path"
)
self.output_directory = (f"synfs:/{mssparkutils.env.getJobId()}/mnt/test/output_path"
)
'''
Reading the schema from csv file
'''
#exception(logger)
def readConfig(self):
try:
logger.info(f"Reading the Config present in {self.input_directory} ")
dfConfig = self.spark_session.read.option("multiline","true") \
.json(self.input_directory)
#for f in dfConfig.select("Entity","Query","Business_Rule").collect():
dfConfig=dfConfig.select(explode('Input').alias('Input_Data'))\
.select('Input_Data.Validation_Type','Input_Data.Entity','Input_Data.Query','Input_Data.Business_Rule')
for f in dfConfig.rdd.toLocalIterator():
#for index, f in dfConfig.toPandas().iterrows():
self.Validation_Type=f[0]
self.container=f[1]
self.query=f[2]
self.rule=f[3]
self.readCosmos(self)
except:
raise ValueError("")
#exception(logger)
def readCosmos(self,*params):
#from cosmosconnect import azurecosmos
#a=[]
linkedService='fg'
df=azurecosmos.cosmosConnect(linkedService,self.query,self.container)
df.cache()
if len(df.head(1)) >0:
outputpath=self.output_directory+'/'+self.container
df.coalesce(1).write.mode('overwrite').parquet(outputpath)
Status="Validation Failure"
Data= {"Validation_Type":[],"Status":[],"Container":[],"Business_Rule":[]}
Data["Validation_Type"].append(self.Validation_Type)
Data["Status"].append(Status)
Data["Container"].append(self.container)
Data["Business_Rule"].append(self.rule)
CloudLink._buildJSONdata(Data)
if __name__ == "__main__":
p = ReadData()
p.readConfig()
CloudLink class
import json
import datetime
import hashlib
import json
import sys
import traceback
import adal
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import logging
from functools import wraps
import sys
def create_logger():
#create a logger object
#logger = logging.getLogger()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logfile = logging.FileHandler('exc_logger.log')
#logfile = logging.StreamHandler(sys.stdout)
fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
formatter = logging.Formatter(fmt)
logfile.setFormatter(formatter)
logger.addHandler(logfile)
return logger
logger = create_logger()
def exception(logger):
def decorator(func):
#wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
issue = "exception in "+func.__name__+"\n"
issue = issue+"-------------------------\
------------------------------------------------\n"
logger.exception(issue)
raise
return wrapper
return decorator
class CloudLink(object):
_token = None
_instance = None
http = None
cloudclient = TokenLibrary.getSecret("xxxx", "rtrt")
clientid = TokenLibrary.getSecret("xxxx", "tyty")
clientcredentials = TokenLibrary.getSecret("xxxx", "abcabc")
authority_url = TokenLibrary.getSecret("xxxx", "abab")
cloudtest = TokenLibrary.getSecret("xxxx", "yyyy")
#staticmethod
def getInstance():
if not CloudLink._instance:
CloudLink._instance = CloudLink()
return CloudLink._instance
def __init__(self):
retry_strategy = Retry(
total=3,
backoff_factor=0,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self.http = requests.Session()
self.http.mount("https://", adapter)
self.http.mount("http://", adapter)
print("Inside init")
def parseJSON(self, t):
try:
eventData = json.loads(t)
logger.info(f"Sending {eventData} to cloud")
self.sendToCloud(eventData)
except ValueError as e:
print("Error: %s Please validate JSON in https://www.jsonschemavalidator.net/"% e)
return None # or: raise
def sendToCloud(self, eventData):
cloudData = {"eventData": eventData, "metadata": self._buildMetadata()}
logger.info(f"Raising alert with data=({cloudData}")
response = self.http.post(
self.cloudtest, headers=self._buildHeaders(), json=cloudData
)
logger.info(f"cloud alert response={response}")
if response.status_code == 202 or response.status_code == 200:
logger.info("Mail sent to Cloud")
else:
raise Exception(f"Cloud reporting failed with Error {response}")
def _buildJSONdata(self,Data):
if len(Data) == 0:
raise Exception("JSON is empty")
else:
t = json.dumps(self.Data)
self.parseJSON(t)
def _buildMetadata(self):
return {
"messageType": "Send Email",
"messageVersion": "0.0.1",
"sender": "Send Email",
}
def _buildHeaders(self):
self._refreshADToken()
headers = {
"Authorization": "Bearer {}".format(self._token["accessToken"]),
"Content-type": "application/json",
"Accept": "text/plain",
}
return headers
def _refreshADToken(self):
def shouldRenew(token):
"""Returns True if the token should be renewed"""
expiresOn = datetime.datetime.strptime(
token["expiresOn"], "%Y-%m-%d %H:%M:%S.%f"
)
now = datetime.datetime.now()
return (expiresOn - now) < datetime.timedelta(minutes=5)
if not self._token or shouldRenew(self._token):
logger.info("Renewing credentials for Alerting")
result = None
try:
context = adal.AuthenticationContext(CloudLink.authority_url)
result = context.acquire_token_with_client_credentials(CloudLink.cloudclient, CloudLink.clientid,CloudLink.clientcredentials)
except Exception as e:
error = "Failed to renew client credentials."
logger.info(error)
raise
if result and "accessToken" in result:
self._token = result
else:
logger.error(
"Failed to acquire bearer token. accessToken not found in result object on renewing credentials."
)
raise Exception("Could not acquire a bearer token")
I'm trying to put python objects into a JSON file by getting the API from one of the sites but somehow when I run the code nothing has been put in the JSON file. API is working well, as well when I print out the code by json.load I get the output but I have no idea why does dump doesn't work.
here is my code:
from django.shortcuts import render
import requests
import json
import datetime
import re
def index(request):
now = datetime.datetime.now()
format = "{}-{}-{}".format(now.year, now.month, now.day)
source = []
author = []
title = []
date = []
url = "http://newsapi.org/v2/everything"
params = {
'q': 'bitcoin',
'from': format,
'sortBy': 'publishedAt',
'apiKey': '1186d3b0ccf24e6a91ab9816de603b90'
}
response = requests.request("GET", url, params=params)
for news in response.json()['articles']:
matching = re.match("\d+-\d+-\d+", news['publishedAt'])
if format == matching.group():
source.append(news['source'])
author.append(news['author'])
title.append(news['title'])
date.append(news['publishedAt'])
data = \
{
'source': source,
'author': author,
'title': title,
'date': date
}
with open('data.json', "a+") as fp:
x = json.dump(data, fp, indent=4)
return render(request, 'news/news.html', {'response': response})
I'm using Django and Python 3.7. I'm having trouble returning JSON from one of my views. I have this view code ...
def get_hints(request):
article_id = request.GET.get('article_id', None)
article = Article.objects.get(pk=article_id)
s = ArticlesService()
objects = s.get_hints(article)
data = ArticleSerializer(objects, many=True).data
print("data: ", data)
return HttpResponse(data, content_type="application/json")
The service method referenced returns the following data ...
def get_hints(self, article):
...
sorted_articles_map = OrderedDict(sorted(rank_map.items(), key=operator.itemgetter(1), reverse=True))
return list(sorted_articles_map.keys())
The data returned from the serializer looks like this, definitely not json ...
[OrderedDict([('id', 10777935), ('created_on_ms', 1577985486000.0), ('label', 'World'), ('title', "Outrage and Disgust After 'Serial Killer' ...,
How do I render proper JSON?
Edit: Adding Article serializer ...
class ArticleSerializer(serializers.ModelSerializer):
class Meta:
model = Article
fields = ['id', 'created_on_ms', 'label', 'title', 'mobile_path', 'path', 'url', 'is_media', 'checked_for_dup', 'original_path', 'is_spiked']
Option 1
Use the #api_view()--(drf doc) decorator of DRF along with Response--(drf doc) class
from rest_framework.decorators import api_view
from rest_framework.response import Response
#api_view(['GET'])
def get_hints(request):
article_id = request.GET.get('article_id', None)
article = Article.objects.get(pk=article_id)
s = ArticlesService()
objects = s.get_hints(article)
data = ArticleSerializer(objects, many=True).data
return Response(data)
Option 2
You will get similar response by using JsonResponse--(django doc)
from django.http.response import JsonResponse
def get_hints(request):
article_id = request.GET.get('article_id', None)
article = Article.objects.get(pk=article_id)
s = ArticlesService()
objects = s.get_hints(article)
data = ArticleSerializer(objects, many=True).data
return JsonResponse(data, safe=False)
I use django-rest-framework first time. I use json format by default.
I need to send file for creating a new instance. This file has being saved in models.ImageField. However, I don't know which format this field requires for incoming file. I tried to send it in base64, but it isn't suitable.
TestCase for this problem:
class PersonTestCase(APITestCase):
def setUp(self):
self.c = APIClient()
def test_sign_up_with_valid_data(self):
with open('persons/test/bg.jpg', 'rb') as bg:
valid_registration_data = {
...,
'background': base64.b64encode(bg.read()).decode('utf-8'),
}
response = self.c.post('/persons/sign_up', valid_registration_data)
self.assertEqual(response.status_code, 201)
self.assertEqual(Person.objects.count(), 1)
self.assertEqual(Person.objects.get('username'), 'test_client74562984')
View:
class SignUpView(APIView):
"""
Create new user
"""
def post(self, request):
serializer = UserSerializer(data=request.data)
if serializer.is_valid():
serializer.save()
response = Response(serializer.data, status=status.HTTP_201_CREATED)
return response
response = Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
return response
I have a Django app. In the view I call another function (in stats.py) which then makes a HTTP POST.
views.py
from stats import Stat
a = Stat(example="12345")
a.use(id='query')
stats.py
self.data = { example : "12345" }
req = urllib2.Request(api_url)
req.add_header('Content-Type', 'application/json')
response = urllib2.urlopen(req, json.dumps(self.data))
The problem that occurs is that I get the error,
<django.utils.functional.SimpleLazyObject object at 0x2b4d1fe47650> is not JSON serializable
Django Traceback
From looking at the Django Traceback I get the following,
/prod/tools/lx/views.py in update_input
a.use(id='query')
...
/prod/tools/main/stats.py in log_use
response = urllib2.urlopen(req, json.dumps(self.data))
...
/usr/local/lib/python2.7/json/__init__.py in dumps
return _default_encoder.encode(obj)
...
/usr/local/lib/python2.7/json/encoder.py in encode
chunks = self.iterencode(o, _one_shot=True)
...
/usr/local/lib/python2.7/json/encoder.py in iterencode
return _iterencode(o, 0)
...
/usr/local/lib/python2.7/json/encoder.py in default
raise TypeError(repr(o) + " is not JSON serializable")
...
Any ideas ?
Thanks,
Try this using urllib
import urllib
...
self.data = { example : "12345", 'Content-type':'application/json' }
self.data = urllib.urlencode(self.data)
req = urllib2.Request(api_url, self.data)
response = urllib2.urlopen(req)