Creating UTF-8 JsonResponse in Django - json

Is there any simple way to override DjangoJSONEncoder.ensure_ascii and set it to False or output non-ascii text in django.http.JsonResponse in any other way?

As of Django 1.9 you can configure JSONResponse to disable the ensure_ascii switch, by passing in a value for the json_dumps_params argument:
return JsonResponse(response_data, safe=False, json_dumps_params={'ensure_ascii': False})
With ensure_ascii=False, json.dumps() outputs UTF-8 data for non-ASCII codepoints.
You could subclass JsonResponse to make it the default unless set differently:
from django.http.response import JsonResponse
class UTF8JsonResponse(JsonResponse):
def __init__(self, *args, json_dumps_params=None, **kwargs):
json_dumps_params = {"ensure_ascii": False, **(json_dumps_params or {})}
super().__init__(*args, json_dumps_params=json_dumps_params, **kwargs)
then use that throughout instead of JsonResponse.
At the extreme end, you could monkey-patch the class to set ensure_ascii to False by default; put the following in a suitable module of your Django app (say, in a file named patches.py):
import logging
from functools import wraps
from django.http.response import JsonResponse
logger = logging.getLogger(__name__)
def patch_jsonresponse_disable_ensure_ascii():
if getattr(JsonResponse, '_utf8_patched', False):
# Already patched. Add warning in logs with stack to see what location
# is trying to patch this a second time.
logger.warning("JSONResponse UTF8 patch already applied", stack_info=True)
return
logger.debug("Patching JSONResponse to disable ensure_ascii")
orig_init = JsonResponse.__init__
#wraps(orig_init)
def utf8_init(self, *args, json_dumps_params=None, **kwargs):
json_dumps_params = {"ensure_ascii": False, **(json_dumps_params or {})}
orig_init(self, *args, json_dumps_params=json_dumps_params, **kwargs)
JsonResponse.__init__ = utf8_init
JsonResponse._utf8_patched = True # to prevent accidental re-patching
then import patch_jsonresponse_disable_ensure_ascii into your Django settings file and call it based on your desired config:
from yourapp.patches import patch_jsonresponse_disable_ensure_ascii
JSON_RESPONSES_UTF8 = True
if JSON_RESPONSES_UTF8:
patch_jsonresponse_disable_ensure_ascii()

EDIT:
Or if you tend to the utf-8 format, use instead of Django's JsonResponse():
return HttpResponse(json.dumps(response_data, ensure_ascii=False),
content_type="application/json")
or
return JsonResponse(json.dumps(response_data, ensure_ascii=False), safe=False)
more about the safe=False HERE
OLD:
You don't have to whatever alter.
Although Django creates JSON data in ASCII (from UTF-8), Javascript will automatically decode it back to UTF-8.

from django.core.serializers.json import DjangoJSONEncoder
from django.http import JsonResponse
class MyJsonResponse(JsonResponse):
def __init__(self, data, encoder=DjangoJSONEncoder, safe=True, **kwargs):
json_dumps_params = dict(ensure_ascii=False)
super().__init__(data, encoder, safe, json_dumps_params, **kwargs)

I didn't find any better way yet than to utilize an already installed REST Framework:
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from .models import INITIATOR_TYPES
#api_view(['GET'])
#permission_classes((IsAuthenticatedOrReadOnly, ))
def initiator_types(request):
data = {t[0]: str(t[1]) for t in INITIATOR_TYPES}
return Response(data)
But I don't really like it. It's much more complicated than JsonResponse: https://stackoverflow.com/a/24411716/854477

Related

Is it possible to send a request to django rest api to run a script?

I installed Django and Django Rest Api. I want to send some data to rest api. Rest api will take the data and run a script with this data and get a result. Then send this result back to me.
There won't be database usage.
Like this, request : http://testerapi.com:8000/search?q=title:xfaster564CertVal9body:A%22&fl=id
Response : {validation : true}
Is it possible?
Yes it is possible ! But i will try to respond with api function based view.
Let's suppose that our worker function to call when call the API (GET or POST) is in the utilities.py file, the models.py, serializers.py and views.py.
utilities.py
def my_worker(a, b=0, c=0):
# do something with a, b, c
return a + b + c > 10
models.py
from datetime import datetime
class User(object):
def __init__(self, email, name, created = None):
self.email = email
self.name = name
self.created = created or datetime.now()
serializers.py
I use simple Serializer but ModelSerializer is better i think
from rest_framework import serializers
class UserSerializer(serializers.Serializer):
# initialize fields
email = serializers.EmailField()
name = serializers.CharField(max_length = 200)
created = serializers.DateTimeField()
views.py
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt # Allow request without csrf_token set
from rest_framework.decorators import api_view
from .models import User
from .serializers import UserSerializer
# Import my_worker from .utilities
from .utilities import my_worker
#csrf_exempt
#api_view('GET') # Only get request is allowed
def user_worker(request, a, b, c):
"""
Do something with
"""
if request.method == 'GET':
# Do some stuff
users = User.objects.all()
serializer = UserSerializer(users, many=True)
# Call the utilities script here
result = my_worker(a, b, c)
if result: # a+b+c > 10
return JsonResponse({"validation": "true"}, safe=False)
else:
return JsonResponse({"validation": "false"}, safe=False)
Note that i dont use the UserSerializer but show it at example.
You can then execute a more complex function (here the my_worker).
Adapt it according to your needs.

The problem is "method' object is not iterable. I am doing a small Django app

Here is the error when I go to employees page (employees is the model):
TypeError at /employees/
'method' object is not iterable
Request Method: GET
Request URL: http://127.0.0.1:8000/employees/
Django Version: 3.2
Exception Type: TypeError
Exception Value:
'method' object is not iterable
Exception Location: C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\serializers.py, line 677, in to_representation
Python Executable: C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\Scripts\python.exe
Python Version: 3.9.4
Python Path:
['C:\\Users\\USER\\PycharmProjects\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject1\\health_recommender_final',
'C:\\Users\\USER\\PycharmProjects\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject1\\health_recommender_final',
'C:\\Program Files\\JetBrains\\PyCharm '
'2021.1.1\\plugins\\python\\helpers\\pycharm_display',
'C:\\Users\\USER\\AppData\\Local\\Programs\\Python\\Python39\\python39.zip',
'C:\\Users\\USER\\AppData\\Local\\Programs\\Python\\Python39\\DLLs',
'C:\\Users\\USER\\AppData\\Local\\Programs\\Python\\Python39\\lib',
'C:\\Users\\USER\\AppData\\Local\\Programs\\Python\\Python39',
'C:\\Users\\USER\\PycharmProjects\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject1\\health_recommender_final\\venv',
'C:\\Users\\USER\\PycharmProjects\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject\\djangoProject1\\health_recommender_final\\venv\\lib\\site-packages',
'C:\\Program Files\\JetBrains\\PyCharm '
'2021.1.1\\plugins\\python\\helpers\\pycharm_matplotlib_backend']
Server time: Fri, 18 Jun 2021 01:49:34 +0000
My main files are serializers and models.
from rest_framework import serializers
from rest_framework import Person
class PersonSerializer(serializers.ModelSerializer):
class Meta:
models = Person
fields = '__all__'
from django.db import models
# Create your models here.
from django.db import models
from django.utils import timezone
from django.contrib.auth.models import User
class Person(models.Model):
EDUCATION_CHOICES = (
('NA', 'NONE'),
('SSC', 'SSC'),
('HSC', 'HSC'),
('undergraduate', 'UNDERGRADUATE'),
('graduate', 'POSTGRADUATE'),
('Further Studies or PhD', 'FURTHER/PhD'),
)
DIET = (
('Vegetarian', 'VEG'),
('Omnivore', 'OMNI'),
('Fish', 'FISH'),
)
EXERCISE = (
('na', 'NA'),
('yes', 'YES'),
('no', 'NO'),
)
name = models.CharField(max_length=250,default="")
education_details = models.CharField(max_length=150, choices=EDUCATION_CHOICES, default='NA')
diet = models.CharField(max_length=80, choices=DIET, default='VEG')
exercise = models.CharField(max_length=80, choices=EXERCISE, default='NA')
class Meta:
ordering = ('-name',)
#property
def __str__(self):
return self.name
So what should I do now? I think the method that is not iterable is coming because I left something out somewhere, where can it be?
I am trying to create a small app, which when taken has the above input as the model, which returns a JSON file. After that I would manipulate the JSON file with some programming to create health recommendations based on lifestyle choice. Can someone tell me where I am going wrong and how I can complete the JSON manipulation thing to get simple for - if submission things to create the recommendations??
views.py
from django.shortcuts import render
# Create your views here.
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from .models import Person
from .serializers import PersonSerializer
from django.shortcuts import get_object_or_404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from .models import Person
from .serializers import PersonSerializer
class PersonList(APIView):
def get(self, request):
person1 = Person.objects.all
serializer = PersonSerializer(person1, many=True)
return Response(serializer.data)
def post(self):
pass
urls.py
from django.contrib import admin
from django.urls import path
from django.conf.urls import url
from rest_framework.urlpatterns import format_suffix_patterns
from healthrecomm import views
urlpatterns = [
path('admin/', admin.site.urls),
path('employees/', views.PersonList.as_view())
]
System check identified no issues (0 silenced).
June 18, 2021 - 07:49:18
Django version 3.2, using settings 'health_recommender_final.settings'
Starting development server at http://127.0.0.1:8000/
Quit the server with CTRL-BREAK.
[18/Jun/2021 07:49:34] "GET /employees HTTP/1.1" 301 0
Internal Server Error: /employees/
Traceback (most recent call last):
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\django\views\generic\base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\views.py", line 476, in raise_uncaught_exception
raise exc
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\healthrecomm\views.py", line 23, in get
return Response(serializer.data)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\serializers.py", line 760, in data
ret = super().data
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\serializers.py", line 260, in data
self._data = self.to_representation(self.instance)
File "C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\venv\lib\site-packages\rest_framework\serializers.py", line 677, in to_representation
return [
TypeError: 'method' object is not iterable
[18/Jun/2021 07:49:34] "GET /employees/ HTTP/1.1" 500 105485
Not Found: /favicon.ico
[18/Jun/2021 07:49:35] "GET /favicon.ico HTTP/1.1" 404 2123
C:\Users\USER\PycharmProjects\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject\djangoProject1\health_recommender_final\healthrecomm\models.py changed, reloading.
Watching for file changes with StatReloader
views.py
The missing field is parenthesis in
person1 = Person.objects.all
() is missing
class PersonList(APIView):
def get(self, request):
# this person1 line is mistake ,
person1 = Person.objects.all # this all()
# ok ? ^^^
serializer = PersonSerializer(person1, many=True)
return Response(serializer.data)
def post(self):
pass
.
right is:
.
class PersonList(APIView):
def get(self, request):
# solution is this line
person1 = Person.objects.all() # this all()
# ok ? ^^^
serializer = PersonSerializer(person1, many=True)
return Response(serializer.data)
def post(self):
pass
it will fix the problem
Try to Remove #property from Person model

crud operation without using serializers

This is the error:
In line 7 and line 9;
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
This is my code:
import requests
BASE_URL = 'http://127.0.0.1:8000/'
ENDPOINT = 'api/'
def get_resource(id):
resp = requests.get(f"{BASE_URL}{ENDPOINT}{id}/")
print(resp.status_code)
print(resp.json())
id = input("enter some id:")
get_resource(id)
Response returns a big data from this site with headers, status code and other...
You should to dump also content of this response. Read there
Try next:
resp.text.json()
Or:
resp.content.json()
content gives you access to the raw bytes of the response payload, you will often want to convert them into a string using a character encoding such as UTF-8
This is my view.py:
from django.shortcuts import render
from django.views.generic import View
from withoutrest.models import employee
from django.http import HttpResponse
import json
from django.core.serializers import serialize
class EmployeeDetails(View):
def get(self, request, id, *args, **kwargs):
emp = employee.objects.get(id=id)
json_data = serialize('json', [emp,])
return HttpResponse(json_data, content_type='application/json')

Multiprocessing, sqlAlchemy and scoped_sessions

I want to run multiple strategies in concurrent processes. I came up with something like this
import logging
import multiprocessing
import os
from sqlalchemy.orm import scoped_session, Session
from pyutil.sql.interfaces.symbols.symbol import Symbol
from pyutil.sql.session import get_one_or_create
class StratRunner(object):
def __init__(self, session_scope, logger=None):
assert isinstance(session_scope, scoped_session)
self.__session_scope = session_scope
self.__logger = logger or logging.getLogger(__name__)
# this function is the target for mp.Process
def _run(self, strategy):
self.__logger.debug("Pid {pid}".format(pid=os.getpid()))
symbols = self.symbols
self.__logger.info("Run strategy {s}".format(s=strategy))
configuration = strategy.configuration()
strategy.upsert(portfolio=configuration.portfolio, symbols=symbols, days=5)
def run_strategies(self):
# loop over all active strategies!
jobs = []
# we are in the main thread here...
for s in self.active_strategies:
# what shall I give to the Process? The strategy object, the strategy_id, a session instance, the session_scope...
job = multiprocessing.Process(target=self._run, kwargs={"strategy": s})
job.name = s.name
jobs.append(job)
run_jobs(jobs, logger=self.__logger)
#property
def symbols(self):
return {s.name: s for s in self.__session_scope().query(Symbol)}
#property
def active_strategies(self):
return self.__session_scope().query(Strategy).filter(Strategy.active == True).all()
I am aware of tons of documentation on this project but I am overwhelmed.
I loop over the rows of a table (The active_strategies). class Strategies(Base)... . I then hand over the strategy object to the _run method and update the strategy object within the very same method. Please feel free to shred my code.
I am in particular puzzled about what to give to the _run method? Shall I hand over the strategy object, the strategy ID, the session, the scoped_session, ... ?
I have now created a runner object:
import abc
import logging
import os
from sqlalchemy.orm import sessionmaker
class Runner(object):
__metaclass__ = abc.ABCMeta
def __init__(self, engine, logger=None):
self.__engine = engine
self._logger = logger or logging.getLogger(__name__)
self.__jobs = []
#property
def _session(self):
""" Create a fresh new session... """
self.__engine.dispose()
factory = sessionmaker(self.__engine)
return factory()
def _run_jobs(self):
self._logger.debug("PID main {pid}".format(pid=os.getpid()))
for job in self.jobs:
# all jobs get the trigge
self._logger.info("Job {j}".format(j=job.name))
job.start()
for job in self.jobs:
self._logger.info("Wait for job {j}".format(j=job.name))
job.join()
self._logger.info("Job {j} done".format(j=job.name))
#property
def jobs(self):
return self.__jobs
#abc.abstractmethod
def run(self):
""" Described in the child class """
In particular this class can provide a fresh session (via ._session). However, using this setup I see plenty of :
psycopg2.OperationalError: server closed the connection unexpectedly
| This probably means the server terminated abnormally
| before or while processing the request.

Scrapy / Pipeline not inserting data to MySQL database

I'm making a pipeline in scrapy to store scraped data in a mysql database. When the spider is run in terminal it works perfectly. Even the pipeline is opened. However the data is not being sent to the database. Any help appreciated! :)
here's the pipeline code:
import sys
import MySQLdb
import hashlib
from scrapy.exceptions import DropItem
from scrapy.http import Request
from tutorial.items import TutorialItem
class MySQLTest(object):
def __init__(self):
db = MySQLdb.connect(user='root', passwd='', host='localhost', db='python')
cursor = db.cursor()
def process_item(self, spider, item):
try:
cursor.execute("INSERT INTO info (venue, datez) VALUES (%s, %s)", (item['artist'], item['date']))
self.conn.commit()
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
return item
and heres the spider code
import scrapy # Import required libraries.
from scrapy.selector import HtmlXPathSelector # Allows for path detection in a websites code.
from scrapy.spider import BaseSpider # Used to create a simple spider to extract data.
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor # Needed for the extraction of href links in HTML to crawl further pages.
from scrapy.contrib.spiders import CrawlSpider # Needed to make the crawl spider.
from scrapy.contrib.spiders import Rule # Allows specified rules to affect what the link
import spotipy
import soundcloud
import mysql.connector
from tutorial.items import TutorialItem
class AllGigsSpider(CrawlSpider):
name = "allGigs" # Name of the Spider. In command promt, when in the correct folder, enter "scrapy crawl Allgigs".
allowed_domains = ["www.allgigs.co.uk"] # Allowed domains is a String NOT a URL.
start_urls = [
"http://www.allgigs.co.uk/whats_on/London/clubbing-1.html",
"http://www.allgigs.co.uk/whats_on/London/festivals-1.html",
"http://www.allgigs.co.uk/whats_on/London/comedy-1.html",
"http://www.allgigs.co.uk/whats_on/London/theatre_and_opera-1.html",
"http://www.allgigs.co.uk/whats_on/London/dance_and_ballet-1.html"
] # Specify the starting points for the web crawler.
rules = [
Rule(SgmlLinkExtractor(restrict_xpaths='//div[#class="more"]'), # Search the start URL's for
callback="parse_me",
follow=True),
]
def parse_me(self, response):
for info in response.xpath('//div[#class="entry vevent"]|//div[#class="resultbox"]'):
item = TutorialItem() # Extract items from the items folder.
item ['artist'] = info.xpath('.//span[#class="summary"]//text()').extract() # Extract artist information.
item ['date'] = info.xpath('.//span[#class="dates"]//text()').extract() # Extract date information.
#item ['endDate'] = info.xpath('.//abbr[#class="dtend"]//text()').extract() # Extract end date information.
#item ['startDate'] = info.xpath('.//abbr[#class="dtstart"]//text()').extract() # Extract start date information.
item ['genre'] = info.xpath('.//div[#class="header"]//text()').extract()
yield item # Retreive items in item.
client = soundcloud.Client(client_id='401c04a7271e93baee8633483510e263')
tracks = client.get('/tracks', limit=1, license='cc-by-sa', q= item['artist'])
for track in tracks:
print(tracks)
I believe the problem was in my settings.py file where i had missed a comma... yawn.
ITEM_PIPELINES = {
'tutorial.pipelines.MySQLTest': 300,
}