Using the JSON response from 3rd party api with geodjango - json

I am try to create a way to narrow down posts on a django app by location & radius. The way I am trying to do this is:
Use 3rd party (OpenCage) API to convert location to lat / lng
Gather the JSON response, filter for the most correct one (first response in US)
Use the response lat & lng to filter listings via GIS
What I have done is successfully connect to the API, and get a lat & lng back, however I am having to filter the response in my templates rather than in my models. I have also found the following which appears to be the solution needed for narrowing results in a radius:
from django.contrib.gis.geos import Point
from django.contrib.gis.measure import Distance
lat = 52.5
lng = 1.0
radius = 10
point = Point(lng, lat)
Place.objects.filter(location__distance_lt=(point, Distance(km=radius)))
What I need to solve, and am hoping for your help with is how do I
filter the api responses and only get what I need from them (components country, geometry lat, geometry lng)
gather the lat & lng for use in the above code?
views.py
from posts import services
def search(request):
queryset_list = posts.objects.order_by('id')
if 'location' in request.GET:
q = request.GET['location']
locations_list = services.get_location(q)
context = {
'posts': queryset_list,
'locations_list': locations_list
}
return render(request, 'posts/search.html', context)
services.py
import requests
def get_location(location):
url = 'https://api.opencagedata.com/geocode/v1/json'
params = {'q': location, 'key': '###', 'language': 'en', 'pretty': 1}
r = requests.get(url, params=params)
locations = r.json()
return locations

I think you can try like this(Based on demo JSON response provided by OpenCage):
import requests
from django.db.models import Q
def get_location(location):
url = 'https://api.opencagedata.com/geocode/v1/json'
params = {'q': location, 'key': '###', 'language': 'en', 'pretty': 1}
r = requests.get(url, params=params)
locations = r.json()
filter_query = Q()
radius = 10
for item in locations.get('results', []):
geo = item.get('geometry')
lat = geo.get('lat')
lng = geo.get('lng')
point = Point(lng, lat)
filter_query |= Q(location__distance_lt=(point, Distance(km=radius)))
return Place.objects.filter(filter_query)

Related

why does JSON dump doesn't work in my code?

I'm trying to put python objects into a JSON file by getting the API from one of the sites but somehow when I run the code nothing has been put in the JSON file. API is working well, as well when I print out the code by json.load I get the output but I have no idea why does dump doesn't work.
here is my code:
from django.shortcuts import render
import requests
import json
import datetime
import re
def index(request):
now = datetime.datetime.now()
format = "{}-{}-{}".format(now.year, now.month, now.day)
source = []
author = []
title = []
date = []
url = "http://newsapi.org/v2/everything"
params = {
'q': 'bitcoin',
'from': format,
'sortBy': 'publishedAt',
'apiKey': '1186d3b0ccf24e6a91ab9816de603b90'
}
response = requests.request("GET", url, params=params)
for news in response.json()['articles']:
matching = re.match("\d+-\d+-\d+", news['publishedAt'])
if format == matching.group():
source.append(news['source'])
author.append(news['author'])
title.append(news['title'])
date.append(news['publishedAt'])
data = \
{
'source': source,
'author': author,
'title': title,
'date': date
}
with open('data.json', "a+") as fp:
x = json.dump(data, fp, indent=4)
return render(request, 'news/news.html', {'response': response})

Save json data into my db using django and postgresql

I'm working with django and postgres, I've a large json data that i want to save into my db but idk how.
My json data looks like this:
{
"Name": "some name",
"Direction": "Street name",
"City": "City",
"Region": "Region",
"DirectionGeo": "Street with geopy",
"Category": "Category",
"ExternalCode": "Code",
"lat": latitude,
"lon": longitude
}
That's all the data I want to save, can someone give me an advice on how I can save my data into the db using python please
this is my model.py:
from django.db import models
class Bidder(models.Model):
name = models.CharField(max_length=500)
direction = models.CharField(max_length=500)
city = models.CharField(max_length=500)
region = models.CharField(max_length=500)
category = models.CharField(max_length=500)
geopy_direction = models.CharField(max_length=500)
lat = models.FloatField(20)
lon = models.FloatField(20)
So if you were trying to consume API data and want to store in your database using python, may this help take reference or in-case i lack to provide correct information, please do let me know
import pymysql
import requests
BASE_URL = 'http://127.0.0.1:8000/'
ENDPOINT = 'api/'
json_data = requests.get(BASE_URL+ENDPOINT)
data = json_data.json()
list_data = []
for edata in data:
list_data.append((edata['EmployeeNo'], edata['Name'], edata['Salary'], edata['Address']))
conn = pymysql.connect(user='root', password='Ashish#123', db='black')
cursor = conn.cursor()
query = 'insert into black_table(EmployeeNo, Name, Salary, Address) values(%s, %s, %s, %s)'
data = list_data
try:
cursor.executemany(query, data)
conn.commit()
print("Data stored to database")
except conn.DatabaseError as message:
if conn:
conn.rollback()
print("Error occured", message)
finally:
if cursor:
cursor.close()
if conn:
conn.close()

Return valid GeoJSON with Flask + GeoAlchemy2 app

Since few days I am wondering how to make my Flask app return valid GeoJSON, here is what I got so far:
models.py
class Building(base):
__tablename__ = 'buildings'
id = Column(Integer, primary_key=True)
district = Column(Unicode)
address = Column(Unicode)
name = Column(Unicode)
building_type = Column(Unicode)
mpoly = Column(Geometry('MULTIPOLYGON'))
building = relationship("User")
# this part is done as answered here: https://stackoverflow.com/questions/41684512/cant-transform-geometry-to-geojson
def building_to_dict(self):
return mapping(shapely.wkb.loads(str(self.mpoly), True))
def __str__(self):
d = dict()
d["id"] = self.id
d["district"] = self.district
d["address"] = self.address
d["name"] = self.name
d["building_type"] = self.building_type
d["mpoly"] = self.building_to_dict()
return shapely.dumps(d)
Now at main file I have following routing:
app.py
#app.route('/geojson')
def get_json():
features = session.query(Building.mpoly.ST_AsGeoJSON()).all()
return jsonify(features)
And here are my two problems:
1)
Returned JSON-like response that looks like following:
"{\"type\":\"MultiPolygon\",\"coordinates\":[[[[16.8933137,52.471446],...,]]]}"
Is not proper GeoJSON.
Features variable before jsonify looks this way:
[('{"type":"MultiPolygon","coordinates":[[[[16.914159616,52.473822807],...,]]]}',)]
2)
How my GeoAlchemy query should look like, to return not just geometry field, but others as well?
Any kind of hints or helps highly appreciated, thanks in advance!
You may use marshmallow-sqlalchemy for creating json like responses.
create
schemas.py
#!schemas.py
from marshmallow import fields
from marshmallow_sqlalchemy import ModelSchema
from app.models import Building
from geoalchemy.shape import to_shape
class BuildingSchema(ModelSchema):
id = fields.Integer()
district = fileds.Unicode()
address = fileds.Unicode()
name = fields.Unicode()
building_type = fields.Unicode()
mpoly = fileds.Method("geom_to_json")
#staticmethod
def geom_to_json(obj):
mpoly = to_shape(obj.mpoly)
return {
lat: mpoly.y,
lon: mpoly.x,
#and so on ...
}
class Meta:
model = Building
exclude = ("mpoly")
bulding_schema = BuildingSchema(many=True)
after this you can use it in your views(routes)
from app.schemas import building_schema
#app.route('/geojson')
def json():
buildings = Buildings.query.all()
response = building_schema.dumps(buildings)
return response

Is directions Api not accepting negative values for coordinates ? Flutter

I'm sending a direction request to an API and I'm facing this problem.
With coordinates as flutter: start location is :latitude 37.33101308, longitude -122.03065487 and end location is :latitude 37.33097983, longitude -122.03063943 the - symbol creates a problem.
When sending the resulting request
http://www.yournavigation.org/api/1.0/gosmore.php?format=geojson&v=bicycle&fast=0&layer=mapnik&flat=37.33101308&flon=-122.03065487&tlat=37.33097983&tlon=-122.03063943&geometry=1&instructions=1&lang=it
I get an error that I'm not yet able to understand:
[VERBOSE-2:ui_dart_state.cc(157)] Unhandled Exception:
FormatException: Unexpected character (at character 1) ^
0 _ChunkedJsonParser.fail (dart:convert-patch/convert_patch.dart:1394:5)
1 _ChunkedJsonParser.parseNumber (dart:convert-patch/convert_patch.dart:1261:9)
2 _ChunkedJsonParser.parse (dart:convert-patch/convert_patch.dart:926:22)
3 _parseJson (dart:convert-patch/convert_patch.dart:31:10)
4 JsonDecoder.convert (dart:convert/json.dart:495:36)
5 JsonCodec.decode (dart:convert/json.dart:153:41)
6 jsonDecode (dart:convert/json.dart:96:10)
7 DirectionsRepository.postRequest (package:fixit_cloud_biking/directions/directions_repository.dart:39:26)
8 DirectionsBloc.getDirections (package:fixit_cloud_biking/directions/directions_bloc.dart:46:12)
9 _AsyncStarStreamController.runBody (dart:async-patch/async_patch.dart:155:5)
10 _rootRun (dart:async/zone.dart:1122:3<…>
Is it because of a json formatting issue or does it means that www.yournavigation.org/api doesn't accept negative coordinates values ?
This error doesn't happen if the sent request has positive coordinates as this
'http://www.yournavigation.org/api/1.0/gosmore.php?format=geojson&v=bicycle&fast=0&layer=mapnik&flat=44.5018645003438&flon=11.340018709036542&tlat=44.502138&tlon=11.340402&geometry=1&instructions=1&lang=it'
Just for testing I tried just making them positive coordinates with .abs(). but of course the results is a wrong location on the map.
Is there another coordinate system that has all positive values?
Thank you very much for your help.
This is the class I'm sending the request from:
import 'package:http/http.dart';
import 'package:latlong/latlong.dart';
import 'dart:convert' as convert;
import 'dart:math';
class DirectionsRepository {
Future<List<LatLng>> postRequest(LatLng start, LatLng end) async {
print('postRequest() called');
print(
'start location is :latitude ${start.latitude}, longitude ${start.longitude}');
print(
'end location is :latitude ${end.latitude}, longitude ${end.longitude}');
// NORMAL NEGATIVE LONGITUDE THROWS = API ERROR:
// format error [VERBOSE-2:ui_dart_state.cc(157)] Unhandled Exception: FormatException: Unexpected character (at character 1)
String flat = (start.latitude).toString();
String flon = (start.longitude).toString();
String tlat = (end.latitude).toString();
String tlon = (end.longitude).toString();
// ALL POSITIVE VALUES DON'T THROW AN ERROR BUT IS WRONG PLACE
// String flat = (start.latitude).abs().toString();
// String flon = (start.longitude).abs().toString();
// String tlat = (end.latitude).abs().toString();
// String tlon = (end.longitude).abs().toString();
final request =
'http://www.yournavigation.org/api/1.0/gosmore.php?format=geojson&v=bicycle&fast=0&layer=mapnik&flat=$flat&flon=$flon&tlat=$tlat&tlon=$tlon&geometry=1&instructions=1&lang=it';
print('final request is $request');
// working properly
// final request =
// 'http://www.yournavigation.org/api/1.0/gosmore.php?format=geojson&v=bicycle&fast=0&layer=mapnik&flat=44.5018645003438&flon=11.340018709036542&tlat=44.502138&tlon=11.340402&geometry=1&instructions=1&lang=it';
// Await the http get response, then decode the json-formatted response.
var response = await get(request);
if (response.statusCode == 200) {
var jsonResponse = convert.jsonDecode(response.body);
print('${jsonResponse.runtimeType} : $jsonResponse');
List<dynamic> coordinates = jsonResponse['coordinates'];
print('coordinates are : $coordinates');
print('coordinates are: ${coordinates.length}');
Map<String, dynamic> properties = jsonResponse['properties'];
// print('properties are $properties');
String distance = properties['distance'];
print('Route is $distance Km long.');
String instructions = properties['description'];
print('instructions are $instructions');
List<LatLng> suggestedRoute = [];
for (int i = 0; i < (coordinates.length); i++) {
dynamic coordinate = coordinates[i];
LatLng position = LatLng(coordinate[1], coordinate[0]);
suggestedRoute.add(position);
print('position is $position');
print(i);
}
print('postRequest() suggestedRoute is $suggestedRoute');
return suggestedRoute;
} else {
print(
'postRequest() Request failed with status: ${response.statusCode}.');
}
}
}
After a few tries, I've nailed down the problem to be a direction API server problem. So, request is fine, but method's json decoding just fails because gets a 242 response status code which passes the check but fails as the response body does not match what's handled in the method.

GeoAlchemy2: Get the lat, lon of a point

Consider the following SQLAalchemy / GeoAlchemy2 ORM with a geometry field:
from geoalchemy2 import Geometry, WKTElement
class Item(Base):
__tablename__ = 'item'
id = Column(Integer, primary_key=True)
...
geom = Column(Geometry(geometry_type='POINTZ', srid=4326))
When I update an item in the PostgreSQL shell:
UPDATE item SET geom = st_geomFromText('POINT(2 3 0)', 4326) WHERE id = 5;
Fetching the field:
items = session.query(Item).\
filter(Item.id == 3)
for item in items:
print item.geom
Gives:
01e9030000000000000000004000000000000008400000000000000000
This isn't a proper WKB - at least, it does not parse with Shapely's loads.
How do I get the lat/lon of the geom field?
Fetching the lat, lon via ST_X and ST_Y might not be the most elegant approach, but it works:
from sqlalchemy import func
items = session.query(
Item,
func.st_y(Item.geom),
func.st_x(Item.geom)
).filter(Item.id == 3)
for item in items:
print(item.geom)
Gives:
(<Item 3>, 3.0, 2.0)
geoalchemy2 to_shape function to convert a :class:geoalchemy2.types.SpatialElement
to a Shapely geometry.
in Item class :
from geoalchemy2.shape import to_shape
point = to_shape(self.geo)
return {
'latitude': point.y,
'longitude': point.x
}
Point type syntax is
Point ( Lat, Long)
So basic from mosi_kha's answer, return should be
from geoalchemy2.shape import to_shape
point = to_shape(self.geo)
return {
'latitude': point.x,
'longitude': point.y
}