I have following piece of code:
def detail(request, popular_id):
try:
popular = Popular.objects.get(pk = popular_id)
share = Share.objects.get(isin = popular.isin) #LINE 1
chart_data_json = share.get_chart_data_json()
except Popular.DoesNotExist:
raise Http404
return render(request, 'popular/detail.html', {'popular': popular, 'chart_data': chart_data_json})
In LINE 1 I noticed using debug-toolbar that there are two queries get executed:
SELECT `share_share`.`id`, `share_share`.`symbol`, `share_share`.`isin`, `share_share`.`name`, `share_share`.`market`, `share_share`.`updated` FROM `share_share` WHERE `share_share`.`id` = 1
and
SELECT `share_share`.`id`, `share_share`.`symbol`, `share_share`.`isin`, `share_share`.`name`, `share_share`.`market`, `share_share`.`updated` FROM `share_share` WHERE `share_share`.`isin` = 'US5949181045'
I cannot understand why we need the first query and how to avoid it?
EDIT:
Model definition of share:
class Share(models.Model):
symbol = models.CharField(max_length = 32)
isin = models.CharField(max_length = 12)
name = models.CharField(max_length = 256)
market = models.CharField(max_length = 64)
updated = models.BooleanField(default = False)
def get_chart_data_json(self):
quote_model = create_quote_model(str(self.isin))
data = quote_model.objects.values('date', 'adj_close', 'volume')
chart_data = []
for d in data.iterator():
chart_data.append({'date': d['date'].isoformat(), 'value': d['adj_close'], 'volume': d['volume']})
chart_data_json = json.dumps(chart_data)
return chart_data_json
def __unicode__(self):
return self.isin
Model definition of popular:
class Popular(models.Model):
title = models.CharField(max_length = 120)
text = models.CharField(max_length = 1024)
isin = models.ForeignKey(Share)
def __unicode__(self):
return self.title
First query is evaluated when you access foreign key isin from popular object:
share = Share.objects.get(isin = popular.isin)
Second query gets Share object:
share = Share.objects.get(isin = popular.isin)
If you want just one query at #LINE 1 you should replace it with:
share = popular.isin #LINE 1
Related
I'm trying to show manytomany data in json format(without using serializer), here is my models.py
class CustomerInvoice(models.Model):
customer = models.CharField(max_length=50)
items_model = models.ManyToManyField(Item,through='InvoiceItem')
created_at = models.DateTimeField(auto_now_add=True)
class InvoiceItem(models.Model):
item = models.ForeignKey(Item,on_delete=models.CASCADE)
invoice = models.ForeignKey(CustomerInvoice,on_delete=models.CASCADE,related_name='invoice')
quantity = models.IntegerField()
price = models.DecimalField(max_digits=20,decimal_places=2)
is it possible to make a look up base on many to many data?
something like this : Q(items_model__icontains=query_search) ,and also how to return the M2M data into a json format using values() and json.dumps please? this returns the ID Values('items_model') and this dont work Values('items_model__all')
and here is my views.py
def invoices_all_lists(request):
if request.is_ajax():
query_search = request.GET.get('filter')
if query_search:
all_item_qs = CustomerInvoice.objects.all()
a = []
for i in all_item_qs.items_model.all():
a.append(i.item.name)
invoices = CustomerInvoice.objects.annotate(
total=Sum((F('invoice__quantity') * F('invoice__price')),output_field=DecimalField(decimal_places=2,max_digits=20))
).filter(
Q(id__icontains=query_search) | Q(seller__username__icontains=query_search) |
Q(customer__icontains=query_search)).values(
'id','seller__username','customer','total','created_at','items_model').order_by('-id')
else:
all_item_qs = CustomerInvoice.objects.all()
a = []
for data in all_item_qs:
for i in data.items_model.all():
a.append(i.item.name)
invoices = CustomerInvoice.objects.annotate(
total=Sum((F('invoice__quantity') * F('invoice__price')) ,output_field=DecimalField(decimal_places=2,max_digits=20))
).values(
'id','seller__username','customer','total','created_at','items_model').order_by('-id')
start_from = 0
if request.GET.get('start'):
start_from = int(request.GET.get('start'))
limit = 10
if request.GET.get('limit'):
limit = int(request.GET.get('limit'))
data_lists = []
for index,value in enumerate(invoices[start_from:start_from+limit],start_from):
value['counter'] = index+1
data_lists.append(value)
data = {
'objects':data_lists,
'length':invoices.count(),
}
return HttpResponse(json.dumps(data, indent=4, sort_keys=True, default=str),'application/json')
else:
return redirect('invoiceapp:list-all-invoice')
can i add this part of the code into the query please?
a = []
for data in all_item_qs:
for i in data.items_model.all():
a.append(i.item.name)
note : i've used datatable server side in the client side
I want to save all data of both these classes (Product_Items and Variant_Product) as JSON output files.
getProductDetails() : In this function I want to extract the data for just 1st element in product_variants list and ading it to the dict(item_list) and for rest of the elements I am creating a req to hit the same function recursively untill I have all the keys in my dict(item_list).
At the end of the function I want to write the extracted data as JSON file, but I can't return two values from a function.
Similarly, in getListingDetails() function I need to save the item as JSON file. PLEASE HELP!!!
Following is the snippet:
import scrapy
from scrapy.http import Request
from scrapy.selector import Selector
from scrapy.item import Item, Field
import re,json
class Product_Items(Item):
Image_URL = Field()
Product_Title = Field()
Price = Field()
PPU_Price = Field()
Product_URL = Field()
Product_SKU = Field()
Product_UPC = Field()
class Variant_Product(Item):
Image_URL = Field()
Product_Title = Field()
Price = Field()
PPU_Price = Field()
Product_URL = Field()
Product_SKU = Field()
Product_UPC = Field()
Product_Size = Field()
Meta = Field()
class walmartSpider(scrapy.Spider):
name = "walmart"
start_urls = ['https://www.walmart.com/all-departments']
item_list = {}
def parse(self,response):
reqs = []
base_url='https://www.walmart.com/'
hxs = Selector(text=response.body)
json_response = hxs.xpath('//script[#id="home"]//text()').get()
data = json.loads(json_response)
cat_urls = self.getCategoryUrls(data)
for url in cat_urls:
if url[:7] == '/browse':
url = base_url + url
link=Request(url=url,callback=self.getListingDetails)
reqs.append(link)
return reqs
def getCategoryUrls(self,data):
.....
return final_cat_url
def getListingDetails(self,response):
reqs = []
hxs = Selector(text=response)
data = json.loads(hxs.xpath('//script[#id="searchContent"]//text()').get())
products = data['searchContent']['preso']['items']
item = Product_Items()
for product in products:
item['Image_URL'] = product['imageUrl']
item['Product_Title'] = product['title']
item['Product_URL'] = base_url + product['productPageUrl']
item['Product_SKU'] = product['productId']
item['Product_UPC'] = product['standardUpc'][0]
try:
item['PPU_Price'] = product['primaryOffer']['unitPriceDisplayCondition']
except:
item['PPU_Price'] = ''
try:
regular_price = product['primaryOffer']['offerPrice']
except:
regular_price = ''
if regular_price:
item['Price'] = product['primaryOffer']['offerPrice']
else:
product_req = Request(url=item['Product_URL'],callback=self.getProductDetails)
reqs.append(product_req)
**Want to save this item as JSON file**
**#Pagination**
try:
next_page = data['searchContent']['preso']['pagination']['next']['url']
except:
next_page = ''
if next_page:
next_page_url = str(re.findall(r'^[\S]+\?',response.url)[0])+str(next_page)
req = Request(url=next_page_url,callback=self.getListingDetails)
reqs.append(req)
return reqs
def getProductDetails(self,response):
reqs = []
base_url = 'https://www.walmart.com/ip/'
hxs = Selector(text=response)
variant = Variant_Product()
prod_data = json.loads(hxs.xpath('//script[#id="item"]//text()').get())
product_variants = prod_data['item']['product']['buyBox']['products']
for product_variant in product_variants[1:]:
item_id = product_variant['usItemId']
if item_id not in self.item_list.keys():
self.item_list[item_id] = ''
req = Request(url=base_url+str(item_id),callback=self.getProductDetails)
reqs.append(req)
product_0 = prod_data['item']['product']['buyBox']['products'][0]
variant['Product_Title'] = product_0['productName']
variant['Product_SKU'] = product_0['walmartItemNumber']
variant['Product_UPC'] = product_0['upc']
variant['Product_Size'] = product_0['variants'][0]['value']
variant['Product_URL'] = product_0['canonicalUrl ']
variant['Price'] = product_0['priceMap']['price']
variant['PPU_Price'] = product_0['priceMap']['unitPriceDisplayValue']
variant['Meta'] = (product_0['categoryPath']).replace('Home Page/','')
**Want to save this item as JSON file**
return reqs
According to the scrapy docs, there are several built in "Exporters" that can serialize your data into several different formats (including JSON).
You should be able to do something like:
# ...
from scrapy.exporters import JsonItemExporter
# ...
def getListingDetails(self, response):
# ...
for product in products:
item = Product_Items(
Image_URL = product['imageUrl'],
Product_Title = product['title'],
Product_URL = base_url + product['productPageUrl'],
Product_SKU = product['productId'],
Product_UPC = product['standardUpc'][0],
PPU_Price = product.get('primaryOffer', {}).get('unitPriceDisplayCondition', ''),
Price = product.get('primaryOffer', {}).get('offerPrice', '')
)
if not item['Price']:
product_req = Request(url=item['Product_URL'],callback=self.getProductDetails)
reqs.append(product_req)
JsonItemExporter(open(f"{item['Product_SKU']}.json", "wb")).export_item(item)
Some notes:
The JsonItemExporter.__init__ method expects a file-like object whose write method accepts bytes, which is why the "wb"
dict.get() in Python allows you to specify a default value as the second argument, in case a key doesn't exist (not strictly necessary here, but reduces the try/except logic)
When handling exceptions, it's recommended by PEP8 standards to catch more specific exception types (in the above cases, except KeyError: might be appropriate) than just a bare except clause
Please let me know if the above works for you!
Try to serialize this Models
Model:
class Order (models.Model):
id = models.AutoField(primary_key=True)
date_create = models.DateField(auto_now_add=True)
date_change = models.DateField(auto_now=True)
summ =models.CharField(max_length=15,default='0')
delivery = models.ForeignKey('Delivery')
success = models.BooleanField(default=False)
paymentMethod = models.ForeignKey('Payments')
def __unicode__(self):
return unicode(self.id)
class OrderProduct(models.Model):
order=models.ForeignKey('Order')
id = models.AutoField(primary_key=True)
date_create = models.DateField(auto_now_add=True)
date_change = models.DateField(auto_now=True)
price = models.IntegerField()
product = models.ForeignKey('product.Product')
additionals = models.IntegerField(null=True,default=0)
count = models.IntegerField()
def __unicode__(self):
return self.id
class Delivery(models.Model):
id = models.AutoField(primary_key=True)
date_create = models.DateField(auto_now_add=True)
date_change = models.DateField(auto_now=True)
delivery_time = models.DateTimeField()
delivery_adress = models.TextField()
phone = models.TextField()
def __unicode__(self):
return self.phone
class Payments(models.Model):
id = models.AutoField(primary_key=True)
date_create = models.DateField(auto_now_add=True)
date_change = models.DateField(auto_now=True)
title = models.TextField();
def __unicode__(self):
return self.title
Serializers:
class DeliverySerializer(serializers.ModelSerializer):
class Meta:
model = Delivery
fields = ('id', 'delivery_time','delivery_adress','phone')
def create(self, validated_data):
return Delivery.objects.create(**validated_data)
class PaymentsSerializer(serializers.ModelSerializer):
class Meta:
model = Payments
fields = ('id', 'title')
def create(self, validated_data):
return Payments.objects.create(**validated_data)
class OrderSerializer(serializers.ModelSerializer):
delivery = DeliverySerializer(read_only=True)
paymentMethod = PaymentsSerializer(read_only=True)
class Meta:
model = Order
fields = ('id', 'delivery', 'paymentMethod','summ','success')
def create(self, validated_data):
deliverys_data = validated_data.pop('delivery')
paymentsMethod_data = validated_data.pop('paymentMethod')
order = Order.objects.create(**validated_data)
for delivery_data in deliverys_data:
Delivery.objects.create(order=order, **delivery_data)
for paymentMethod_data in paymentsMethod_data:
Payments.objects.create(order=order, **paymentMethod_data)
return order
View:
#api_view(['POST'])
def order_post(request, format=None):
#List all snippets, or create a new snippet.
if request.method == 'POST':
serializer = OrderSerializer(data=request.data)
if serializer.is_valid():
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
I need to get data by 1 packet, and then write data in DB
EveryTime I get a error:
deliverys_data = validated_data.pop('delivery')
KeyError: 'delivery'
Example of JSON packet
[{"delivery":{"delivery_time":"2016-05-31T12:18:47Z","delivery_adress":"123","phone":"123"},"paymentMethod":{"id":1,"title":"123124123"},"summ":"23","success":false}]
You are getting KeyError: 'delivery' because you have set delivery field as read_only. If DRF finds this field in the input, it will ignore that field.
From docs on read_only argument:
Read-only fields are included in the API output, but should not be
included in the input during create or update operations. Any
'read_only' fields that are incorrectly included in the serializer
input will be ignored.
Also, since you are using paymentMethod field in the create() method, you need to tell DRF to consider that field also in the input.
So, you need to remove the read_only argument from your serializer for both delivery and paymentMethod fields so that these fields are considered when deserializing.
class OrderSerializer(serializers.ModelSerializer):
delivery = DeliverySerializer() # remove read_only argument
paymentMethod = PaymentsSerializer() # remove read_only argument
Secondly, you are sending the data incorrectly. You need to send a single order input instead of list of orders you are sending.
# send without the list
{"delivery":{"delivery_time":"2016-05-31T12:18:47Z","delivery_adress":"123","phone":"123"},"paymentMethod":{"id":1,"title":"123124123"},"summ":"23","success":false}
EveryTime I get a error: deliverys_data = validated_data.pop('delivery') KeyError: 'delivery'
This is to be expected since the OrderSerialzier has DeliverySerializer flagged as read_only.
If you want to get the data writable, you'll need to remove that flag first.
I am trying to get only approved responses (staged>1) but for some reason the Reply.staged>1 in the primaryjoin is not working
I am very new to sqlAlchemy so I am not really sure what the relationship for parent is doing with the remote_side
class Story(Base):
__tablename__ = ’Story'
id = Column(‘ID', Integer, primary_key=True)
anonymous = Column(TINYINT)
detail = Column('detail',String, js_name='detail')
summary = Column('summary',String, js_name='questionSummary')
user_id = Column('uid', Integer, ForeignKey('rlUser.uid'), js_name='userId')
user = relationship("User")
inspire_id = Column('inspireID', Integer, js_name='inspireId')
staged = Column(TINYINT)
class Reply(Base):
__tablename__ = 'Reply'
id = Column('replyID', Integer, primary_key=True)
parent_id = Column('parentID', Integer, ForeignKey('rlReply.replyID'),js_name='parentId')
user_id = Column('userID', Integer, js_name='userId')
detail = Column(String, js_name='detail')
reply_date = Column('replyDate', TIMESTAMP, js_name='replyDate')
expertise_type = Column('expertiseType', Integer, js_name='expertiseType')
status = Column(Integer)
staged = Column(Integer)
story = relationship(“Story",primaryjoin='and_(Reply.story_id == Story.id, Reply.parent_id==0, Reply.staged>1)', backref=backref("replies", uselist=True ))
parent = relationship('Reply',primaryjoin='and_(Reply.parent_id==Reply.id, Reply.staged>1)', backref='responses', remote_side='Reply.id', viewonly=True, uselist=True)
def __init__(self):
self.staged = constants.POST_QUEUE_STATUS['STAGED']
#property
def reply_count(self):
return len(self.responses)
The where clause for story.replies:
WHERE Reply.rushlineID = %s AND Reply.parentID = %s AND Reply.staged > %s
parms sqlalchemy.engine.base.Engine (110L, 0, 1)
Which is perfect, however:
the where clause for story.replies[0].responses:
WHERE Reply.parentID = %s AND %s > %s
params are sqlalchemy.engine.base.Engine (68L, 2, 1)
what I need is WHERE Reply.parentID = %s AND AND Reply.staged > %s AND %s > %s
sqlalchemy.engine.base.Engine (68L, 1, 2, 1)
The magic incantation for this, I have to admit, I got to through trial and a lot of error. Here are the relevant bits:
parent_id = RlColumn('parentID', Integer,
ForeignKey('rlReply.replyID'),
js_name='parentId')
parent = RlRelationship('Reply',
primaryjoin='and_(Reply.parent_id==remote(Reply.id),Reply.flagged_status <2)',
remote_side=id, uselist=True,
foreign_keys=parent_id, backref='replies')
I have a Django app that uses MySQL as a backend. I'm having difficulties where the raw MySQL records show one value, but Django presents something else in the web app.
For example, I have a table for client data. One of the fields in each record is called snailMailInvoice and is a Y/N choice -- default is Y (varchar type).
+-------------------+--------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+-------------------+--------------+------+-----+---------+----------------+
| snailMailInvoice | varchar(3) | NO | | Y | |
The raw MySQL:
select * from systems_system_contact where lastName="SomeClient";
...a bunch of other fields... | snailMailInvoice |
...a bunch of other fields... | N
Then, in the Django App, the form displays Y (the other choice). It is like the Django App can't see the MySQL value, so it defaults to Y. If, through the Django App, I select N and save the form, THEN the value sticks to N in Django.
Why would this be happening?
EDIT - to add some code
Forms.py:
class System_Contact_Form(ModelForm):
class Meta:
model = System_Contact
exclude = ('isMainContact', 'systemOwner', 'companyName', 'isRessyContact')
Views.py:
def contact_details(request, scID):
redirect_to = request.REQUEST.get('next', '/systems/contacts/')
if request.method == 'POST':
syscontEdit = System_Contact.objects.get(pk=scID)
form = System_Contact_Form(request.POST, instance=syscontEdit)
if form.is_valid():
form.save()
return HttpResponseRedirect(redirect_to)
else:
syscontView = System_Contact.objects.get(pk=scID)
form = System_Contact_Form(instance=syscontView)
c = {
'form':form,
'cancel':redirect_to
}
return render_to_response('pages/systems/contact_details.html', c, context_instance=RequestContext(request))
Models.py:
class System_Contact(models.Model):
IS_MAIN_CONTACT_CHOICES = (
('Y', 'Yes'),
('N', 'No'),
)
IS_SYSTEM_OWNER_CHOICES = (
('Y', 'Yes'),
('N', 'No'),
)
IS_RESSY_CONTACT_CHOICES = (
('Y', 'Yes'),
('N', 'No, this is a commercial contact'),
)
TRADE_CHOICES = (
('EL', 'Electrician'),
('LA', 'Landscaper'),
('PL', 'Plumber'),
('TR', 'Trencher'),
)
SNAIL_MAIL_CHOICES = (
('Y', 'Yes'),
('N', 'No'),
)
SNAIL_MAIL_INVOICE_CHOICES = (
('Y', 'Yes'),
('N', 'No'),
)
firstInitial = models.CharField(max_length = 10, verbose_name = 'First Initial', blank = True, null = True)
firstName = models.CharField(max_length = 60, verbose_name = 'First Name', blank = True, null = True)
lastName = models.CharField(max_length = 160, verbose_name = 'Last Name', blank = True, null = True)
phonetically = models.CharField(max_length = 100, verbose_name = 'Phonetically', blank = True, null = True)
companyName = models.CharField (max_length = 160, verbose_name = 'Company Name', blank = True, null = True) #Only used for Commercial Owners, no other field needed
homePhone = models.CharField(max_length = 60, verbose_name = 'Home Phone Number', blank = True, null = True)
officePhone = models.CharField(max_length = 60, verbose_name = 'Office Phone Number', blank = True, null = True)
cellPhone = models.CharField(max_length = 60, verbose_name = 'Cell Phone Number', blank = True, null = True)
faxNumber = models.CharField (max_length= 60, blank=True, null=True, verbose_name = 'Fax Number')
isMainContact = models.CharField (max_length = 3, verbose_name = 'Is the Main Contact?', choices = IS_MAIN_CONTACT_CHOICES, default='N')
isRessyContact = models.CharField (max_length = 3, verbose_name = 'Is this a Ressy Contact?', choices = IS_RESSY_CONTACT_CHOICES, default='Y')
isArchived = models.BooleanField(verbose_name = 'Archived?', default = False)
systemOwner = models.CharField (max_length = 3, verbose_name = 'Is a System Owner?', choices = IS_SYSTEM_OWNER_CHOICES, default='N') #this is just a flag to say they own a system
worksFor = models.CharField (max_length = 70, verbose_name = 'Works For', blank = True, null = True)
tradeType = models.ForeignKey(Contact_Trade, blank=True, null=True, verbose_name='Trade')
emailAddress = models.EmailField(verbose_name = 'Email Address', blank = True, null = True)
billingAddress = models.CharField(max_length = 150, verbose_name = 'Billing Address', blank=True, null=True )
billingCity = models.CharField(max_length = 90, verbose_name = 'Billing City', blank=True, null=True)
billingProvince = models.CharField(max_length = 30, verbose_name = 'Billing Province', blank=True, null=True)
billingPostalCode = models.CharField(max_length = 10, verbose_name = 'Billing Postal Code', blank=True, null=True)
snailMailOnly = models.CharField(max_length = 3, verbose_name = 'Snail Mail Only?', choices = SNAIL_MAIL_CHOICES, default='Y')
snailMailInvoice = models.CharField(max_length = 3, verbose_name = 'Snail Mail Invoice?', choices = SNAIL_MAIL_INVOICE_CHOICES, default='Y')
OK -- I figured it out.
I realised there was something connected with the fact that in my CSV file, the snailMailInvoice field is the LAST field on each line. Thus, at the end of the line there is a carriage return. I assumed that this was a \n -- thus in my MySQL command to import the CSV, I state terminated by '\n'.
However, the MySQL picked up a '\r' on EVERY line and was adding it to the snailMailInvoice field. Thus, every record has either a Y or a N with a \r attached.
I amended my MySQL import statement to have: lines terminated with '\r\n' Now everything is working as expected.
Lesson learned.
Thanks for the help.