Save class items as json file in python scrapy - json

Question

I want to save all data of both these classes (Product_Items and Variant_Product) as JSON output files.
getProductDetails() : In this function I want to extract the data for just 1st element in product_variants list and ading it to the dict(item_list) and for rest of the elements I am creating a req to hit the same function recursively untill I have all the keys in my dict(item_list).
At the end of the function I want to write the extracted data as JSON file, but I can't return two values from a function.
Similarly, in getListingDetails() function I need to save the item as JSON file. PLEASE HELP!!!
Following is the snippet:
import scrapy
from scrapy.http import Request
from scrapy.selector import Selector
from scrapy.item import Item, Field
import re,json
class Product_Items(Item):
Image_URL = Field()
Product_Title = Field()
Price = Field()
PPU_Price = Field()
Product_URL = Field()
Product_SKU = Field()
Product_UPC = Field()
class Variant_Product(Item):
Image_URL = Field()
Product_Title = Field()
Price = Field()
PPU_Price = Field()
Product_URL = Field()
Product_SKU = Field()
Product_UPC = Field()
Product_Size = Field()
Meta = Field()
class walmartSpider(scrapy.Spider):
name = "walmart"
start_urls = ['https://www.walmart.com/all-departments']
item_list = {}
def parse(self,response):
reqs = []
base_url='https://www.walmart.com/'
hxs = Selector(text=response.body)
json_response = hxs.xpath('//script[#id="home"]//text()').get()
data = json.loads(json_response)
cat_urls = self.getCategoryUrls(data)
for url in cat_urls:
if url[:7] == '/browse':
url = base_url + url
link=Request(url=url,callback=self.getListingDetails)
reqs.append(link)
return reqs
def getCategoryUrls(self,data):
.....
return final_cat_url
def getListingDetails(self,response):
reqs = []
hxs = Selector(text=response)
data = json.loads(hxs.xpath('//script[#id="searchContent"]//text()').get())
products = data['searchContent']['preso']['items']
item = Product_Items()
for product in products:
item['Image_URL'] = product['imageUrl']
item['Product_Title'] = product['title']
item['Product_URL'] = base_url + product['productPageUrl']
item['Product_SKU'] = product['productId']
item['Product_UPC'] = product['standardUpc'][0]
try:
item['PPU_Price'] = product['primaryOffer']['unitPriceDisplayCondition']
except:
item['PPU_Price'] = ''
try:
regular_price = product['primaryOffer']['offerPrice']
except:
regular_price = ''
if regular_price:
item['Price'] = product['primaryOffer']['offerPrice']
else:
product_req = Request(url=item['Product_URL'],callback=self.getProductDetails)
reqs.append(product_req)
**Want to save this item as JSON file**
**#Pagination**
try:
next_page = data['searchContent']['preso']['pagination']['next']['url']
except:
next_page = ''
if next_page:
next_page_url = str(re.findall(r'^[\S]+\?',response.url)[0])+str(next_page)
req = Request(url=next_page_url,callback=self.getListingDetails)
reqs.append(req)
return reqs
def getProductDetails(self,response):
reqs = []
base_url = 'https://www.walmart.com/ip/'
hxs = Selector(text=response)
variant = Variant_Product()
prod_data = json.loads(hxs.xpath('//script[#id="item"]//text()').get())
product_variants = prod_data['item']['product']['buyBox']['products']
for product_variant in product_variants[1:]:
item_id = product_variant['usItemId']
if item_id not in self.item_list.keys():
self.item_list[item_id] = ''
req = Request(url=base_url+str(item_id),callback=self.getProductDetails)
reqs.append(req)
product_0 = prod_data['item']['product']['buyBox']['products'][0]
variant['Product_Title'] = product_0['productName']
variant['Product_SKU'] = product_0['walmartItemNumber']
variant['Product_UPC'] = product_0['upc']
variant['Product_Size'] = product_0['variants'][0]['value']
variant['Product_URL'] = product_0['canonicalUrl ']
variant['Price'] = product_0['priceMap']['price']
variant['PPU_Price'] = product_0['priceMap']['unitPriceDisplayValue']
variant['Meta'] = (product_0['categoryPath']).replace('Home Page/','')
**Want to save this item as JSON file**
return reqs

Answer 1

According to the scrapy docs, there are several built in "Exporters" that can serialize your data into several different formats (including JSON).
You should be able to do something like:
# ...
from scrapy.exporters import JsonItemExporter
# ...
def getListingDetails(self, response):
# ...
for product in products:
item = Product_Items(
Image_URL = product['imageUrl'],
Product_Title = product['title'],
Product_URL = base_url + product['productPageUrl'],
Product_SKU = product['productId'],
Product_UPC = product['standardUpc'][0],
PPU_Price = product.get('primaryOffer', {}).get('unitPriceDisplayCondition', ''),
Price = product.get('primaryOffer', {}).get('offerPrice', '')
)
if not item['Price']:
product_req = Request(url=item['Product_URL'],callback=self.getProductDetails)
reqs.append(product_req)
JsonItemExporter(open(f"{item['Product_SKU']}.json", "wb")).export_item(item)
Some notes:
The JsonItemExporter.__init__ method expects a file-like object whose write method accepts bytes, which is why the "wb"
dict.get() in Python allows you to specify a default value as the second argument, in case a key doesn't exist (not strictly necessary here, but reduces the try/except logic)
When handling exceptions, it's recommended by PEP8 standards to catch more specific exception types (in the above cases, except KeyError: might be appropriate) than just a bare except clause
Please let me know if the above works for you!