Scrapy MySQL pipeline: spider closed before pipeline finished - mysql

Question

I use scrapy to crawl a page which contains a list of items, and I save each of the item in MySQL databases.
But the problem is that I found spider closed before all items are stored in mysql. Each time I ran the spider the result count is different.
Could you please help let me know how to solve this?
Below is my sample code:
Spider
class FutianSpider(scrapy.Spider):
name = 'futian_soufang'
allowed_domain = ["fang.com"]
start_urls = []
def __init__(self, category=None, *args, **kwargs):
self.count = 0
pass
def closed(self, reason):
print "*" * 20 + str(self.count)
def start_requests(self):
url = "http://fangjia.fang.com/pghouse-c0sz/a085-h321-i3{}/"
response = requests.get(url.format(1))
response.encoding = 'gb2312'
strpages = Selector(text=response.text).xpath('//p[contains(#class, "pages")]/span[last()]/a/text()').extract()
# print response.text
pages = int(strpages[0])
for num in range(1, pages + 1):
yield scrapy.Request(url.format(num), callback=self.parse_page)
def parse_page(self, response):
houses = response.xpath("//div[#class='list']//div[#class='house']")
for house in houses:
# house = Selector(house.decode("UTF-8", 'ignore'))
self.count += 1
housespan_hyperlink = house.xpath(".//span[#class='housetitle']/a")
house_title = housespan_hyperlink.xpath("text()").extract()[0].strip()
house_link_rel = housespan_hyperlink.xpath("#href").extract()[0].strip()
house_link = response.urljoin(house_link_rel)
# if isinstance(house_link_rel, list) and len(house_link_rel) > 0:
# house_link = response.urljoin(house_link_rel)
address = house.xpath(".//span[#class='pl5']/text()").extract()[0].strip()
esf_keyword = u'二手房'
esf_span = house.xpath(".//span[contains(text(),'%s')]" % (esf_keyword))
esf_number = esf_span.xpath("./a/text()").extract()[0].strip()
esf_number = int(re.findall(r"\d+", esf_number)[0])
esf_link = esf_span.xpath("./a/#href").extract()[0].strip()
zf_hyperlink = house.xpath(".//span[#class='p110']/a")
zf_number = zf_hyperlink.xpath("text()").extract()[0].strip()
zf_number = int(re.findall(r"\d+", zf_number)[0])
zf_link = zf_hyperlink.xpath("#href").extract()[0].strip()
price = 0
try:
price = int(house.xpath(".//span[#class='price']/text()").extract()[0].strip())
except:
None
change = 0.0
try:
increase_span = house.xpath(".//span[contains(#class, 'nor')]")
changetext = increase_span.xpath("text()").extract()[0].strip()
change = float(changetext[:changetext.index('%')])
if len(increase_span.css(".green-down")) > 0:
change *= -1
except:
None
print house_title, house_link, address, esf_number, esf_link, zf_number, zf_link, price, change
item = XiaoquItem(
title=house_title,
url=house_link,
address=address,
esf_number=esf_number,
esf_link=esf_link,
zf_number=zf_number,
zf_link=zf_link,
price=price,
change=change
)
yield item
Item:
class XiaoquItem(Item):
# define the fields for your item here like:
title = Field()
url = Field()
address = Field()
esf_number = Field()
esf_link = Field()
zf_number = Field()
zf_link = Field()
price = Field()
change = Field()
Pipeline:
class MySQLPipeLine(object):
def __init__(self):
settings = get_project_settings()
dbargs = settings.get('DB_CONNECT')
db_server = settings.get('DB_SERVER')
dbpool = adbapi.ConnectionPool(db_server, **dbargs)
self.dbpool = dbpool
def close_spider(self, spider):
self.dbpool.close()
def process_item(self, item, spider):
if isinstance(item, XiaoquItem):
self._process_plot(item)
elif isinstance(item, PlotMonthlyPriceItem):
self._process_plot_price(item)
return item
def _process_plot(self, item):
# run db query in thread pool
query = self.dbpool.runInteraction(self._conditional_insert, item)
query.addErrback(self._handle_error, item)
# query.addBoth(lambda _: item)
def _conditional_insert(self, conn, item):
# create record if doesn't exist.
# all this block run on it's own thread
conn.execute("select * from houseplot where title = %s", item["title"])
result = conn.fetchone()
if result:
log.msg("Item already stored in db: %s" % item, level = log.DEBUG)
else:
conn.execute("insert into houseplot(title, url, address, esf_number, esf_link, zf_number, zf_link, price, price_change, upsert_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", \
(item["title"], item["url"], item["address"], int(item["esf_number"]), item["esf_link"], item["zf_number"], item["zf_link"], item["price"], item["change"], datetime.datetime.now())
)
log.msg("Item stored in db: %s" % item, level=log.DEBUG)
def _handle_error(self, e):
log.err(e)
def _process_plot_price(self, item):
query = self.dbpool.runInteraction(self._conditional_insert_price, item)
query.addErrback(self._handle_error, item)
def _conditional_insert_price(self, conn, item):
# create record if doesn't exist.
# all this block run on it's own thread
conn.execute("select * from houseplot_monthly_price where title = %s and price_date= %s", (item["title"], item["price_date"]))
result = conn.fetchone()
if result:
log.msg("Price Item already stored in db: %s" % item, level=log.DEBUG)
else:
conn.execute(
"insert into houseplot_monthly_price(title, price_date, price) values (%s, %s, %s)", (item["title"], item["price_date"], item["price"])
)
log.msg("Price Item stored in db: %s" % item, level=log.DEBUG)