
class FooSpider(baseSpider): def __init__(self, *args, **kwargs): super(FooSpider, self).__init__(*args, **kwargs) dispatcher.connect(self.dont_close_me, signals.spider_idle) def dont_close_me(self, spider): if spider != self: return self.crawler.engine.crawl(self.create_request(), spider) raise DontCloseSpider("..I prefer live spiders.")2016年更新:
class FooSpider(baseSpider): yet = False @classmethod def from_crawler(cls, crawler, *args, **kwargs): from_crawler = super(FooSpider, cls).from_crawler spider = from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.idle, signal=scrapy.signals.spider_idle) return spider def idle(self): if not self.yet: self.crawler.engine.crawl(self.create_request(), self) self.yet = True
欢迎分享,转载请注明来源:内存溢出
微信扫一扫
支付宝扫一扫
评论列表(0条)