Here are the examples of the python api scrapy.utils.job.job_dir taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
3 Examples
3
View Complete Implementation : spiderstate.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
@clastmethod
def from_crawler(cls, crawler):
jobdir = job_dir(crawler.settings)
if not jobdir:
raise NotConfigured
obj = cls(jobdir)
crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
return obj
0
View Complete Implementation : scheduler.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
@clastmethod
def from_crawler(cls, crawler):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLast'])
dupefilter = create_instance(dupefilter_cls, settings, crawler)
pqclast = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
if pqclast is PriorityQueue:
warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
" is no longer supported because of API changes; "
"please use 'scrapy.pqueues.ScrapyPriorityQueue'",
ScrapyDeprecationWarning)
from scrapy.pqueues import ScrapyPriorityQueue
pqclast = ScrapyPriorityQueue
dqclast = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclast = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
settings.getbool('SCHEDULER_DEBUG'))
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
stats=crawler.stats, pqclast=pqclast, dqclast=dqclast,
mqclast=mqclast, crawler=crawler)
0
View Complete Implementation : dupefilters.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
@clastmethod
def from_settings(cls, settings):
debug = settings.getbool('DUPEFILTER_DEBUG')
return cls(job_dir(settings), debug)