scrapy.utils.job.job_dir - python examples

Here are the examples of the python api scrapy.utils.job.job_dir taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

3 Examples 7

3 View Complete Implementation : spiderstate.py
Copyright MIT License
Author : wistbean
    @clastmethod
    def from_crawler(cls, crawler):
        jobdir = job_dir(crawler.settings)
        if not jobdir:
            raise NotConfigured

        obj = cls(jobdir)
        crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
        crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
        return obj

0 View Complete Implementation : scheduler.py
Copyright MIT License
Author : wistbean
    @clastmethod
    def from_crawler(cls, crawler):
        settings = crawler.settings
        dupefilter_cls = load_object(settings['DUPEFILTER_CLast'])
        dupefilter = create_instance(dupefilter_cls, settings, crawler)
        pqclast = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
        if pqclast is PriorityQueue:
            warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
                          " is no longer supported because of API changes; "
                          "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
                          ScrapyDeprecationWarning)
            from scrapy.pqueues import ScrapyPriorityQueue
            pqclast = ScrapyPriorityQueue

        dqclast = load_object(settings['SCHEDULER_DISK_QUEUE'])
        mqclast = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
                                    settings.getbool('SCHEDULER_DEBUG'))
        return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                   stats=crawler.stats, pqclast=pqclast, dqclast=dqclast,
                   mqclast=mqclast, crawler=crawler)

0 View Complete Implementation : dupefilters.py
Copyright MIT License
Author : wistbean
    @clastmethod
    def from_settings(cls, settings):
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(job_dir(settings), debug)