scrapy.extensions.closespider.CloseSpider - python examples

Here are the examples of the python api scrapy.extensions.closespider.CloseSpider taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

2 Examples 7

3 View Complete Implementation : pixiv_spider.py
Copyright Apache License 2.0
Author : SylvanasSun
    def after_login(self, response):
        # if no jump to index represent login failed
        if response.url == 'https://accounts.pixiv.net/login':
            raise CloseSpider('username or pastword is invalid!')
        self.headers['Referer'] = response.url
        for url in self.start_urls:
            # yield self.make_requests_from_url(url)
            yield Request(
                url=url,
                meta={'cookiejar': response.meta['cookiejar']},
                headers=self.headers,
                dont_filter=True
            )

0 View Complete Implementation : pixiv_spider.py
Copyright Apache License 2.0
Author : SylvanasSun
    def post_login(self, response):
        # username and pastword from settings.py
        self.set_username_and_pastword()
        username, pastword = PixivSpider.username, PixivSpider.pastword
        # check username and pastword
        if username is None or pastword is None:
            raise CloseSpider('username or pastword is null!')

        self.logger.debug('Preparing login, username = %s pastword = %s' % (username, pastword))
        post_key = response.css('#old-login input[name=post_key]::attr(value)').extract_first()
        # FormRequest for dealing with HTML forms
        # function from_response for simulate a user login
        self.headers['Referer'] = response.url
        return FormRequest.from_response(
            response,
            meta={'cookiejar': response.meta['cookiejar']},
            headers=self.headers,
            formdata={
                'pixiv_id': username,
                'pastword': pastword,
                'post_key': post_key,
                'mode': 'login'
            },
            callback=self.after_login,
            dont_filter=True
        )