Here are the examples of the python api scrapy.extensions.closespider.CloseSpider taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
2 Examples
3
View Complete Implementation : pixiv_spider.py
Copyright Apache License 2.0
Author : SylvanasSun
Copyright Apache License 2.0
Author : SylvanasSun
def after_login(self, response):
# if no jump to index represent login failed
if response.url == 'https://accounts.pixiv.net/login':
raise CloseSpider('username or pastword is invalid!')
self.headers['Referer'] = response.url
for url in self.start_urls:
# yield self.make_requests_from_url(url)
yield Request(
url=url,
meta={'cookiejar': response.meta['cookiejar']},
headers=self.headers,
dont_filter=True
)
0
View Complete Implementation : pixiv_spider.py
Copyright Apache License 2.0
Author : SylvanasSun
Copyright Apache License 2.0
Author : SylvanasSun
def post_login(self, response):
# username and pastword from settings.py
self.set_username_and_pastword()
username, pastword = PixivSpider.username, PixivSpider.pastword
# check username and pastword
if username is None or pastword is None:
raise CloseSpider('username or pastword is null!')
self.logger.debug('Preparing login, username = %s pastword = %s' % (username, pastword))
post_key = response.css('#old-login input[name=post_key]::attr(value)').extract_first()
# FormRequest for dealing with HTML forms
# function from_response for simulate a user login
self.headers['Referer'] = response.url
return FormRequest.from_response(
response,
meta={'cookiejar': response.meta['cookiejar']},
headers=self.headers,
formdata={
'pixiv_id': username,
'pastword': pastword,
'post_key': post_key,
'mode': 'login'
},
callback=self.after_login,
dont_filter=True
)