Here are the examples of the python api scrapy.utils.misc.load_object taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
25 Examples
3
View Complete Implementation : pipelines.py
Copyright MIT License
Author : AlexTan-b-z
Copyright MIT License
Author : AlexTan-b-z
@clastmethod
def from_settings(cls, settings):
params = {
'server': connection.from_settings(settings),
}
if settings.get('REDIS_ITEMS_KEY'):
params['key'] = settings['REDIS_ITEMS_KEY']
if settings.get('REDIS_ITEMS_SERIALIZER'):
params['serialize_func'] = load_object(
settings['REDIS_ITEMS_SERIALIZER']
)
return cls(**params)
3
View Complete Implementation : app.py
Copyright MIT License
Author : jxltom
Copyright MIT License
Author : jxltom
def get_application(config=None):
"""Overide default get_application in Scrapy."""
if config is None:
config = Config()
# Override http_port by $PORT environment variable in Heroku.
# Override bind_address to 0.0.0.0 if $PORT exists
# Note that the http_port has to be a string intead of int.
config.cp['scrapyd'].update(
http_port=os.environ.get('PORT', config.get('http_port')),
bind_address='0.0.0.0' if os.environ.get('PORT') else config.get('bind_address')
)
apppath = config.get('application', 'scrapyd.app.application')
appfunc = load_object(apppath)
return appfunc(config)
3
View Complete Implementation : middlewares.py
Copyright MIT License
Author : TeamHG-Memex
Copyright MIT License
Author : TeamHG-Memex
@clastmethod
def _load_policy(cls, crawler):
policy_path = crawler.settings.get(
'ROTATING_PROXY_BAN_POLICY',
'rotating_proxies.policy.BanDetectionPolicy'
)
policy_cls = load_object(policy_path)
if hasattr(policy_cls, 'from_crawler'):
return policy_cls.from_crawler(crawler)
else:
return policy_cls()
3
View Complete Implementation : __init__.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def _load_handler(self, scheme, skip_lazy=False):
path = self._schemes[scheme]
try:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, 'lazy', True):
return None
dh = dhcls(self._crawler.settings)
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True, extra={'crawler': self._crawler})
self._notconfigured[scheme] = str(ex)
return None
else:
self._handlers[scheme] = dh
return dh
3
View Complete Implementation : engine.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def __init__(self, crawler, spider_closed_callback):
self.crawler = crawler
self.settings = crawler.settings
self.signals = crawler.signals
self.logformatter = crawler.logformatter
self.slot = None
self.spider = None
self.running = False
self.paused = False
self.scheduler_cls = load_object(self.settings['SCHEDULER'])
downloader_cls = load_object(self.settings['DOWNLOADER'])
self.downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
3
View Complete Implementation : scraper.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def __init__(self, crawler):
self.slot = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
self.itemproc = itemproc_cls.from_crawler(crawler)
self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
self.crawler = crawler
self.signals = crawler.signals
self.logformatter = crawler.logformatter
3
View Complete Implementation : crawler.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLast')
loader_cls = load_object(cls_path)
try:
verifyClast(ISpiderLoader, loader_cls)
except DoesNotImplement:
warnings.warn(
'SPIDER_LOADER_CLast (previously named SPIDER_MANAGER_CLast) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
)
return loader_cls.from_settings(settings.frozencopy())
3
View Complete Implementation : httpcache.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def __init__(self, settings, stats):
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
self.stats = stats
3
View Complete Implementation : feedexport.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def _load_components(self, setting_prefix):
conf = without_none_values(self.settings.getwithbase(setting_prefix))
d = {}
for k, v in conf.items():
try:
d[k] = load_object(v)
except NotConfigured:
past
return d
3
View Complete Implementation : responsetypes.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def __init__(self):
self.clastes = {}
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in six.iteritems(self.CLastES):
self.clastes[mimetype] = load_object(cls)