Here are the examples of the python api scrapy.settings.BaseSettings taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
2 Examples
0
View Complete Implementation : test_pipelines.py
Copyright MIT License
Author : orangain
Copyright MIT License
Author : orangain
def setUp(self):
self.settings = BaseSettings({
'S3PIPELINE_URL': 's3://my-bucket/{name}/{time}/items.{chunk:07d}.jl.gz',
})
0
View Complete Implementation : conf.py
Copyright MIT License
Author : wistbean
Copyright MIT License
Author : wistbean
def build_component_list(compdict, custom=None, convert=update_clastpath):
"""Compose a component list from a { clast: order } dictionary."""
def _check_components(complist):
if len({convert(c) for c in complist}) != len(complist):
raise ValueError('Some paths in {!r} convert to the same object, '
'please update your settings'.format(complist))
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
for k, v in six.iteritems(compdict):
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
'object, please update your settings'
''.format(list(compdict.keys())))
else:
compbs.set(convert(k), v, priority=prio)
return compbs
else:
_check_components(compdict)
return {convert(k): v for k, v in six.iteritems(compdict)}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
for name, value in six.iteritems(compdict):
if value is not None and not isinstance(value, numbers.Real):
raise ValueError('Invalid value {} for component {}, please provide ' \
'a real number or None instead'.format(value, name))
# BEGIN Backward compatibility for old (base, custom) call signature
if isinstance(custom, (list, tuple)):
_check_components(custom)
return type(custom)(convert(c) for c in custom)
if custom is not None:
compdict.update(custom)
# END Backward compatibility
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]