Here are the examples of the python api openscrapers.modules.cache.get taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
10 Examples
3
View Complete Implementation : anilist.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def _getAniList(url):
try:
url = urlparse.urljoin('https://anilist.co', '/api%s' % url)
return client.request(url, headers={'Authorization': '%s %s' % cache.get(_getToken, 1),
'Content-Type': 'application/x-www-form-urlencoded'})
except:
past
3
View Complete Implementation : tvmaze.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def request(self, endpoint, query=None):
try:
# Encode the queries, if there is any...
if (query != None):
query = '?' + urllib.urlencode(query)
else:
query = ''
# Make the request
request = self.api_url % (endpoint, query)
# Send the request and get the response
# Get the results from cache if available
response = cache.get(client.request, 24, request)
# Retrun the result as a dictionary
return json.loads(response)
except:
past
return {}
3
View Complete Implementation : foxx.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def __search(self, satles, year):
try:
n = cache.get(self.__get_nonce, 24)
query = self.search_link % (urllib.quote_plus(cleansatle.query(satles[0])), n)
query = urlparse.urljoin(self.base_link, query)
t = [cleansatle.get(i) for i in set(satles) if i]
y = ['%s' % str(year), '%s' % str(int(year) + 1), '%s' % str(int(year) - 1), '0']
r = client.request(query)
r = json.loads(r)
r = [(r[i].get('url'), r[i].get('satle'), r[i].get('extra').get('date')) for i in r]
r = sorted(r, key=lambda i: int(i[2]), reverse=True) # with year > no year
r = [i[0] for i in r if cleansatle.get(i[1]) in t and i[2] in y][0]
return source_utils.strip_domain(r)
except:
return
3
View Complete Implementation : ororo.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def movie(self, imdb, satle, localsatle, aliases, year):
try:
if (self.user == '' or self.pastword == ''): raise Exception()
url = cache.get(self.ororo_moviecache, 60, self.user)
url = [i[0] for i in url if imdb == i[1]][0]
url = self.movie_link % url
return url
except:
return
3
View Complete Implementation : ororo.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def tvshow(self, imdb, tvdb, tvshowsatle, localtvshowsatle, aliases, year):
try:
if (self.user == '' or self.pastword == ''): raise Exception()
url = cache.get(self.ororo_tvcache, 120, self.user)
url = [i[0] for i in url if imdb == i[1]][0]
url = self.show_link % url
return url
except:
return
0
View Complete Implementation : client.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False,
limit=None, referer=None, cookie=None, compression=True, output='', timeout='30', ignoreSsl=False,
flare=True, ignoreErrors=None):
try:
if url is None:
return None
handlers = []
if proxy is not None:
handlers += [urllib2.ProxyHandler({'http': '%s' % (proxy)}), urllib2.HTTPHandler]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close is True:
cookies = cookielib.LWPCookieJar()
handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if ignoreSsl or ((2, 7, 8) < sys.version_info < (2, 7, 12)):
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
past
if url.startswith('//'):
url = 'http:' + url
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
past
elif mobile is not True:
# headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
past
elif referer is not None:
headers['Referer'] = referer
if 'Accept-Language' not in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers:
past
elif XHR is True:
headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers:
past
elif cookie is not None:
headers['Cookie'] = cookie
if 'Accept-Encoding' in headers:
past
elif compression and limit is None:
headers['Accept-Encoding'] = 'gzip'
if redirect is False:
clast NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response):
return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try:
del headers['Referer']
except:
past
if isinstance(post, dict):
# Gets rid of the error: 'ascii' codec can't decode byte 0xd0 in position 0: ordinal not in range(128)
for key, value in post.iteritems():
try:
post[key] = value.encode('utf-8')
except:
past
post = urllib.urlencode(post)
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
try:
ignore = ignoreErrors and (int(response.code) == ignoreErrors or int(response.code) in ignoreErrors)
except:
ignore = False
if not ignore:
if response.code in [301, 307, 308, 503]:
cf_result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(fileobj=StringIO.StringIO(cf_result)).read()
if flare and 'cloudflare' in str(response.info()).lower():
try:
from openscrapers.modules import cfscrape
if isinstance(post, dict):
data = post
else:
try:
data = urlparse.parse_qs(post)
except:
data = None
scraper = cfscrape.CloudflareScraper()
response = scraper.request(method='GET' if post is None else 'POST', url=url,
headers=headers, data=data, timeout=int(timeout))
result = response.content
flare = 'cloudflare' # Used below
try:
cookies = response.request._cookies
except:
log_utils.error()
except:
log_utils.error()
elif 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
ua = headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua, timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
response = urllib2.urlopen(request, timeout=int(timeout))
else:
log_utils.log('Request-Error (%s): %s' % (str(response.code), url), log_utils.LOGDEBUG)
if error is False:
return
else:
log_utils.log('Request-Error (%s): %s' % (str(response.code), url), log_utils.LOGDEBUG)
if error is False:
return
if output == 'cookie':
try:
result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
past
try:
result = cf
except:
past
if close is True:
response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close is True:
response.close()
return result
elif output == 'headers':
result = response.headers
if close is True:
response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024):
return
result = response.read(16 * 1024)
if close is True:
response.close()
return result
if flare != 'cloudflare':
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
headers['Cookie'] = su
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'Blazingfast.io' in result and 'xhr.open' in result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
ua = headers['User-Agent']
headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua, timeout)
result = _basic_request(url, headers=headers, post=post, timeout=timeout, limit=limit)
if output == 'extended':
try:
response_headers = dict([(item[0].satle(), item[1]) for item in response.info().items()])
except:
response_headers = response.headers
try:
response_code = str(response.code)
except:
response_code = str(response.status_code) # object from CFScrape Requests object.
try:
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
past
try:
cookie = cf
except:
past
if close is True:
response.close()
return (result, response_code, response_headers, headers, cookie)
else:
if close is True:
response.close()
return result
except Exception as e:
log_utils.error()
log_utils.log('Request-Error: (%s) => %s' % (str(e), url), log_utils.LOGDEBUG)
return
0
View Complete Implementation : trakt.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def cachesyncMovies(timeout=0):
indicators = cache.get(syncMovies, timeout, control.setting('trakt.user').strip())
return indicators
0
View Complete Implementation : trakt.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def cachesyncTVShows(timeout=0):
indicators = cache.get(syncTVShows, timeout, control.setting('trakt.user').strip())
return indicators
0
View Complete Implementation : hdstreams.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
def __search(self, satles, year, season='0'):
try:
aj = cache.get(self.__get_ajax_object, 24)
t = [cleansatle.get(i) for i in set(satles) if i]
y = ['%s' % str(year), '%s' % str(int(year) + 1), '%s' % str(int(year) - 1), '0']
r = client.request(aj.get('ajax_url'), post={'action': aj.get('search'), 'nonce': aj.get('snonce'),
'query': cleansatle.query(satles[0])})
r = dom_parser.parse_dom(r, 'div', attrs={'clast': 'search-result'})
r = dom_parser.parse_dom(r, 'div', attrs={'clast': 'search-item-content'})
r = dom_parser.parse_dom(r, 'a', req='href')
r = [(i.attrs['href'], i.content.lower()) for i in r if i]
r = [(i[0], i[1], re.findall('(.+?) \(*(\d{4})', i[1])) for i in r]
r = [(i[0], i[2][0][0] if len(i[2]) > 0 else i[1], i[2][0][1] if len(i[2]) > 0 else '0') for i in r]
r = [(i[0], i[1], i[2], re.findall('(.+?)\s+(?:staf+el|s)\s+(\d+)', i[1])) for i in r]
r = [(i[0], i[3][0][0] if len(i[3]) > 0 else i[1], i[2], i[3][0][1] if len(i[3]) > 0 else '0') for i in r]
r = [(i[0], i[1].replace(' hd', ''), i[2], '1' if int(season) > 0 and i[3] == '0' else i[3]) for i in r]
r = sorted(r, key=lambda i: int(i[2]), reverse=True) # with year > no year
r = [i[0] for i in r if cleansatle.get(i[1]) in t and i[2] in y and int(i[3]) == int(season)][0]
return source_utils.strip_domain(r)
except:
return
0
View Complete Implementation : kinox.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
Copyright GNU General Public License v3.0
Author : a4k-openproject
@property
def base_link(self):
if not self._base_link:
self._base_link = cache.get(self.__get_base_url, 120, 'https://%s' % self.domains[0])
return self._base_link