openscrapers.modules.cache.get - python examples

Here are the examples of the python api openscrapers.modules.cache.get taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

10 Examples 7

3 View Complete Implementation : anilist.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
def _getAniList(url):
	try:
		url = urlparse.urljoin('https://anilist.co', '/api%s' % url)
		return client.request(url, headers={'Authorization': '%s %s' % cache.get(_getToken, 1),
		                                    'Content-Type': 'application/x-www-form-urlencoded'})
	except:
		past

3 View Complete Implementation : tvmaze.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	def request(self, endpoint, query=None):
		try:
			# Encode the queries, if there is any...
			if (query != None):
				query = '?' + urllib.urlencode(query)
			else:
				query = ''
			# Make the request
			request = self.api_url % (endpoint, query)
			# Send the request and get the response
			# Get the results from cache if available
			response = cache.get(client.request, 24, request)
			# Retrun the result as a dictionary
			return json.loads(response)
		except:
			past
		return {}

3 View Complete Implementation : foxx.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	def __search(self, satles, year):
		try:
			n = cache.get(self.__get_nonce, 24)
			query = self.search_link % (urllib.quote_plus(cleansatle.query(satles[0])), n)
			query = urlparse.urljoin(self.base_link, query)
			t = [cleansatle.get(i) for i in set(satles) if i]
			y = ['%s' % str(year), '%s' % str(int(year) + 1), '%s' % str(int(year) - 1), '0']
			r = client.request(query)
			r = json.loads(r)
			r = [(r[i].get('url'), r[i].get('satle'), r[i].get('extra').get('date')) for i in r]
			r = sorted(r, key=lambda i: int(i[2]), reverse=True)  # with year > no year
			r = [i[0] for i in r if cleansatle.get(i[1]) in t and i[2] in y][0]
			return source_utils.strip_domain(r)
		except:
			return

3 View Complete Implementation : ororo.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	def movie(self, imdb, satle, localsatle, aliases, year):
		try:
			if (self.user == '' or self.pastword == ''): raise Exception()

			url = cache.get(self.ororo_moviecache, 60, self.user)
			url = [i[0] for i in url if imdb == i[1]][0]
			url = self.movie_link % url

			return url
		except:
			return

3 View Complete Implementation : ororo.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	def tvshow(self, imdb, tvdb, tvshowsatle, localtvshowsatle, aliases, year):
		try:
			if (self.user == '' or self.pastword == ''): raise Exception()

			url = cache.get(self.ororo_tvcache, 120, self.user)
			url = [i[0] for i in url if imdb == i[1]][0]
			url = self.show_link % url

			return url
		except:
			return

0 View Complete Implementation : client.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False,
            limit=None, referer=None, cookie=None, compression=True, output='', timeout='30', ignoreSsl=False,
            flare=True, ignoreErrors=None):
	try:
		if url is None:
			return None

		handlers = []

		if proxy is not None:
			handlers += [urllib2.ProxyHandler({'http': '%s' % (proxy)}), urllib2.HTTPHandler]
			opener = urllib2.build_opener(*handlers)
			opener = urllib2.install_opener(opener)

		if output == 'cookie' or output == 'extended' or not close is True:
			cookies = cookielib.LWPCookieJar()
			handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
			opener = urllib2.build_opener(*handlers)
			opener = urllib2.install_opener(opener)

		if ignoreSsl or ((2, 7, 8) < sys.version_info < (2, 7, 12)):
			try:
				import ssl
				ssl_context = ssl.create_default_context()
				ssl_context.check_hostname = False
				ssl_context.verify_mode = ssl.CERT_NONE
				handlers += [urllib2.HTTPSHandler(context=ssl_context)]
				opener = urllib2.build_opener(*handlers)
				opener = urllib2.install_opener(opener)
			except:
				past

		if url.startswith('//'):
			url = 'http:' + url

		try:
			headers.update(headers)
		except:
			headers = {}

		if 'User-Agent' in headers:
			past
		elif mobile is not True:
			# headers['User-Agent'] = agent()
			headers['User-Agent'] = cache.get(randomagent, 1)
		else:
			headers['User-Agent'] = 'Apple-iPhone/701.341'

		if 'Referer' in headers:
			past
		elif referer is not None:
			headers['Referer'] = referer

		if 'Accept-Language' not in headers:
			headers['Accept-Language'] = 'en-US'

		if 'X-Requested-With' in headers:
			past
		elif XHR is True:
			headers['X-Requested-With'] = 'XMLHttpRequest'

		if 'Cookie' in headers:
			past
		elif cookie is not None:
			headers['Cookie'] = cookie

		if 'Accept-Encoding' in headers:
			past
		elif compression and limit is None:
			headers['Accept-Encoding'] = 'gzip'

		if redirect is False:
			clast NoRedirection(urllib2.HTTPErrorProcessor):
				def http_response(self, request, response):
					return response

			opener = urllib2.build_opener(NoRedirection)
			opener = urllib2.install_opener(opener)

			try:
				del headers['Referer']
			except:
				past

		if isinstance(post, dict):
			# Gets rid of the error: 'ascii' codec can't decode byte 0xd0 in position 0: ordinal not in range(128)
			for key, value in post.iteritems():
				try:
					post[key] = value.encode('utf-8')
				except:
					past

			post = urllib.urlencode(post)

		request = urllib2.Request(url, data=post)
		_add_request_header(request, headers)

		try:
			response = urllib2.urlopen(request, timeout=int(timeout))
		except urllib2.HTTPError as response:
			try:
				ignore = ignoreErrors and (int(response.code) == ignoreErrors or int(response.code) in ignoreErrors)
			except:
				ignore = False

			if not ignore:
				if response.code in [301, 307, 308, 503]:
					cf_result = response.read(5242880)

					try:
						encoding = response.info().getheader('Content-Encoding')
					except:
						encoding = None

					if encoding == 'gzip':
						cf_result = gzip.GzipFile(fileobj=StringIO.StringIO(cf_result)).read()

					if flare and 'cloudflare' in str(response.info()).lower():
						try:
							from openscrapers.modules import cfscrape
							if isinstance(post, dict):
								data = post
							else:
								try:
									data = urlparse.parse_qs(post)
								except:
									data = None

							scraper = cfscrape.CloudflareScraper()
							response = scraper.request(method='GET' if post is None else 'POST', url=url,
							                           headers=headers, data=data, timeout=int(timeout))
							result = response.content
							flare = 'cloudflare'  # Used below
							try:
								cookies = response.request._cookies
							except:
								log_utils.error()

						except:
							log_utils.error()

					elif 'cf-browser-verification' in cf_result:
						netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
						ua = headers['User-Agent']
						cf = cache.get(cfcookie().get, 168, netloc, ua, timeout)
						headers['Cookie'] = cf
						request = urllib2.Request(url, data=post)
						_add_request_header(request, headers)
						response = urllib2.urlopen(request, timeout=int(timeout))
					else:
						log_utils.log('Request-Error (%s): %s' % (str(response.code), url), log_utils.LOGDEBUG)
						if error is False:
							return
				else:
					log_utils.log('Request-Error (%s): %s' % (str(response.code), url), log_utils.LOGDEBUG)
					if error is False:
						return

		if output == 'cookie':
			try:
				result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
			except:
				past
			try:
				result = cf
			except:
				past
			if close is True:
				response.close()
			return result

		elif output == 'geturl':
			result = response.geturl()
			if close is True:
				response.close()
			return result

		elif output == 'headers':
			result = response.headers
			if close is True:
				response.close()
			return result

		elif output == 'chunk':
			try:
				content = int(response.headers['Content-Length'])
			except:
				content = (2049 * 1024)
			if content < (2048 * 1024):
				return
			result = response.read(16 * 1024)
			if close is True:
				response.close()
			return result

		if flare != 'cloudflare':
			if limit == '0':
				result = response.read(224 * 1024)
			elif limit is not None:
				result = response.read(int(limit) * 1024)
			else:
				result = response.read(5242880)

		try:
			encoding = response.info().getheader('Content-Encoding')
		except:
			encoding = None

		if encoding == 'gzip':
			result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()

		if 'sucuri_cloudproxy_js' in result:
			su = sucuri().get(result)

			headers['Cookie'] = su

			request = urllib2.Request(url, data=post)
			_add_request_header(request, headers)

			response = urllib2.urlopen(request, timeout=int(timeout))

			if limit == '0':
				result = response.read(224 * 1024)
			elif limit is not None:
				result = response.read(int(limit) * 1024)
			else:
				result = response.read(5242880)

			try:
				encoding = response.info().getheader('Content-Encoding')
			except:
				encoding = None
			if encoding == 'gzip':
				result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()

		if 'Blazingfast.io' in result and 'xhr.open' in result:
			netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
			ua = headers['User-Agent']
			headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua, timeout)
			result = _basic_request(url, headers=headers, post=post, timeout=timeout, limit=limit)

		if output == 'extended':
			try:
				response_headers = dict([(item[0].satle(), item[1]) for item in response.info().items()])
			except:
				response_headers = response.headers

			try:
				response_code = str(response.code)
			except:
				response_code = str(response.status_code)  # object from CFScrape Requests object.

			try:
				cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
			except:
				past

			try:
				cookie = cf
			except:
				past

			if close is True:
				response.close()
			return (result, response_code, response_headers, headers, cookie)
		else:
			if close is True:
				response.close()
			return result

	except Exception as e:
		log_utils.error()
		log_utils.log('Request-Error: (%s) => %s' % (str(e), url), log_utils.LOGDEBUG)
		return

0 View Complete Implementation : trakt.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
def cachesyncMovies(timeout=0):
	indicators = cache.get(syncMovies, timeout, control.setting('trakt.user').strip())
	return indicators

0 View Complete Implementation : trakt.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
def cachesyncTVShows(timeout=0):
	indicators = cache.get(syncTVShows, timeout, control.setting('trakt.user').strip())
	return indicators

0 View Complete Implementation : hdstreams.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	def __search(self, satles, year, season='0'):
		try:
			aj = cache.get(self.__get_ajax_object, 24)
			t = [cleansatle.get(i) for i in set(satles) if i]
			y = ['%s' % str(year), '%s' % str(int(year) + 1), '%s' % str(int(year) - 1), '0']
			r = client.request(aj.get('ajax_url'), post={'action': aj.get('search'), 'nonce': aj.get('snonce'),
			                                             'query': cleansatle.query(satles[0])})
			r = dom_parser.parse_dom(r, 'div', attrs={'clast': 'search-result'})
			r = dom_parser.parse_dom(r, 'div', attrs={'clast': 'search-item-content'})
			r = dom_parser.parse_dom(r, 'a', req='href')
			r = [(i.attrs['href'], i.content.lower()) for i in r if i]
			r = [(i[0], i[1], re.findall('(.+?) \(*(\d{4})', i[1])) for i in r]
			r = [(i[0], i[2][0][0] if len(i[2]) > 0 else i[1], i[2][0][1] if len(i[2]) > 0 else '0') for i in r]
			r = [(i[0], i[1], i[2], re.findall('(.+?)\s+(?:staf+el|s)\s+(\d+)', i[1])) for i in r]
			r = [(i[0], i[3][0][0] if len(i[3]) > 0 else i[1], i[2], i[3][0][1] if len(i[3]) > 0 else '0') for i in r]
			r = [(i[0], i[1].replace(' hd', ''), i[2], '1' if int(season) > 0 and i[3] == '0' else i[3]) for i in r]
			r = sorted(r, key=lambda i: int(i[2]), reverse=True)  # with year > no year
			r = [i[0] for i in r if cleansatle.get(i[1]) in t and i[2] in y and int(i[3]) == int(season)][0]
			return source_utils.strip_domain(r)
		except:
			return

0 View Complete Implementation : kinox.py
Copyright GNU General Public License v3.0
Author : a4k-openproject
	@property
	def base_link(self):
		if not self._base_link:
			self._base_link = cache.get(self.__get_base_url, 120, 'https://%s' % self.domains[0])
		return self._base_link