You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
63 lines
1.9 KiB
Python
63 lines
1.9 KiB
Python
2 years ago
|
import requests,time,json
|
||
|
ideal_delta = 60*5 #5 minutes
|
||
|
|
||
|
crawler_header = {'User-agent': 'interverse-crawler','info':'https://libresolutions.network/videos/interverse-demo-1/'}
|
||
|
schemes = ['http://','https://']
|
||
|
locations = [
|
||
|
'/.well-known/discover.json',
|
||
|
'/.well-known/interverse',
|
||
|
'/interverse.json',
|
||
|
'/discover.json'
|
||
|
]
|
||
|
|
||
|
|
||
|
|
||
|
class Cache:
|
||
|
def __init__(self,delta=None):
|
||
|
if delta==None:
|
||
|
self.delta = ideal_delta
|
||
|
else:
|
||
|
self.delta = delta
|
||
|
self.links={}
|
||
|
# link = key:{data,time}
|
||
|
|
||
|
|
||
|
def load_data(self,url):
|
||
|
data = None
|
||
|
t = time.time()
|
||
|
if url in self.links:
|
||
|
if t - self.links[url]['time'] <= self.delta:
|
||
|
print(f"Using cached result for {url}")
|
||
|
return self.links[url]['data']
|
||
|
for s in schemes:
|
||
|
for l in locations:
|
||
|
try:
|
||
|
data = requests.get(s+url+l,headers=crawler_header,timeout=3).json()
|
||
|
if l.find('discover'):
|
||
|
#translate discover to interverse
|
||
|
data = json.loads(json.dumps(data).replace("preview_connections","connection_groups"))
|
||
|
print(f"Interverse connection found at {l}")
|
||
|
t = time.time()
|
||
|
self.links[url] = {
|
||
|
'time':t,
|
||
|
'data':data,
|
||
|
}
|
||
|
|
||
|
return data
|
||
|
except:
|
||
|
pass
|
||
|
if data != None:
|
||
|
t = time.time()
|
||
|
self.links[url] = {
|
||
|
'time':t,
|
||
|
'data':data,
|
||
|
}
|
||
|
|
||
|
if data == None:
|
||
|
#If no data is returned, wait longer before attempting again
|
||
|
self.links[url] = {
|
||
|
'data':None,
|
||
|
'time':t+ideal_delta
|
||
|
}
|
||
|
return data
|