Add default values
This commit is contained in:
parent
7bb0a5ec5a
commit
c1c431cf12
1 changed files with 107 additions and 72 deletions
|
|
@ -45,7 +45,9 @@ class DiscoverMastodonServers:
|
|||
self.servers[self.config['initial_server']] = {
|
||||
"name": self.config['initial_server'],
|
||||
"last_update": time.time(),
|
||||
"private": False
|
||||
"private": False,
|
||||
"peers": True,
|
||||
"timeline": True,
|
||||
}
|
||||
new_servers_count = 1
|
||||
while new_servers_count > 0:
|
||||
|
|
@ -60,9 +62,11 @@ class DiscoverMastodonServers:
|
|||
data = None
|
||||
try:
|
||||
result = self.session.get(
|
||||
f"https://{server}/{endpoint}",
|
||||
f"https://{server['name']}/{endpoint}",
|
||||
timeout=10
|
||||
)
|
||||
server['status'] = result.status_code
|
||||
server['state'] = 'OK'
|
||||
if result.status_code < 400:
|
||||
if 'Content-Type' in result.headers:
|
||||
if 'application/json' in result.headers['Content-Type']:
|
||||
|
|
@ -70,68 +74,78 @@ class DiscoverMastodonServers:
|
|||
if 'error' not in data:
|
||||
return data
|
||||
else:
|
||||
server['state'] = 'Error'
|
||||
self._log.debug(
|
||||
"Server '%s' didn't reply with JSON data.", server
|
||||
"Server '%s' didn't reply with JSON data.", server['name']
|
||||
)
|
||||
else:
|
||||
server['state'] = 'Error'
|
||||
self._log.debug(
|
||||
"Server '%s' didn't return Content-Type header. Headers: '%s'. Content returned: '%s'",
|
||||
server,
|
||||
server['name'],
|
||||
json.dumps(result.headers, indent=2),
|
||||
result.content
|
||||
)
|
||||
else:
|
||||
server['state'] = 'Error'
|
||||
self._log.debug(
|
||||
"Server '%s' returned error code %s.", server, result.status_code
|
||||
"Server '%s' returned error code %s.", server['name'], result.status_code
|
||||
)
|
||||
except requests.exceptions.ReadTimeout as error:
|
||||
server['state'] = 'Error'
|
||||
self._log.warning(
|
||||
"Server '%s' didn't respond on time. %s",
|
||||
server,
|
||||
server['name'],
|
||||
error
|
||||
)
|
||||
except requests.exceptions.SSLError as error:
|
||||
server['state'] = 'SSL Error'
|
||||
self._log.warning(
|
||||
"Server '%s' don't have a valid SSL certificate. %s",
|
||||
server,
|
||||
server['name'],
|
||||
error
|
||||
)
|
||||
except requests.exceptions.ConnectionError as error:
|
||||
server['state'] = 'Error'
|
||||
self._log.warning(
|
||||
"Server '%s' connection failed. %s",
|
||||
server,
|
||||
server['name'],
|
||||
error
|
||||
)
|
||||
except requests.exceptions.TooManyRedirects as error:
|
||||
server['state'] = 'Error'
|
||||
self._log.warning(
|
||||
"Server '%s' redirected too many times. %s",
|
||||
server['name'],
|
||||
error
|
||||
)
|
||||
return data
|
||||
|
||||
def get_instance_info(self, server):
|
||||
'''Get all server information'''
|
||||
result = {}
|
||||
instance = self.get_path(server, '/api/v1/instance')
|
||||
instance = self.get_path(server['name'], '/api/v1/instance')
|
||||
if instance:
|
||||
result['instance'] = instance
|
||||
directory = []
|
||||
result['directory'] = directory
|
||||
server['instance'] = instance
|
||||
server['directory'] = []
|
||||
offset=0
|
||||
while len(directory) == 0:
|
||||
while len(server['directory']) == 0:
|
||||
directory = self.get_path(
|
||||
server,
|
||||
f"/api/v1/directory?limit=80&offset={offset}"
|
||||
)
|
||||
if directory:
|
||||
result['directory'] = result['directory'] + directory
|
||||
server['directory'] = server['directory'] + directory
|
||||
offset += 80
|
||||
return result
|
||||
|
||||
def test_banned_server(self, server):
|
||||
def test_banned_server(self, server_name):
|
||||
'''Check if a server name match agains any banned regular expressions'''
|
||||
for banned in self.config['regexp_banned_host']:
|
||||
match = re.search(banned, server)
|
||||
match = re.search(banned, server_name)
|
||||
if match:
|
||||
self._log.debug(
|
||||
"Regexp '%s' match server '%s', banned.",
|
||||
banned,
|
||||
server
|
||||
server_name
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
|
@ -140,12 +154,20 @@ class DiscoverMastodonServers:
|
|||
'''Discover new servers'''
|
||||
all_servers = []
|
||||
new_servers_count = 0
|
||||
for server in self.servers.items():
|
||||
all_servers.append(server[0])
|
||||
if not self.test_banned_server(server[0]):
|
||||
if not server[1]['private']:
|
||||
self._log.debug("Fetching peers of the server '%s'", server[0])
|
||||
data = self.get_path(server[0], 'api/v1/instance/peers')
|
||||
for server_name, server in self.servers.items():
|
||||
all_servers.append(server_name)
|
||||
if not self.test_banned_server(server_name):
|
||||
if 'state' not in server:
|
||||
server['state'] = 'Unknown'
|
||||
if 'status' not in server:
|
||||
server['status'] = 0
|
||||
if 'peers' not in server:
|
||||
server['peers'] = True
|
||||
if 'timeline' not in server:
|
||||
server['timeline'] = True
|
||||
if not server['private'] and 'Error' not in server['state']:
|
||||
self._log.debug("Fetching peers of the server '%s'", server_name)
|
||||
data = self.get_path(server, 'api/v1/instance/peers')
|
||||
if data:
|
||||
for new_server in data:
|
||||
if ((not self.test_banned_server(new_server)) and
|
||||
|
|
@ -158,16 +180,18 @@ class DiscoverMastodonServers:
|
|||
)
|
||||
all_servers.append(new_server)
|
||||
self.write_record(
|
||||
(new_server,
|
||||
{
|
||||
"name": new_server,
|
||||
"last_update": time.time(),
|
||||
"private": False
|
||||
"private": False,
|
||||
"peers": True,
|
||||
"timeline": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
self._log.debug("Fetching public timeline in server '%s'", server[0])
|
||||
data = self.get_timeline(server[0])
|
||||
else:
|
||||
server[1]['peers'] = False
|
||||
self._log.debug("Fetching public timeline in server '%s'", server_name)
|
||||
data = self.get_timeline(server)
|
||||
if data:
|
||||
for item in data:
|
||||
if 'uri' in item:
|
||||
|
|
@ -175,7 +199,8 @@ class DiscoverMastodonServers:
|
|||
if match_server:
|
||||
new_server = match_server.group(1)
|
||||
if not self.test_banned_server(new_server) and new_server not in all_servers:
|
||||
data = self.get_timeline(new_server)
|
||||
new_server_obj = { "name": new_server }
|
||||
data = self.get_timeline(new_server_obj)
|
||||
if data:
|
||||
new_servers_count += 1
|
||||
self._log.debug(
|
||||
|
|
@ -183,18 +208,10 @@ class DiscoverMastodonServers:
|
|||
new_server
|
||||
)
|
||||
all_servers.append(new_server)
|
||||
private = False
|
||||
new_server_obj['private'] = False
|
||||
else:
|
||||
private = True
|
||||
self.write_record(
|
||||
(new_server,
|
||||
{
|
||||
"name": new_server,
|
||||
"last_update": time.time(),
|
||||
"private": private
|
||||
}
|
||||
)
|
||||
)
|
||||
new_server_obj['private'] = True
|
||||
self.write_record(new_server_obj)
|
||||
else:
|
||||
# Item in public timeline don't have an URI
|
||||
self._log.debug(
|
||||
|
|
@ -202,55 +219,73 @@ class DiscoverMastodonServers:
|
|||
item
|
||||
)
|
||||
else:
|
||||
server[1]['private'] = True
|
||||
server['timeline'] = False
|
||||
self.write_record(server)
|
||||
return new_servers_count
|
||||
|
||||
def write_record(self, record, table='servers'):
|
||||
'''Write record to a table'''
|
||||
if 'state' not in record:
|
||||
record['state'] = 'Unknown'
|
||||
if 'status' not in record:
|
||||
record['status'] = 0
|
||||
if 'peers' not in record:
|
||||
record['peers'] = True
|
||||
if 'timeline' not in record:
|
||||
record['timeline'] = True
|
||||
cur = self.conn.cursor()
|
||||
result_select = cur.execute(f"""
|
||||
SELECT name FROM {table} WHERE name = '{record[1]['name']}'
|
||||
SELECT name FROM {table} WHERE name = '{record['name']}'
|
||||
""")
|
||||
if len(result_select.fetchall()) > 0:
|
||||
self._log.debug('Record exists, updating.')
|
||||
query = f"""
|
||||
UPDATE {table}
|
||||
SET last_update = {record[1]['last_update']},
|
||||
private = {record[1]['private']}
|
||||
WHERE name = '{record[0]}'
|
||||
"""
|
||||
query = f"UPDATE {table} SET "
|
||||
count = 0
|
||||
for key in record.keys():
|
||||
if count == 0:
|
||||
query += f"{key} = :{key} "
|
||||
else:
|
||||
query += f",{key} = :{key} "
|
||||
count += 1
|
||||
query += "WHERE name = :name"
|
||||
else:
|
||||
self._log.debug('Record doesn\'t exist, inserting.')
|
||||
query = f"""
|
||||
INSERT INTO {table} (
|
||||
name,
|
||||
last_update,
|
||||
private
|
||||
)
|
||||
VALUES (
|
||||
'{record[0]}',
|
||||
{record[1]['last_update']},
|
||||
{record[1]['private']}
|
||||
)
|
||||
"""
|
||||
result_update = cur.execute(query)
|
||||
self._log.debug("Written record '%s' with ID %s",
|
||||
record[0],
|
||||
result_update.lastrowid
|
||||
query = f"INSERT INTO {table} VALUES (:" + ",:".join(record.keys()) + ")"
|
||||
self._log.debug("Writing record '%s'...",
|
||||
record
|
||||
)
|
||||
try:
|
||||
result_update = cur.execute(query, record)
|
||||
self._log.debug("Added record %s.", result_update.lastrowid)
|
||||
except Exception as error:
|
||||
self._log.error("Error running query '%s' with record '%s'. %s", query, record, error)
|
||||
sys.exit(1)
|
||||
cur.close()
|
||||
self.conn.commit()
|
||||
|
||||
def read_db(self):
|
||||
'''Read database file'''
|
||||
cur = self.conn.cursor()
|
||||
cur.execute("""CREATE TABLE IF NOT EXISTS servers(
|
||||
name TEXT PRIMARY KEY,
|
||||
last_update REAL,
|
||||
private INT
|
||||
)""")
|
||||
result_select = cur.execute("SELECT * FROM servers ORDER BY last_update DESC")
|
||||
query = """CREATE TABLE IF NOT EXISTS servers(
|
||||
name TEXT PRIMARY KEY,
|
||||
last_update REAL,
|
||||
private INT,
|
||||
peers INT,
|
||||
timeline INT,
|
||||
status INT,
|
||||
state TEXT
|
||||
)"""
|
||||
try:
|
||||
cur.execute(query)
|
||||
except Exception as error:
|
||||
self._log.error("Error running query to create table '%s'. %s", query, error)
|
||||
sys.exit(2)
|
||||
query = "SELECT * FROM servers ORDER BY last_update DESC"
|
||||
try:
|
||||
result_select = cur.execute(query)
|
||||
except Exception as error:
|
||||
self._log.error("Error running query to list servers '%s'. %s", query, error)
|
||||
sys.exit(3)
|
||||
self.servers = {}
|
||||
for item in result_select.fetchall():
|
||||
self.servers[item[0]] = {
|
||||
|
|
|
|||
Loading…
Reference in a new issue