adelgado/discover-mastodon-servers
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add default values

Browse source
This commit is contained in:
Antonio J. Delgado 2024-01-20 19:06:37 +02:00
parent 7bb0a5ec5a
commit c1c431cf12
1 changed files with 107 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

179
discover-mastodon-servers/discover_mastodon_servers.py
View file

@ -45,7 +45,9 @@ class DiscoverMastodonServers:
self.servers[self.config['initial_server']] = { self.servers[self.config['initial_server']] = {
"name": self.config['initial_server'], "name": self.config['initial_server'],
"last_update": time.time(), "last_update": time.time(),
"private": False "private": False,
"peers": True,
"timeline": True,
} }
new_servers_count = 1 new_servers_count = 1
while new_servers_count > 0: while new_servers_count > 0:
@ -60,9 +62,11 @@ class DiscoverMastodonServers:
data = None data = None
try: try:
result = self.session.get( result = self.session.get(
f"https://{server}/{endpoint}", f"https://{server['name']}/{endpoint}",
timeout=10 timeout=10
) )
server['status'] = result.status_code
server['state'] = 'OK'
if result.status_code < 400: if result.status_code < 400:
if 'Content-Type' in result.headers: if 'Content-Type' in result.headers:
if 'application/json' in result.headers['Content-Type']: if 'application/json' in result.headers['Content-Type']:
@ -70,68 +74,78 @@ class DiscoverMastodonServers:
if 'error' not in data: if 'error' not in data:
return data return data
else: else:
server['state'] = 'Error'
self._log.debug( self._log.debug(
"Server '%s' didn't reply with JSON data.", server "Server '%s' didn't reply with JSON data.", server['name']
) )
else: else:
server['state'] = 'Error'
self._log.debug( self._log.debug(
"Server '%s' didn't return Content-Type header. Headers: '%s'. Content returned: '%s'", "Server '%s' didn't return Content-Type header. Headers: '%s'. Content returned: '%s'",
server, server['name'],
json.dumps(result.headers, indent=2), json.dumps(result.headers, indent=2),
result.content result.content
) )
else: else:
server['state'] = 'Error'
self._log.debug( self._log.debug(
"Server '%s' returned error code %s.", server, result.status_code "Server '%s' returned error code %s.", server['name'], result.status_code
) )
except requests.exceptions.ReadTimeout as error: except requests.exceptions.ReadTimeout as error:
server['state'] = 'Error'
self._log.warning( self._log.warning(
"Server '%s' didn't respond on time. %s", "Server '%s' didn't respond on time. %s",
server, server['name'],
error error
) )
except requests.exceptions.SSLError as error: except requests.exceptions.SSLError as error:
server['state'] = 'SSL Error'
self._log.warning( self._log.warning(
"Server '%s' don't have a valid SSL certificate. %s", "Server '%s' don't have a valid SSL certificate. %s",
server, server['name'],
error error
) )
except requests.exceptions.ConnectionError as error: except requests.exceptions.ConnectionError as error:
server['state'] = 'Error'
self._log.warning( self._log.warning(
"Server '%s' connection failed. %s", "Server '%s' connection failed. %s",
server, server['name'],
error
)
except requests.exceptions.TooManyRedirects as error:
server['state'] = 'Error'
self._log.warning(
"Server '%s' redirected too many times. %s",
server['name'],
error error
) )
return data return data
def get_instance_info(self, server): def get_instance_info(self, server):
'''Get all server information''' '''Get all server information'''
result = {} instance = self.get_path(server['name'], '/api/v1/instance')
instance = self.get_path(server, '/api/v1/instance')
if instance: if instance:
result['instance'] = instance server['instance'] = instance
directory = [] server['directory'] = []
result['directory'] = directory
offset=0 offset=0
while len(directory) == 0: while len(server['directory']) == 0:
directory = self.get_path( directory = self.get_path(
server, server,
f"/api/v1/directory?limit=80&offset={offset}" f"/api/v1/directory?limit=80&offset={offset}"
) )
if directory: if directory:
result['directory'] = result['directory'] + directory server['directory'] = server['directory'] + directory
offset += 80 offset += 80
return result
def test_banned_server(self, server): def test_banned_server(self, server_name):
'''Check if a server name match agains any banned regular expressions''' '''Check if a server name match agains any banned regular expressions'''
for banned in self.config['regexp_banned_host']: for banned in self.config['regexp_banned_host']:
match = re.search(banned, server) match = re.search(banned, server_name)
if match: if match:
self._log.debug( self._log.debug(
"Regexp '%s' match server '%s', banned.", "Regexp '%s' match server '%s', banned.",
banned, banned,
server server_name
) )
return True return True
return False return False
@ -140,12 +154,20 @@ class DiscoverMastodonServers:
'''Discover new servers''' '''Discover new servers'''
all_servers = [] all_servers = []
new_servers_count = 0 new_servers_count = 0
for server in self.servers.items(): for server_name, server in self.servers.items():
all_servers.append(server[0]) all_servers.append(server_name)
if not self.test_banned_server(server[0]): if not self.test_banned_server(server_name):
if not server[1]['private']: if 'state' not in server:
self._log.debug("Fetching peers of the server '%s'", server[0]) server['state'] = 'Unknown'
data = self.get_path(server[0], 'api/v1/instance/peers') if 'status' not in server:
server['status'] = 0
if 'peers' not in server:
server['peers'] = True
if 'timeline' not in server:
server['timeline'] = True
if not server['private'] and 'Error' not in server['state']:
self._log.debug("Fetching peers of the server '%s'", server_name)
data = self.get_path(server, 'api/v1/instance/peers')
if data: if data:
for new_server in data: for new_server in data:
if ((not self.test_banned_server(new_server)) and if ((not self.test_banned_server(new_server)) and
@ -158,16 +180,18 @@ class DiscoverMastodonServers:
) )
all_servers.append(new_server) all_servers.append(new_server)
self.write_record( self.write_record(
(new_server,
{ {
"name": new_server, "name": new_server,
"last_update": time.time(), "last_update": time.time(),
"private": False "private": False,
"peers": True,
"timeline": True,
} }
)
) )
self._log.debug("Fetching public timeline in server '%s'", server[0]) else:
data = self.get_timeline(server[0]) server[1]['peers'] = False
self._log.debug("Fetching public timeline in server '%s'", server_name)
data = self.get_timeline(server)
if data: if data:
for item in data: for item in data:
if 'uri' in item: if 'uri' in item:
@ -175,7 +199,8 @@ class DiscoverMastodonServers:
if match_server: if match_server:
new_server = match_server.group(1) new_server = match_server.group(1)
if not self.test_banned_server(new_server) and new_server not in all_servers: if not self.test_banned_server(new_server) and new_server not in all_servers:
data = self.get_timeline(new_server) new_server_obj = { "name": new_server }
data = self.get_timeline(new_server_obj)
if data: if data:
new_servers_count += 1 new_servers_count += 1
self._log.debug( self._log.debug(
@ -183,18 +208,10 @@ class DiscoverMastodonServers:
new_server new_server
) )
all_servers.append(new_server) all_servers.append(new_server)
private = False new_server_obj['private'] = False
else: else:
private = True new_server_obj['private'] = True
self.write_record( self.write_record(new_server_obj)
(new_server,
{
"name": new_server,
"last_update": time.time(),
"private": private
}
)
)
else: else:
# Item in public timeline don't have an URI # Item in public timeline don't have an URI
self._log.debug( self._log.debug(
@ -202,55 +219,73 @@ class DiscoverMastodonServers:
item item
) )
else: else:
server[1]['private'] = True server['timeline'] = False
self.write_record(server) self.write_record(server)
return new_servers_count return new_servers_count
def write_record(self, record, table='servers'): def write_record(self, record, table='servers'):
'''Write record to a table''' '''Write record to a table'''
if 'state' not in record:
record['state'] = 'Unknown'
if 'status' not in record:
record['status'] = 0
if 'peers' not in record:
record['peers'] = True
if 'timeline' not in record:
record['timeline'] = True
cur = self.conn.cursor() cur = self.conn.cursor()
result_select = cur.execute(f""" result_select = cur.execute(f"""
SELECT name FROM {table} WHERE name = '{record[1]['name']}' SELECT name FROM {table} WHERE name = '{record['name']}'
""") """)
if len(result_select.fetchall()) > 0: if len(result_select.fetchall()) > 0:
self._log.debug('Record exists, updating.') self._log.debug('Record exists, updating.')
query = f""" query = f"UPDATE {table} SET "
UPDATE {table} count = 0
SET last_update = {record[1]['last_update']}, for key in record.keys():
private = {record[1]['private']} if count == 0:
WHERE name = '{record[0]}' query += f"{key} = :{key} "
""" else:
query += f",{key} = :{key} "
count += 1
query += "WHERE name = :name"
else: else:
self._log.debug('Record doesn\'t exist, inserting.') self._log.debug('Record doesn\'t exist, inserting.')
query = f""" query = f"INSERT INTO {table} VALUES (:" + ",:".join(record.keys()) + ")"
INSERT INTO {table} ( self._log.debug("Writing record '%s'...",
name, record
last_update,
private
)
VALUES (
'{record[0]}',
{record[1]['last_update']},
{record[1]['private']}
)
"""
result_update = cur.execute(query)
self._log.debug("Written record '%s' with ID %s",
record[0],
result_update.lastrowid
) )
try:
result_update = cur.execute(query, record)
self._log.debug("Added record %s.", result_update.lastrowid)
except Exception as error:
self._log.error("Error running query '%s' with record '%s'. %s", query, record, error)
sys.exit(1)
cur.close() cur.close()
self.conn.commit() self.conn.commit()
def read_db(self): def read_db(self):
'''Read database file''' '''Read database file'''
cur = self.conn.cursor() cur = self.conn.cursor()
cur.execute("""CREATE TABLE IF NOT EXISTS servers( query = """CREATE TABLE IF NOT EXISTS servers(
name TEXT PRIMARY KEY, name TEXT PRIMARY KEY,
last_update REAL, last_update REAL,
private INT private INT,
)""") peers INT,
result_select = cur.execute("SELECT * FROM servers ORDER BY last_update DESC") timeline INT,
status INT,
state TEXT
)"""
try:
cur.execute(query)
except Exception as error:
self._log.error("Error running query to create table '%s'. %s", query, error)
sys.exit(2)
query = "SELECT * FROM servers ORDER BY last_update DESC"
try:
result_select = cur.execute(query)
except Exception as error:
self._log.error("Error running query to list servers '%s'. %s", query, error)
sys.exit(3)
self.servers = {} self.servers = {}
for item in result_select.fetchall(): for item in result_select.fetchall():
self.servers[item[0]] = { self.servers[item[0]] = {