diff --git a/discover-mastodon-servers/discover_mastodon_servers.py b/discover-mastodon-servers/discover_mastodon_servers.py index e09957d..26bf23a 100755 --- a/discover-mastodon-servers/discover_mastodon_servers.py +++ b/discover-mastodon-servers/discover_mastodon_servers.py @@ -45,7 +45,9 @@ class DiscoverMastodonServers: self.servers[self.config['initial_server']] = { "name": self.config['initial_server'], "last_update": time.time(), - "private": False + "private": False, + "peers": True, + "timeline": True, } new_servers_count = 1 while new_servers_count > 0: @@ -60,9 +62,11 @@ class DiscoverMastodonServers: data = None try: result = self.session.get( - f"https://{server}/{endpoint}", + f"https://{server['name']}/{endpoint}", timeout=10 ) + server['status'] = result.status_code + server['state'] = 'OK' if result.status_code < 400: if 'Content-Type' in result.headers: if 'application/json' in result.headers['Content-Type']: @@ -70,68 +74,78 @@ class DiscoverMastodonServers: if 'error' not in data: return data else: + server['state'] = 'Error' self._log.debug( - "Server '%s' didn't reply with JSON data.", server + "Server '%s' didn't reply with JSON data.", server['name'] ) else: + server['state'] = 'Error' self._log.debug( "Server '%s' didn't return Content-Type header. Headers: '%s'. Content returned: '%s'", - server, + server['name'], json.dumps(result.headers, indent=2), result.content ) else: + server['state'] = 'Error' self._log.debug( - "Server '%s' returned error code %s.", server, result.status_code + "Server '%s' returned error code %s.", server['name'], result.status_code ) except requests.exceptions.ReadTimeout as error: + server['state'] = 'Error' self._log.warning( "Server '%s' didn't respond on time. %s", - server, + server['name'], error ) except requests.exceptions.SSLError as error: + server['state'] = 'SSL Error' self._log.warning( "Server '%s' don't have a valid SSL certificate. %s", - server, + server['name'], error ) except requests.exceptions.ConnectionError as error: + server['state'] = 'Error' self._log.warning( "Server '%s' connection failed. %s", - server, + server['name'], + error + ) + except requests.exceptions.TooManyRedirects as error: + server['state'] = 'Error' + self._log.warning( + "Server '%s' redirected too many times. %s", + server['name'], error ) return data def get_instance_info(self, server): '''Get all server information''' - result = {} - instance = self.get_path(server, '/api/v1/instance') + instance = self.get_path(server['name'], '/api/v1/instance') if instance: - result['instance'] = instance - directory = [] - result['directory'] = directory + server['instance'] = instance + server['directory'] = [] offset=0 - while len(directory) == 0: + while len(server['directory']) == 0: directory = self.get_path( server, f"/api/v1/directory?limit=80&offset={offset}" ) if directory: - result['directory'] = result['directory'] + directory + server['directory'] = server['directory'] + directory offset += 80 - return result - def test_banned_server(self, server): + def test_banned_server(self, server_name): '''Check if a server name match agains any banned regular expressions''' for banned in self.config['regexp_banned_host']: - match = re.search(banned, server) + match = re.search(banned, server_name) if match: self._log.debug( "Regexp '%s' match server '%s', banned.", banned, - server + server_name ) return True return False @@ -140,12 +154,20 @@ class DiscoverMastodonServers: '''Discover new servers''' all_servers = [] new_servers_count = 0 - for server in self.servers.items(): - all_servers.append(server[0]) - if not self.test_banned_server(server[0]): - if not server[1]['private']: - self._log.debug("Fetching peers of the server '%s'", server[0]) - data = self.get_path(server[0], 'api/v1/instance/peers') + for server_name, server in self.servers.items(): + all_servers.append(server_name) + if not self.test_banned_server(server_name): + if 'state' not in server: + server['state'] = 'Unknown' + if 'status' not in server: + server['status'] = 0 + if 'peers' not in server: + server['peers'] = True + if 'timeline' not in server: + server['timeline'] = True + if not server['private'] and 'Error' not in server['state']: + self._log.debug("Fetching peers of the server '%s'", server_name) + data = self.get_path(server, 'api/v1/instance/peers') if data: for new_server in data: if ((not self.test_banned_server(new_server)) and @@ -158,16 +180,18 @@ class DiscoverMastodonServers: ) all_servers.append(new_server) self.write_record( - (new_server, { "name": new_server, "last_update": time.time(), - "private": False + "private": False, + "peers": True, + "timeline": True, } - ) ) - self._log.debug("Fetching public timeline in server '%s'", server[0]) - data = self.get_timeline(server[0]) + else: + server[1]['peers'] = False + self._log.debug("Fetching public timeline in server '%s'", server_name) + data = self.get_timeline(server) if data: for item in data: if 'uri' in item: @@ -175,7 +199,8 @@ class DiscoverMastodonServers: if match_server: new_server = match_server.group(1) if not self.test_banned_server(new_server) and new_server not in all_servers: - data = self.get_timeline(new_server) + new_server_obj = { "name": new_server } + data = self.get_timeline(new_server_obj) if data: new_servers_count += 1 self._log.debug( @@ -183,18 +208,10 @@ class DiscoverMastodonServers: new_server ) all_servers.append(new_server) - private = False + new_server_obj['private'] = False else: - private = True - self.write_record( - (new_server, - { - "name": new_server, - "last_update": time.time(), - "private": private - } - ) - ) + new_server_obj['private'] = True + self.write_record(new_server_obj) else: # Item in public timeline don't have an URI self._log.debug( @@ -202,55 +219,73 @@ class DiscoverMastodonServers: item ) else: - server[1]['private'] = True + server['timeline'] = False self.write_record(server) return new_servers_count def write_record(self, record, table='servers'): '''Write record to a table''' + if 'state' not in record: + record['state'] = 'Unknown' + if 'status' not in record: + record['status'] = 0 + if 'peers' not in record: + record['peers'] = True + if 'timeline' not in record: + record['timeline'] = True cur = self.conn.cursor() result_select = cur.execute(f""" - SELECT name FROM {table} WHERE name = '{record[1]['name']}' + SELECT name FROM {table} WHERE name = '{record['name']}' """) if len(result_select.fetchall()) > 0: self._log.debug('Record exists, updating.') - query = f""" - UPDATE {table} - SET last_update = {record[1]['last_update']}, - private = {record[1]['private']} - WHERE name = '{record[0]}' - """ + query = f"UPDATE {table} SET " + count = 0 + for key in record.keys(): + if count == 0: + query += f"{key} = :{key} " + else: + query += f",{key} = :{key} " + count += 1 + query += "WHERE name = :name" else: self._log.debug('Record doesn\'t exist, inserting.') - query = f""" - INSERT INTO {table} ( - name, - last_update, - private - ) - VALUES ( - '{record[0]}', - {record[1]['last_update']}, - {record[1]['private']} - ) - """ - result_update = cur.execute(query) - self._log.debug("Written record '%s' with ID %s", - record[0], - result_update.lastrowid + query = f"INSERT INTO {table} VALUES (:" + ",:".join(record.keys()) + ")" + self._log.debug("Writing record '%s'...", + record ) + try: + result_update = cur.execute(query, record) + self._log.debug("Added record %s.", result_update.lastrowid) + except Exception as error: + self._log.error("Error running query '%s' with record '%s'. %s", query, record, error) + sys.exit(1) cur.close() self.conn.commit() def read_db(self): '''Read database file''' cur = self.conn.cursor() - cur.execute("""CREATE TABLE IF NOT EXISTS servers( - name TEXT PRIMARY KEY, - last_update REAL, - private INT - )""") - result_select = cur.execute("SELECT * FROM servers ORDER BY last_update DESC") + query = """CREATE TABLE IF NOT EXISTS servers( + name TEXT PRIMARY KEY, + last_update REAL, + private INT, + peers INT, + timeline INT, + status INT, + state TEXT + )""" + try: + cur.execute(query) + except Exception as error: + self._log.error("Error running query to create table '%s'. %s", query, error) + sys.exit(2) + query = "SELECT * FROM servers ORDER BY last_update DESC" + try: + result_select = cur.execute(query) + except Exception as error: + self._log.error("Error running query to list servers '%s'. %s", query, error) + sys.exit(3) self.servers = {} for item in result_select.fetchall(): self.servers[item[0]] = {