Add banned hostnames

discover-mastodon-servers/discover_mastodon_servers.py

@@ -12,6 +12,7 @@ from logging.handlers import SysLogHandler
 import sqlite3
 import time
 import re
+import json
 import click
 import click_config_file
 import requests
@@ -63,6 +64,7 @@ class DiscoverMastodonServers:
                 timeout=10
                 )
             if result.status_code < 400:
+                if 'Content-Type' in result.headers:
                     if 'application/json' in result.headers['Content-Type']:
                         data = result.json()
                         if 'error' not in data:
@@ -71,6 +73,13 @@ class DiscoverMastodonServers:
                         self._log.debug(
                             "Server '%s' didn't reply with JSON data.", server
                         )
+                else:
+                    self._log.debug(
+                        "Server '%s' didn't return Content-Type header. Headers: '%s'. Content returned: '%s'",
+                        server,
+                        json.dumps(result.headers, indent=2),
+                        result.content
+                    )
             else:
                 self._log.debug(
                     "Server '%s' returned error code %s.", server, result.status_code
@@ -95,12 +104,45 @@ class DiscoverMastodonServers:
             )
         return data
 
+    def get_instance_info(self, server):
+        '''Get all server information'''
+        result = {}
+        instance = self.get_path(server, '/api/v1/instance')
+        if instance:
+            result['instance'] = instance
+        directory = []
+        result['directory'] = directory
+        offset=0
+        while len(directory) == 0:
+            directory = self.get_path(
+                server,
+                f"/api/v1/directory?limit=80&offset={offset}"
+            )
+            if directory:
+                result['directory'] = result['directory'] + directory
+            offset += 80
+        return result
+
+    def test_banned_server(self, server):
+        '''Check if a server name match agains any banned regular expressions'''
+        for banned in self.config['regexp_banned_host']:
+            match = re.search(banned, server)
+            if match:
+                self._log.debug(
+                    "Regexp '%s' match server '%s'",
+                    banned,
+                    server
+                )
+                return True
+        return False
+
     def discover(self):
         '''Discover new servers'''
         all_servers = []
         new_servers_count = 0
         for server in self.servers.items():
             all_servers.append(server[0])
+            if not self.test_banned_server(server[0]):
                 if not server[1]['private']:
                     self._log.debug("Fetching peers of the server '%s'", server[0])
                     data = self.get_path(server[0], 'api/v1/instance/peers')
@@ -279,6 +321,10 @@ class DiscoverMastodonServers:
     '--database-file', '-d', default='mastodon-servers.db',
     help='File with the database of results.'
 )
+@click.option(
+    '--regexp-banned-host', '-r', multiple=True,
+    help='Regular expression for banned host names.'
+)
 @click_config_file.configuration_option()
 def __main__(**kwargs):
     return DiscoverMastodonServers(**kwargs)