From eb237443d3f510378a09f2cdeb648004ed3d1414 Mon Sep 17 00:00:00 2001 From: "Antonio J. Delgado" Date: Fri, 2 Aug 2024 18:53:29 +0300 Subject: [PATCH] Avoid exceding rate limits --- .../mastodon_email_bridge.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mastodon_email_bridge/mastodon_email_bridge.py b/mastodon_email_bridge/mastodon_email_bridge.py index 95e88a5..bd8ada3 100755 --- a/mastodon_email_bridge/mastodon_email_bridge.py +++ b/mastodon_email_bridge/mastodon_email_bridge.py @@ -65,6 +65,8 @@ class MastodonEmailBridge: next_url = self.process_url(next_url) while next_url != "" and (count < self.config['limit'] or self.config['limit'] == 0): count = count + 1 + self._log.debug("Waiting 2 seconds for next request...") + time.sleep(2) self._log.debug("Getting URL '%s'", next_url) next_url = self.process_url(next_url) @@ -73,11 +75,15 @@ class MastodonEmailBridge: self.sqlite = sqlite3.connect(self.config['sent_items_file']) self.sqlite.row_factory = sqlite3.Row cur = self.sqlite.cursor() - cur.execute("CREATE TABLE IF NOT EXISTS sent_items(id, date)") + cur.execute("CREATE TABLE IF NOT EXISTS sent_items(id PRIMARY KEY, date)") self.sqlite.commit() res = cur.execute("SELECT id FROM sent_items") rows = res.fetchall() for row in rows: + if row[0] in self.sent_items: + self._log.warning("Duplicate id in database '%s'", row[0]) + else: + self._log.debug("Found sent item with id '%s' (%s)", row[0], type(row[0])) self.sent_items.append(row[0]) return True @@ -85,9 +91,17 @@ class MastodonEmailBridge: '''Process a home endpoint URL''' next_url = '' result = self.session.get(url) + # for header in result.headers: + # self._log.debug("%s = %s", header, result.headers[header]) + if 'X-RateLimit-Remaining' in result.headers and int(result.headers['X-RateLimit-Remaining']) < 10: + self._log.warning("X-RateLimit-Reset: %s", result.headers['X-RateLimit-Reset']) + self._log.warning('Wait until that time to try again.') + return url data = result.json()[0] - if data['id'] not in self.sent_items: + if int(data['id']) not in self.sent_items: self.send_mail(data) + else: + self._log.debug("Skipping post %s that was sent in the past", data['id']) if 'Link' in result.headers: for link in result.headers['Link'].split(', '): slink = link.split('; ') @@ -133,7 +147,7 @@ class MastodonEmailBridge: conn=smtplib.SMTP_SSL(self.config['mail_server'],self.config['mail_server_port']) if self.config['mail_user'] is not None: conn.login(self.config['mail_user'], self.config['mail_pass']) - self._log.debug("Sending email...") + self._log.debug("Sending email for post with id '%s'...", data['id']) conn.sendmail(self.config['sender'], self.config['recipient'], msg.as_string()) conn.quit() self._log.debug("Adding entry to database...")