get_youtube_videos/get_youtube_videos.py

@@ -8,12 +8,10 @@
 import sys
 import os
 import re
-import time
 import stat
 import json
 import logging
 from logging.handlers import SysLogHandler
-import yaml
 from yaml import dump
 try:
     from yaml import CDumper as Dumper
@@ -25,14 +23,6 @@ import requests
 import feedparser
 import yt_dlp
 
-HOME_FOLDER = os.environ.get('HOME', os.environ.get('USERPROFILE', '/'))
-if HOME_FOLDER == '/':
-    CACHE_FOLDER = '/var/cache'
-    LOG_FOLDER = '/var/log/'
-else:
-    CACHE_FOLDER = f"{HOME_FOLDER}/.local/"
-    LOG_FOLDER = f"{HOME_FOLDER}/log/"
-
 
 class GetYoutubeVideos:
     '''Get YouTube videos from a series of channels'''
@@ -59,13 +49,6 @@ class GetYoutubeVideos:
                 'get_youtube_videos.log'
             )
         self._init_log()
-
-        self._default_data = {
-            "last_update": 0,
-            "proxies": {},
-        }
-        self.data = self._read_cached_data()
-
         self.summary = {
             'config': self.config,
             'entries_count': 0,
@@ -79,16 +62,9 @@ class GetYoutubeVideos:
             'downloaded_videos_titles': [],
         }
         if len(self.config['proxy']) > 0:
-            for proxy in self.config['proxy']:
+            self.selected_proxy = self.config['proxy'][0]
-                if proxy not in self._default_data['proxies']:
+            self.proxies = self.config['proxy']
-                    self._default_data['proxies'][proxy]['failures'] = 0
-            less_failures = -1
-            for proxy in self._default_data['proxies']:
-                if less_failures == -1 or self._default_data['proxies'][proxy]['failures'] < less_failures:
-                    less_failures = self._default_data['proxies'][proxy]['failures']
-                    self.selected_proxy = proxy
         else:
-            self._default_data['proxies'] = {}
             self.selected_proxy = ''
         self.proxy_index = 0
         if os.path.exists(self.config['downloaded_database']):
@@ -108,60 +84,6 @@ class GetYoutubeVideos:
         )
         self._save_metrics(self.summary)
 
-    def close(self):
-        '''Close class and save data'''
-        self._save_cached_data(self.data)
-
-    def _read_cached_data(self):
-        if os.path.exists(self.config['cache_file']):
-            with open(self.config['cache_file'], 'r', encoding='utf-8') as cache_file:
-                try:
-                    cached_data = json.load(cache_file)
-                    if (
-                        'last_update' in cached_data and
-                        cached_data['last_update'] + self.config['max_cache_age'] > time.time()
-                    ):
-                        cached_data = self._default_data
-                except json.decoder.JSONDecodeError:
-                    cached_data = self._default_data
-                return cached_data
-        else:
-            return self._default_data
-
-    def _save_cached_data(self, data):
-        data['last_update'] = time.time()
-        with open(self.config['cache_file'], 'w', encoding='utf-8') as cache_file:
-            json.dump(data, cache_file, indent=2)
-        self._debug(
-            f"Saved cached data in '{self.config['cache_file']}'",
-        )
-
-    def _output(self, message):
-        if self.config['output_format'] == 'JSON':
-            return json.dumps(message, indent=2)
-        elif self.config['output_format'] == 'YAML':
-            return yaml.dump(message, Dumper=yaml.Dumper)
-        elif self.config['output_format'] == 'PLAIN':
-            return f"{message}"
-        else:
-            self._log.warning(
-                "Output format '%s' not supported",
-                self.config['output_format']
-            )
-            return message
-
-    def _info(self, message):
-        return self._log.info(self._output(message))
-
-    def _warning(self, message):
-        return self._log.warning(self._output(message))
-
-    def _error(self, message):
-        return self._log.error(self._output(message))
-
-    def _debug(self, message):
-        return self._log.debug(self._output(message))
-
     def _is_numeric(self, variable):
         if isinstance(variable, (int, float, complex)):
             return True
@@ -186,21 +108,28 @@ class GetYoutubeVideos:
             os.chmod('/var/lib/prometheus/node-exporter/get_youtube_videos.prom', stat.S_IROTH)
 
     def _change_proxy(self, video_id):
-        less_failures = -1
+        if len(self.proxies) > 0:
-        for proxy in self._default_data['proxies']:
+            if isinstance(self.proxies, list):
-            if less_failures == -1 or self._default_data['proxies'][proxy]['failures'] < less_failures:
-                less_failures = self._default_data['proxies'][proxy['failures']]
-                self.selected_proxy = proxy
-        if less_failures == -1:
-            self.selected_proxy = list(self._default_data['proxies'].keys())[0]
                 self._log.debug(
-                "All proxies have the same number of errors, using first proxy of the list."
+                    "Removing failing proxy '%s'",
+                    self.selected_proxy
                 )
+                self.proxies.pop(self.proxy_index)
+            if self.proxy_index<len(self.proxies)-1:
+                self.selected_proxy = self.proxies[self.proxy_index]
+                self.proxy_index += 1
                 self._log.warning(
                     "Got an error fetching video information. Setting proxy to '%s'...",
                     self.selected_proxy
                 )
                 return self._get_video_info(video_id)
+            self._log.debug(
+                "No more proxies to try. Resetting list."
+            )
+            self.proxy_index = 0
+            self.proxies = self.config['proxy']
+            self.selected_proxy = self.proxies[0]
+        return None
 
     def _get_video_info(self, video_id):
         uri=f"https://www.youtube.com/watch?v={video_id}"
@@ -624,22 +553,9 @@ class GetYoutubeVideos:
     default=False,
     help='Skip live videos'
 )
-@click.option(
-    '--cache-file',
-    '-f',
-    default=f"{CACHE_FOLDER}/get_youtube_videos.json",
-    help='Cache file to store data from each run',
-)
-@click.option(
-    '--max-cache-age',
-    '-a',
-    default=60*60*24*7,
-    help='Max age in seconds for the cache'
-)
 @click_config_file.configuration_option()
 def __main__(**kwargs):
-    obj = GetYoutubeVideos(**kwargs)
+    return GetYoutubeVideos(**kwargs)
-    obj.close()
 
 if __name__ == "__main__":
     __main__()