Handle errors and add proxies

get_youtube_videos/get_youtube_videos.py

@@ -54,7 +54,12 @@ class GetYoutubeVideos:
             'skipped_videos': 0,
             'downloaded_videos': 0,
             'videos_with_error': 0,
+            'possible_proxy_errors': 0,
+            'possible_ban_errors': 0,
+            'age_limit_errors': 0,
         }
+        self.selected_proxy = ''
+        self.proxy_index = 0
         if os.path.exists(self.config['downloaded_database']):
             with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
                 self.downloaded_items = db_file.read().split('\n')
@@ -71,6 +76,47 @@ class GetYoutubeVideos:
             )
         )
 
+    def _get_video_info(self, video_id):
+        uri=f"https://www.youtube.com/watch?v={video_id}"
+        ydl_opts = {
+            'logger': self._log,
+            'progress_hooks': [self._yt_progress_hook],
+        }
+        if self.selected_proxy != '':
+            ydl_opts['proxy'] = self.selected_proxy
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            try:
+                raw_video_info = ydl.extract_info(uri, download=False)
+            except yt_dlp.utils.DownloadError as error:
+                self._log.debug(
+                    "Skipping video. Error: %s",
+                    error
+                )
+                if 'The uploader has not made this video available in your country' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                elif 'HTTP Error 403: Forbidden' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                    self.summary['possible_proxy_errors'] += 1
+                    if len(self.config['proxy']) > 0:
+                        if self.proxy_index<len(self.config['proxy']):
+                            self.selected_proxy = self.config['proxy'][self.proxy_index]
+                            self.proxy_index += 1
+                            return self._get_video_info(video_id)
+                        self.proxy_index = 0
+                        self.selected_proxy = ''
+                        return None
+                elif 'not a bot' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                    self.summary['possible_ban_errors'] += 1
+                    # elif age_limit_errors?
+                elif 'This live event will begin in a few moments' in f"{error}":
+                    self.summary['skipped_videos'] += 1
+                else:
+                    self.summary['skipped_videos'] += 1
+                return None
+        video_info = ydl.sanitize_info(raw_video_info)
+        return video_info
+
     def _process_channels(self):
         self.total_count = 0
         self.channels_count = 0
@@ -99,24 +145,11 @@ class GetYoutubeVideos:
                 if result:
                     video_id=result.group(1)
                     if video_id not in self.downloaded_items:
-                        # print(json.dumps(entry, indent=2))
+                        video_info = self._get_video_info(video_id)
-                        uri=f"https://www.youtube.com/watch?v={video_id}"
+                        info_filename = os.path.join(
-                        ydl_opts = {
+                            self.config['download_dir'],
-                            'logger': self._log,
+                            f"{video_id}.json"
-                            'progress_hooks': [self._yt_progress_hook],
-                        }
-                        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                            try:
-                                raw_video_info = ydl.extract_info(uri, download=False)
-                            except yt_dlp.utils.DownloadError as error:
-                                self._log.debug(
-                                    "%s Skipping video.",
-                                    error
                         )
-                                self.summary['skipped_videos'] += 1
-                                break
-                            video_info = ydl.sanitize_info(raw_video_info)
-                            info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
                         self._log.debug(
                             "Writting information in to file '%s'",
                             info_filename
@@ -163,10 +196,9 @@ class GetYoutubeVideos:
                             self._save_downloaded_items(video_id)
                             break
                         self._log.info(
-                            "Downloading. Filename: '%s'. Video ID: '%s'. Video URL: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
+                            "Downloading. Filename: '%s'. Video ID: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
                             video_info.get('title', '?'),
                             video_id,
-                            uri,
                             self._human_time_duration(video_info.get('duration', '-1')),
                             self.total_count+1,
                             self.config['total_limit'],
@@ -181,8 +213,6 @@ class GetYoutubeVideos:
                         else:
                             download_dir = self.config['download_dir']
                         ydl_opts = {
-                            'logger': self._log,
-                            'progress_hooks': [self._yt_progress_hook],
                             'paths': {
                                 'temp': '/tmp',
                                 'home': download_dir
@@ -200,8 +230,11 @@ class GetYoutubeVideos:
                             'merge_output_format': 'mkv',
                             'format': 'bestvideo+bestaudio[ext=m4a]/best',
                         }
+                        if self.selected_proxy != '':
+                            ydl_opts['proxy'] = self.selected_proxy
                         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                             try:
+                                uri=f"https://www.youtube.com/watch?v={video_id}"
                                 return_code = ydl.download(uri)
                                 self._process_download(
                                     {
@@ -349,6 +382,11 @@ class GetYoutubeVideos:
 @click.option('--log-file', '-l', help="File to store all debug messages.")
 # @click.option("--dummy","-n", is_flag=True,
 # help="Don't do anything, just show what would be done.")
+@click.option(
+    '--proxy', '-p',
+    multiple=True,
+    help='URLs for proxies to use in case of errors.'
+)
 @click.option(
     '--downloaded-database', '-d',
     default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",