From ab1bc5af9296ffc7508c27eb76f6dc6dab97c552 Mon Sep 17 00:00:00 2001
From: "Antonio J. Delgado" <ad@susurrando.com>
Date: Sun, 29 Dec 2024 17:28:43 +0200
Subject: [PATCH] Handle errors and add proxies

---
 get_youtube_videos/get_youtube_videos.py | 168 ++++++++++++++---------
 1 file changed, 103 insertions(+), 65 deletions(-)

diff --git a/get_youtube_videos/get_youtube_videos.py b/get_youtube_videos/get_youtube_videos.py
index eaf9980..6b7de6a 100644
--- a/get_youtube_videos/get_youtube_videos.py
+++ b/get_youtube_videos/get_youtube_videos.py
@@ -54,7 +54,12 @@ class GetYoutubeVideos:
             'skipped_videos': 0,
             'downloaded_videos': 0,
             'videos_with_error': 0,
+            'possible_proxy_errors': 0,
+            'possible_ban_errors': 0,
+            'age_limit_errors': 0,
         }
+        self.selected_proxy = ''
+        self.proxy_index = 0
         if os.path.exists(self.config['downloaded_database']):
             with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
                 self.downloaded_items = db_file.read().split('\n')
@@ -71,6 +76,47 @@ class GetYoutubeVideos:
             )
         )
 
+    def _get_video_info(self, video_id):
+        uri=f"https://www.youtube.com/watch?v={video_id}"
+        ydl_opts = {
+            'logger': self._log,
+            'progress_hooks': [self._yt_progress_hook],
+        }
+        if self.selected_proxy != '':
+            ydl_opts['proxy'] = self.selected_proxy
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            try:
+                raw_video_info = ydl.extract_info(uri, download=False)
+            except yt_dlp.utils.DownloadError as error:
+                self._log.debug(
+                    "Skipping video. Error: %s",
+                    error
+                )
+                if 'The uploader has not made this video available in your country' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                elif 'HTTP Error 403: Forbidden' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                    self.summary['possible_proxy_errors'] += 1
+                    if len(self.config['proxy']) > 0:
+                        if self.proxy_index<len(self.config['proxy']):
+                            self.selected_proxy = self.config['proxy'][self.proxy_index]
+                            self.proxy_index += 1
+                            return self._get_video_info(video_id)
+                        self.proxy_index = 0
+                        self.selected_proxy = ''
+                        return None
+                elif 'not a bot' in f"{error}":
+                    self.summary['videos_with_error'] += 1
+                    self.summary['possible_ban_errors'] += 1
+                    # elif age_limit_errors?
+                elif 'This live event will begin in a few moments' in f"{error}":
+                    self.summary['skipped_videos'] += 1
+                else:
+                    self.summary['skipped_videos'] += 1
+                return None
+        video_info = ydl.sanitize_info(raw_video_info)
+        return video_info
+
     def _process_channels(self):
         self.total_count = 0
         self.channels_count = 0
@@ -99,74 +145,60 @@ class GetYoutubeVideos:
                 if result:
                     video_id=result.group(1)
                     if video_id not in self.downloaded_items:
-                        # print(json.dumps(entry, indent=2))
-                        uri=f"https://www.youtube.com/watch?v={video_id}"
-                        ydl_opts = {
-                            'logger': self._log,
-                            'progress_hooks': [self._yt_progress_hook],
-                        }
-                        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                            try:
-                                raw_video_info = ydl.extract_info(uri, download=False)
-                            except yt_dlp.utils.DownloadError as error:
-                                self._log.debug(
-                                    "%s Skipping video.",
-                                    error
-                                )
-                                self.summary['skipped_videos'] += 1
-                                break
-                            video_info = ydl.sanitize_info(raw_video_info)
-                            info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
+                        video_info = self._get_video_info(video_id)
+                        info_filename = os.path.join(
+                            self.config['download_dir'],
+                            f"{video_id}.json"
+                        )
+                        self._log.debug(
+                            "Writting information in to file '%s'",
+                            info_filename
+                        )
+                        with open(info_filename, 'w', encoding='utf-8') as info_file:
+                            json.dump(video_info, info_file, indent=2)
+                        if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
                             self._log.debug(
-                                "Writting information in to file '%s'",
-                                info_filename
+                                "Skipping video '%s' as it's a live video",
+                                video_info.get('title', '?')
                             )
-                            with open(info_filename, 'w', encoding='utf-8') as info_file:
-                                json.dump(video_info, info_file, indent=2)
-                            if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
-                                self._log.debug(
-                                    "Skipping video '%s' as it's a live video",
-                                    video_info.get('title', '?')
-                                )
-                                self.summary['skipped_videos'] += 1
-                                self.downloaded_items.append(video_id)
-                                with open(
-                                    self.config['downloaded_database'], 'w', encoding='utf-8'
-                                ) as db_file:
-                                    for item in  self.downloaded_items:
-                                        db_file.write(f"{item}\n")
-                                break
-                            if video_info['was_live']:
-                                self._log.debug(
-                                    "Skipping video '%s' as it was a live video",
-                                    video_info.get('title', '?')
-                                )
-                                self.summary['skipped_videos'] += 1
-                                self._save_downloaded_items(video_id)
-                                break
-                            if ('duration' in video_info and
-                                video_info['duration'] > self.config['max_length']):
-                                self._log.debug(
-                                    "Skipping video '%s' as it was larger than %s",
-                                    video_info.get('title', '?'),
-                                    self._human_time_duration(self.config['max_length'])
-                                )
-                                self.summary['skipped_videos'] += 1
-                                self._save_downloaded_items(video_id)
-                                break
-                            if 'duration' not in video_info:
-                                self._log.debug(
-                                    "Skipping video '%s' as there is no video duration",
-                                    video_info.get('title', '?')
-                                )
-                                self.summary['skipped_videos'] += 1
-                                self._save_downloaded_items(video_id)
-                                break
+                            self.summary['skipped_videos'] += 1
+                            self.downloaded_items.append(video_id)
+                            with open(
+                                self.config['downloaded_database'], 'w', encoding='utf-8'
+                            ) as db_file:
+                                for item in  self.downloaded_items:
+                                    db_file.write(f"{item}\n")
+                            break
+                        if video_info['was_live']:
+                            self._log.debug(
+                                "Skipping video '%s' as it was a live video",
+                                video_info.get('title', '?')
+                            )
+                            self.summary['skipped_videos'] += 1
+                            self._save_downloaded_items(video_id)
+                            break
+                        if ('duration' in video_info and
+                            video_info['duration'] > self.config['max_length']):
+                            self._log.debug(
+                                "Skipping video '%s' as it was larger than %s",
+                                video_info.get('title', '?'),
+                                self._human_time_duration(self.config['max_length'])
+                            )
+                            self.summary['skipped_videos'] += 1
+                            self._save_downloaded_items(video_id)
+                            break
+                        if 'duration' not in video_info:
+                            self._log.debug(
+                                "Skipping video '%s' as there is no video duration",
+                                video_info.get('title', '?')
+                            )
+                            self.summary['skipped_videos'] += 1
+                            self._save_downloaded_items(video_id)
+                            break
                         self._log.info(
-                            "Downloading. Filename: '%s'. Video ID: '%s'. Video URL: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
+                            "Downloading. Filename: '%s'. Video ID: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
                             video_info.get('title', '?'),
                             video_id,
-                            uri,
                             self._human_time_duration(video_info.get('duration', '-1')),
                             self.total_count+1,
                             self.config['total_limit'],
@@ -181,8 +213,6 @@ class GetYoutubeVideos:
                         else:
                             download_dir = self.config['download_dir']
                         ydl_opts = {
-                            'logger': self._log,
-                            'progress_hooks': [self._yt_progress_hook],
                             'paths': {
                                 'temp': '/tmp',
                                 'home': download_dir
@@ -200,8 +230,11 @@ class GetYoutubeVideos:
                             'merge_output_format': 'mkv',
                             'format': 'bestvideo+bestaudio[ext=m4a]/best',
                         }
+                        if self.selected_proxy != '':
+                            ydl_opts['proxy'] = self.selected_proxy
                         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                             try:
+                                uri=f"https://www.youtube.com/watch?v={video_id}"
                                 return_code = ydl.download(uri)
                                 self._process_download(
                                     {
@@ -349,6 +382,11 @@ class GetYoutubeVideos:
 @click.option('--log-file', '-l', help="File to store all debug messages.")
 # @click.option("--dummy","-n", is_flag=True,
 # help="Don't do anything, just show what would be done.")
+@click.option(
+    '--proxy', '-p',
+    multiple=True,
+    help='URLs for proxies to use in case of errors.'
+)
 @click.option(
     '--downloaded-database', '-d',
     default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",