Handle errors and add proxies
This commit is contained in:
parent
89f561d043
commit
ab1bc5af92
1 changed files with 103 additions and 65 deletions
|
|
@ -54,7 +54,12 @@ class GetYoutubeVideos:
|
|||
'skipped_videos': 0,
|
||||
'downloaded_videos': 0,
|
||||
'videos_with_error': 0,
|
||||
'possible_proxy_errors': 0,
|
||||
'possible_ban_errors': 0,
|
||||
'age_limit_errors': 0,
|
||||
}
|
||||
self.selected_proxy = ''
|
||||
self.proxy_index = 0
|
||||
if os.path.exists(self.config['downloaded_database']):
|
||||
with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
|
||||
self.downloaded_items = db_file.read().split('\n')
|
||||
|
|
@ -71,6 +76,47 @@ class GetYoutubeVideos:
|
|||
)
|
||||
)
|
||||
|
||||
def _get_video_info(self, video_id):
|
||||
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||
ydl_opts = {
|
||||
'logger': self._log,
|
||||
'progress_hooks': [self._yt_progress_hook],
|
||||
}
|
||||
if self.selected_proxy != '':
|
||||
ydl_opts['proxy'] = self.selected_proxy
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
try:
|
||||
raw_video_info = ydl.extract_info(uri, download=False)
|
||||
except yt_dlp.utils.DownloadError as error:
|
||||
self._log.debug(
|
||||
"Skipping video. Error: %s",
|
||||
error
|
||||
)
|
||||
if 'The uploader has not made this video available in your country' in f"{error}":
|
||||
self.summary['videos_with_error'] += 1
|
||||
elif 'HTTP Error 403: Forbidden' in f"{error}":
|
||||
self.summary['videos_with_error'] += 1
|
||||
self.summary['possible_proxy_errors'] += 1
|
||||
if len(self.config['proxy']) > 0:
|
||||
if self.proxy_index<len(self.config['proxy']):
|
||||
self.selected_proxy = self.config['proxy'][self.proxy_index]
|
||||
self.proxy_index += 1
|
||||
return self._get_video_info(video_id)
|
||||
self.proxy_index = 0
|
||||
self.selected_proxy = ''
|
||||
return None
|
||||
elif 'not a bot' in f"{error}":
|
||||
self.summary['videos_with_error'] += 1
|
||||
self.summary['possible_ban_errors'] += 1
|
||||
# elif age_limit_errors?
|
||||
elif 'This live event will begin in a few moments' in f"{error}":
|
||||
self.summary['skipped_videos'] += 1
|
||||
else:
|
||||
self.summary['skipped_videos'] += 1
|
||||
return None
|
||||
video_info = ydl.sanitize_info(raw_video_info)
|
||||
return video_info
|
||||
|
||||
def _process_channels(self):
|
||||
self.total_count = 0
|
||||
self.channels_count = 0
|
||||
|
|
@ -99,74 +145,60 @@ class GetYoutubeVideos:
|
|||
if result:
|
||||
video_id=result.group(1)
|
||||
if video_id not in self.downloaded_items:
|
||||
# print(json.dumps(entry, indent=2))
|
||||
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||
ydl_opts = {
|
||||
'logger': self._log,
|
||||
'progress_hooks': [self._yt_progress_hook],
|
||||
}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
try:
|
||||
raw_video_info = ydl.extract_info(uri, download=False)
|
||||
except yt_dlp.utils.DownloadError as error:
|
||||
self._log.debug(
|
||||
"%s Skipping video.",
|
||||
error
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
break
|
||||
video_info = ydl.sanitize_info(raw_video_info)
|
||||
info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
|
||||
video_info = self._get_video_info(video_id)
|
||||
info_filename = os.path.join(
|
||||
self.config['download_dir'],
|
||||
f"{video_id}.json"
|
||||
)
|
||||
self._log.debug(
|
||||
"Writting information in to file '%s'",
|
||||
info_filename
|
||||
)
|
||||
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
||||
json.dump(video_info, info_file, indent=2)
|
||||
if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
|
||||
self._log.debug(
|
||||
"Writting information in to file '%s'",
|
||||
info_filename
|
||||
"Skipping video '%s' as it's a live video",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
||||
json.dump(video_info, info_file, indent=2)
|
||||
if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as it's a live video",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self.downloaded_items.append(video_id)
|
||||
with open(
|
||||
self.config['downloaded_database'], 'w', encoding='utf-8'
|
||||
) as db_file:
|
||||
for item in self.downloaded_items:
|
||||
db_file.write(f"{item}\n")
|
||||
break
|
||||
if video_info['was_live']:
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as it was a live video",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
if ('duration' in video_info and
|
||||
video_info['duration'] > self.config['max_length']):
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as it was larger than %s",
|
||||
video_info.get('title', '?'),
|
||||
self._human_time_duration(self.config['max_length'])
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
if 'duration' not in video_info:
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as there is no video duration",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
self.summary['skipped_videos'] += 1
|
||||
self.downloaded_items.append(video_id)
|
||||
with open(
|
||||
self.config['downloaded_database'], 'w', encoding='utf-8'
|
||||
) as db_file:
|
||||
for item in self.downloaded_items:
|
||||
db_file.write(f"{item}\n")
|
||||
break
|
||||
if video_info['was_live']:
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as it was a live video",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
if ('duration' in video_info and
|
||||
video_info['duration'] > self.config['max_length']):
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as it was larger than %s",
|
||||
video_info.get('title', '?'),
|
||||
self._human_time_duration(self.config['max_length'])
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
if 'duration' not in video_info:
|
||||
self._log.debug(
|
||||
"Skipping video '%s' as there is no video duration",
|
||||
video_info.get('title', '?')
|
||||
)
|
||||
self.summary['skipped_videos'] += 1
|
||||
self._save_downloaded_items(video_id)
|
||||
break
|
||||
self._log.info(
|
||||
"Downloading. Filename: '%s'. Video ID: '%s'. Video URL: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
|
||||
"Downloading. Filename: '%s'. Video ID: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
|
||||
video_info.get('title', '?'),
|
||||
video_id,
|
||||
uri,
|
||||
self._human_time_duration(video_info.get('duration', '-1')),
|
||||
self.total_count+1,
|
||||
self.config['total_limit'],
|
||||
|
|
@ -181,8 +213,6 @@ class GetYoutubeVideos:
|
|||
else:
|
||||
download_dir = self.config['download_dir']
|
||||
ydl_opts = {
|
||||
'logger': self._log,
|
||||
'progress_hooks': [self._yt_progress_hook],
|
||||
'paths': {
|
||||
'temp': '/tmp',
|
||||
'home': download_dir
|
||||
|
|
@ -200,8 +230,11 @@ class GetYoutubeVideos:
|
|||
'merge_output_format': 'mkv',
|
||||
'format': 'bestvideo+bestaudio[ext=m4a]/best',
|
||||
}
|
||||
if self.selected_proxy != '':
|
||||
ydl_opts['proxy'] = self.selected_proxy
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
try:
|
||||
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||
return_code = ydl.download(uri)
|
||||
self._process_download(
|
||||
{
|
||||
|
|
@ -349,6 +382,11 @@ class GetYoutubeVideos:
|
|||
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
||||
# @click.option("--dummy","-n", is_flag=True,
|
||||
# help="Don't do anything, just show what would be done.")
|
||||
@click.option(
|
||||
'--proxy', '-p',
|
||||
multiple=True,
|
||||
help='URLs for proxies to use in case of errors.'
|
||||
)
|
||||
@click.option(
|
||||
'--downloaded-database', '-d',
|
||||
default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",
|
||||
|
|
|
|||
Loading…
Reference in a new issue