Handle errors and add proxies
This commit is contained in:
parent
89f561d043
commit
ab1bc5af92
1 changed files with 103 additions and 65 deletions
|
|
@ -54,7 +54,12 @@ class GetYoutubeVideos:
|
||||||
'skipped_videos': 0,
|
'skipped_videos': 0,
|
||||||
'downloaded_videos': 0,
|
'downloaded_videos': 0,
|
||||||
'videos_with_error': 0,
|
'videos_with_error': 0,
|
||||||
|
'possible_proxy_errors': 0,
|
||||||
|
'possible_ban_errors': 0,
|
||||||
|
'age_limit_errors': 0,
|
||||||
}
|
}
|
||||||
|
self.selected_proxy = ''
|
||||||
|
self.proxy_index = 0
|
||||||
if os.path.exists(self.config['downloaded_database']):
|
if os.path.exists(self.config['downloaded_database']):
|
||||||
with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
|
with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
|
||||||
self.downloaded_items = db_file.read().split('\n')
|
self.downloaded_items = db_file.read().split('\n')
|
||||||
|
|
@ -71,6 +76,47 @@ class GetYoutubeVideos:
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _get_video_info(self, video_id):
|
||||||
|
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||||
|
ydl_opts = {
|
||||||
|
'logger': self._log,
|
||||||
|
'progress_hooks': [self._yt_progress_hook],
|
||||||
|
}
|
||||||
|
if self.selected_proxy != '':
|
||||||
|
ydl_opts['proxy'] = self.selected_proxy
|
||||||
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
|
try:
|
||||||
|
raw_video_info = ydl.extract_info(uri, download=False)
|
||||||
|
except yt_dlp.utils.DownloadError as error:
|
||||||
|
self._log.debug(
|
||||||
|
"Skipping video. Error: %s",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
if 'The uploader has not made this video available in your country' in f"{error}":
|
||||||
|
self.summary['videos_with_error'] += 1
|
||||||
|
elif 'HTTP Error 403: Forbidden' in f"{error}":
|
||||||
|
self.summary['videos_with_error'] += 1
|
||||||
|
self.summary['possible_proxy_errors'] += 1
|
||||||
|
if len(self.config['proxy']) > 0:
|
||||||
|
if self.proxy_index<len(self.config['proxy']):
|
||||||
|
self.selected_proxy = self.config['proxy'][self.proxy_index]
|
||||||
|
self.proxy_index += 1
|
||||||
|
return self._get_video_info(video_id)
|
||||||
|
self.proxy_index = 0
|
||||||
|
self.selected_proxy = ''
|
||||||
|
return None
|
||||||
|
elif 'not a bot' in f"{error}":
|
||||||
|
self.summary['videos_with_error'] += 1
|
||||||
|
self.summary['possible_ban_errors'] += 1
|
||||||
|
# elif age_limit_errors?
|
||||||
|
elif 'This live event will begin in a few moments' in f"{error}":
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
|
else:
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
|
return None
|
||||||
|
video_info = ydl.sanitize_info(raw_video_info)
|
||||||
|
return video_info
|
||||||
|
|
||||||
def _process_channels(self):
|
def _process_channels(self):
|
||||||
self.total_count = 0
|
self.total_count = 0
|
||||||
self.channels_count = 0
|
self.channels_count = 0
|
||||||
|
|
@ -99,74 +145,60 @@ class GetYoutubeVideos:
|
||||||
if result:
|
if result:
|
||||||
video_id=result.group(1)
|
video_id=result.group(1)
|
||||||
if video_id not in self.downloaded_items:
|
if video_id not in self.downloaded_items:
|
||||||
# print(json.dumps(entry, indent=2))
|
video_info = self._get_video_info(video_id)
|
||||||
uri=f"https://www.youtube.com/watch?v={video_id}"
|
info_filename = os.path.join(
|
||||||
ydl_opts = {
|
self.config['download_dir'],
|
||||||
'logger': self._log,
|
f"{video_id}.json"
|
||||||
'progress_hooks': [self._yt_progress_hook],
|
)
|
||||||
}
|
self._log.debug(
|
||||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
"Writting information in to file '%s'",
|
||||||
try:
|
info_filename
|
||||||
raw_video_info = ydl.extract_info(uri, download=False)
|
)
|
||||||
except yt_dlp.utils.DownloadError as error:
|
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
||||||
self._log.debug(
|
json.dump(video_info, info_file, indent=2)
|
||||||
"%s Skipping video.",
|
if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
|
||||||
error
|
|
||||||
)
|
|
||||||
self.summary['skipped_videos'] += 1
|
|
||||||
break
|
|
||||||
video_info = ydl.sanitize_info(raw_video_info)
|
|
||||||
info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
|
|
||||||
self._log.debug(
|
self._log.debug(
|
||||||
"Writting information in to file '%s'",
|
"Skipping video '%s' as it's a live video",
|
||||||
info_filename
|
video_info.get('title', '?')
|
||||||
)
|
)
|
||||||
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
self.summary['skipped_videos'] += 1
|
||||||
json.dump(video_info, info_file, indent=2)
|
self.downloaded_items.append(video_id)
|
||||||
if self.config['skip_live_videos'] and video_info['live_status'] == 'is_live':
|
with open(
|
||||||
self._log.debug(
|
self.config['downloaded_database'], 'w', encoding='utf-8'
|
||||||
"Skipping video '%s' as it's a live video",
|
) as db_file:
|
||||||
video_info.get('title', '?')
|
for item in self.downloaded_items:
|
||||||
)
|
db_file.write(f"{item}\n")
|
||||||
self.summary['skipped_videos'] += 1
|
break
|
||||||
self.downloaded_items.append(video_id)
|
if video_info['was_live']:
|
||||||
with open(
|
self._log.debug(
|
||||||
self.config['downloaded_database'], 'w', encoding='utf-8'
|
"Skipping video '%s' as it was a live video",
|
||||||
) as db_file:
|
video_info.get('title', '?')
|
||||||
for item in self.downloaded_items:
|
)
|
||||||
db_file.write(f"{item}\n")
|
self.summary['skipped_videos'] += 1
|
||||||
break
|
self._save_downloaded_items(video_id)
|
||||||
if video_info['was_live']:
|
break
|
||||||
self._log.debug(
|
if ('duration' in video_info and
|
||||||
"Skipping video '%s' as it was a live video",
|
video_info['duration'] > self.config['max_length']):
|
||||||
video_info.get('title', '?')
|
self._log.debug(
|
||||||
)
|
"Skipping video '%s' as it was larger than %s",
|
||||||
self.summary['skipped_videos'] += 1
|
video_info.get('title', '?'),
|
||||||
self._save_downloaded_items(video_id)
|
self._human_time_duration(self.config['max_length'])
|
||||||
break
|
)
|
||||||
if ('duration' in video_info and
|
self.summary['skipped_videos'] += 1
|
||||||
video_info['duration'] > self.config['max_length']):
|
self._save_downloaded_items(video_id)
|
||||||
self._log.debug(
|
break
|
||||||
"Skipping video '%s' as it was larger than %s",
|
if 'duration' not in video_info:
|
||||||
video_info.get('title', '?'),
|
self._log.debug(
|
||||||
self._human_time_duration(self.config['max_length'])
|
"Skipping video '%s' as there is no video duration",
|
||||||
)
|
video_info.get('title', '?')
|
||||||
self.summary['skipped_videos'] += 1
|
)
|
||||||
self._save_downloaded_items(video_id)
|
self.summary['skipped_videos'] += 1
|
||||||
break
|
self._save_downloaded_items(video_id)
|
||||||
if 'duration' not in video_info:
|
break
|
||||||
self._log.debug(
|
|
||||||
"Skipping video '%s' as there is no video duration",
|
|
||||||
video_info.get('title', '?')
|
|
||||||
)
|
|
||||||
self.summary['skipped_videos'] += 1
|
|
||||||
self._save_downloaded_items(video_id)
|
|
||||||
break
|
|
||||||
self._log.info(
|
self._log.info(
|
||||||
"Downloading. Filename: '%s'. Video ID: '%s'. Video URL: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
|
"Downloading. Filename: '%s'. Video ID: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
|
||||||
video_info.get('title', '?'),
|
video_info.get('title', '?'),
|
||||||
video_id,
|
video_id,
|
||||||
uri,
|
|
||||||
self._human_time_duration(video_info.get('duration', '-1')),
|
self._human_time_duration(video_info.get('duration', '-1')),
|
||||||
self.total_count+1,
|
self.total_count+1,
|
||||||
self.config['total_limit'],
|
self.config['total_limit'],
|
||||||
|
|
@ -181,8 +213,6 @@ class GetYoutubeVideos:
|
||||||
else:
|
else:
|
||||||
download_dir = self.config['download_dir']
|
download_dir = self.config['download_dir']
|
||||||
ydl_opts = {
|
ydl_opts = {
|
||||||
'logger': self._log,
|
|
||||||
'progress_hooks': [self._yt_progress_hook],
|
|
||||||
'paths': {
|
'paths': {
|
||||||
'temp': '/tmp',
|
'temp': '/tmp',
|
||||||
'home': download_dir
|
'home': download_dir
|
||||||
|
|
@ -200,8 +230,11 @@ class GetYoutubeVideos:
|
||||||
'merge_output_format': 'mkv',
|
'merge_output_format': 'mkv',
|
||||||
'format': 'bestvideo+bestaudio[ext=m4a]/best',
|
'format': 'bestvideo+bestaudio[ext=m4a]/best',
|
||||||
}
|
}
|
||||||
|
if self.selected_proxy != '':
|
||||||
|
ydl_opts['proxy'] = self.selected_proxy
|
||||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
try:
|
try:
|
||||||
|
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||||
return_code = ydl.download(uri)
|
return_code = ydl.download(uri)
|
||||||
self._process_download(
|
self._process_download(
|
||||||
{
|
{
|
||||||
|
|
@ -349,6 +382,11 @@ class GetYoutubeVideos:
|
||||||
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
||||||
# @click.option("--dummy","-n", is_flag=True,
|
# @click.option("--dummy","-n", is_flag=True,
|
||||||
# help="Don't do anything, just show what would be done.")
|
# help="Don't do anything, just show what would be done.")
|
||||||
|
@click.option(
|
||||||
|
'--proxy', '-p',
|
||||||
|
multiple=True,
|
||||||
|
help='URLs for proxies to use in case of errors.'
|
||||||
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
'--downloaded-database', '-d',
|
'--downloaded-database', '-d',
|
||||||
default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",
|
default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue