add req
This commit is contained in:
parent
d19517d24c
commit
63bab306e9
2 changed files with 241 additions and 1 deletions
|
|
@ -7,15 +7,28 @@
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
from logging.handlers import SysLogHandler
|
from logging.handlers import SysLogHandler
|
||||||
import click
|
import click
|
||||||
import click_config_file
|
import click_config_file
|
||||||
|
import requests
|
||||||
|
import feedparser
|
||||||
|
import yt_dlp
|
||||||
|
|
||||||
|
|
||||||
class GetYoutubeVideos:
|
class GetYoutubeVideos:
|
||||||
|
'''Get YouTube videos from a series of channels'''
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
|
self.time_duration_units = (
|
||||||
|
('week', 60*60*24*7),
|
||||||
|
('day', 60*60*24),
|
||||||
|
('hour', 60*60),
|
||||||
|
('min', 60),
|
||||||
|
('sec', 1)
|
||||||
|
)
|
||||||
self.config = kwargs
|
self.config = kwargs
|
||||||
if 'log_file' not in kwargs or kwargs['log_file'] is None:
|
if 'log_file' not in kwargs or kwargs['log_file'] is None:
|
||||||
self.config['log_file'] = os.path.join(
|
self.config['log_file'] = os.path.join(
|
||||||
|
|
@ -30,6 +43,186 @@ class GetYoutubeVideos:
|
||||||
'get_youtube_videos.log'
|
'get_youtube_videos.log'
|
||||||
)
|
)
|
||||||
self._init_log()
|
self._init_log()
|
||||||
|
if os.path.exists(self.config['downloaded_database']):
|
||||||
|
with open(self.config['downloaded_database'], 'r', encoding='utf-8') as db_file:
|
||||||
|
self.downloaded_items = db_file.read().split('\n')
|
||||||
|
else:
|
||||||
|
self.downloaded_items = []
|
||||||
|
self.session = requests.Session()
|
||||||
|
self._process_channels()
|
||||||
|
|
||||||
|
def _process_channels(self):
|
||||||
|
self.total_count = 0
|
||||||
|
self.channels_count = 0
|
||||||
|
for channel in self.config['channels']:
|
||||||
|
self.channels_count += 1
|
||||||
|
self._log.debug(
|
||||||
|
"Processing channel %s/%s '%s'...",
|
||||||
|
self.channels_count,
|
||||||
|
len(self.config['channels']),
|
||||||
|
channel
|
||||||
|
)
|
||||||
|
feed = feedparser.parse(
|
||||||
|
f"https://www.youtube.com/feeds/videos.xml?channel_id={channel}"
|
||||||
|
)
|
||||||
|
self.channel_count = 0
|
||||||
|
self.entries_count = 0
|
||||||
|
for entry in feed['entries']:
|
||||||
|
self.entries_count += 1
|
||||||
|
self._log.debug(
|
||||||
|
"Processing video entry %s/%s...",
|
||||||
|
self.entries_count,
|
||||||
|
len(feed['entries'])
|
||||||
|
)
|
||||||
|
result=re.search('v=([0-9a-zA-Z-_]{11})',entry['link'])
|
||||||
|
if result:
|
||||||
|
video_id=result.group(1)
|
||||||
|
if video_id not in self.downloaded_items:
|
||||||
|
# print(json.dumps(entry, indent=2))
|
||||||
|
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||||
|
ydl_opts = {
|
||||||
|
'logger': self._log,
|
||||||
|
'progress_hooks': [self._yt_progress_hook],
|
||||||
|
}
|
||||||
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
|
try:
|
||||||
|
raw_video_info = ydl.extract_info(uri, download=False)
|
||||||
|
except yt_dlp.utils.DownloadError as error:
|
||||||
|
self._log.debug(
|
||||||
|
"%s Skipping video.",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
break
|
||||||
|
video_info = ydl.sanitize_info(raw_video_info)
|
||||||
|
info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
|
||||||
|
self._log.debug(
|
||||||
|
"Writting information in to file '%s'",
|
||||||
|
info_filename
|
||||||
|
)
|
||||||
|
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
||||||
|
json.dump(video_info, info_file, indent=2)
|
||||||
|
if video_info['live_status'] == 'is_live':
|
||||||
|
self._log.debug(
|
||||||
|
"Skipping video '%s' as it's a live video",
|
||||||
|
video_info.get('title', '?')
|
||||||
|
)
|
||||||
|
self.downloaded_items.append(video_id)
|
||||||
|
with open(
|
||||||
|
self.config['downloaded_database'], 'w', encoding='utf-8'
|
||||||
|
) as db_file:
|
||||||
|
for item in self.downloaded_items:
|
||||||
|
db_file.write(f"{item}\n")
|
||||||
|
break
|
||||||
|
if video_info['was_live']:
|
||||||
|
self._log.debug(
|
||||||
|
"Skipping video '%s' as it was a live video",
|
||||||
|
video_info.get('title', '?')
|
||||||
|
)
|
||||||
|
self._save_downloaded_items(video_id)
|
||||||
|
break
|
||||||
|
if ('duration' in video_info and
|
||||||
|
video_info['duration'] > self.config['max_length']):
|
||||||
|
self._log.debug(
|
||||||
|
"Skipping video '%s' as it was larger than %s",
|
||||||
|
video_info.get('title', '?'),
|
||||||
|
self._human_time_duration(self.config['max_length'])
|
||||||
|
)
|
||||||
|
self._save_downloaded_items(video_id)
|
||||||
|
break
|
||||||
|
if 'duration' not in video_info:
|
||||||
|
self._log.debug(
|
||||||
|
"Skipping video '%s' as there is no video duration",
|
||||||
|
video_info.get('title', '?')
|
||||||
|
)
|
||||||
|
self._save_downloaded_items(video_id)
|
||||||
|
break
|
||||||
|
self._log.info(
|
||||||
|
"Downloading. Filename: '%s'. Video ID: '%s'. Video URL: '%s'. Duration: %s. Counts: %s/%s - %s/%s)",
|
||||||
|
video_info.get('title', '?'),
|
||||||
|
video_id,
|
||||||
|
uri,
|
||||||
|
self._human_time_duration(video_info.get('duration', '-1')),
|
||||||
|
self.total_count+1,
|
||||||
|
self.config['total_limit'],
|
||||||
|
self.channel_count+1,
|
||||||
|
self.config['channel_limit']
|
||||||
|
)
|
||||||
|
ydl_opts = {
|
||||||
|
'logger': self._log,
|
||||||
|
'progress_hooks': [self._yt_progress_hook],
|
||||||
|
'paths': {
|
||||||
|
'temp': '/tmp',
|
||||||
|
'home': self.config['download_dir']
|
||||||
|
},
|
||||||
|
'writesubtitles': True,
|
||||||
|
'writeautomaticsub': True,
|
||||||
|
'writethumbnail': True,
|
||||||
|
'subtitlesformat': 'srt',
|
||||||
|
'subtitleslangs': self.config['subtitle_langs'],
|
||||||
|
'allow_multiple_audio_streams': True,
|
||||||
|
'merge_output_format': 'mp4',
|
||||||
|
'format': 'bestvideo+bestaudio[ext=m4a]/best',
|
||||||
|
}
|
||||||
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
|
try:
|
||||||
|
ydl.download(uri)
|
||||||
|
except yt_dlp.utils.DownloadError as error:
|
||||||
|
self._log.error(
|
||||||
|
"Error getting video. %s",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
self._log.error(
|
||||||
|
"Error! Video ID not found in URI '%s'",
|
||||||
|
entry['link']
|
||||||
|
)
|
||||||
|
|
||||||
|
def _yt_progress_hook(self, data):
|
||||||
|
if data['status'] == 'finished':
|
||||||
|
self.total_count += 1
|
||||||
|
self.channel_count += 1
|
||||||
|
if self.total_count == self.config['total_limit']:
|
||||||
|
self._log.info(
|
||||||
|
"Limit (%s) reached for videos from all channels",
|
||||||
|
self.config['total_limit'],
|
||||||
|
)
|
||||||
|
sys.exit(0)
|
||||||
|
if self.channel_count == self.config['channel_limit']:
|
||||||
|
self._log.info(
|
||||||
|
"Limit (%s) reached for videos for this channel '%s'",
|
||||||
|
self.config['channel_limit'],
|
||||||
|
data['info_dict'].get('channel', '?')
|
||||||
|
)
|
||||||
|
# break
|
||||||
|
info_filename = os.path.join(self.config['download_dir'], f"{os.path.basename(data['filename'])}.download_info.json")
|
||||||
|
self._log.debug(
|
||||||
|
"Writting download information in to file '%s'",
|
||||||
|
info_filename
|
||||||
|
)
|
||||||
|
with open(info_filename, 'w', encoding='utf-8') as info_file:
|
||||||
|
json.dump(data, info_file, indent=2)
|
||||||
|
if 'id' in data['info_dict']:
|
||||||
|
self._save_downloaded_items(data['info_dict']['id'])
|
||||||
|
|
||||||
|
|
||||||
|
def _save_downloaded_items(self, video_id):
|
||||||
|
self.downloaded_items.append(video_id)
|
||||||
|
with open(self.config['downloaded_database'], 'w', encoding='utf-8') as db_file:
|
||||||
|
for item in self.downloaded_items:
|
||||||
|
db_file.write(f"{item}\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _human_time_duration(self, seconds):
|
||||||
|
'''Return time duration in a human readable formated string'''
|
||||||
|
if seconds == 0:
|
||||||
|
return 'inf'
|
||||||
|
parts = []
|
||||||
|
for unit, div in self.time_duration_units:
|
||||||
|
amount, seconds = divmod(int(seconds), div)
|
||||||
|
if amount > 0:
|
||||||
|
parts.append(f'{amount} {unit}{"" if amount == 1 else "s"}')
|
||||||
|
return ', '.join(parts)
|
||||||
|
|
||||||
def _init_log(self):
|
def _init_log(self):
|
||||||
''' Initialize log object '''
|
''' Initialize log object '''
|
||||||
|
|
@ -85,6 +278,50 @@ class GetYoutubeVideos:
|
||||||
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
||||||
# @click.option("--dummy","-n", is_flag=True,
|
# @click.option("--dummy","-n", is_flag=True,
|
||||||
# help="Don't do anything, just show what would be done.")
|
# help="Don't do anything, just show what would be done.")
|
||||||
|
@click.option(
|
||||||
|
'--downloaded-database', '-d',
|
||||||
|
default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/.config/downloaded_youtube_videos",
|
||||||
|
help='File to store the IDs of downloaded videos'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--download-dir', '-f',
|
||||||
|
default=f"{os.environ.get('HOME', os.environ.get('USERPROFILE', ''))}/downloaded_youtube_videos",
|
||||||
|
help='Folder to store the downloaded videos'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--channels', '-c',
|
||||||
|
multiple=True,
|
||||||
|
required=True,
|
||||||
|
help='YouTube channels IDs to look up'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--channel-limit', '-l',
|
||||||
|
default=5,
|
||||||
|
type=int,
|
||||||
|
help='Maximun number of videos to download from a channel'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--total-limit', '-L',
|
||||||
|
default=5,
|
||||||
|
type=int,
|
||||||
|
help='Maximun number of videos to download in total'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--max-length', '-d',
|
||||||
|
default=5400,
|
||||||
|
type=int,
|
||||||
|
help='Maximun duration of videos to download in seconds'
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
'--subtitle-langs', '-s',
|
||||||
|
multiple=True,
|
||||||
|
default=['en.*'],
|
||||||
|
help='''List of languages of the subtitles to download (can be regex).
|
||||||
|
The list may contain "all" to refer to all the available
|
||||||
|
subtitles. The language can be prefixed with a "-" to
|
||||||
|
exclude it from the requested languages, e.g. ['all', '-live_chat'].
|
||||||
|
And you can use wildcards like en.*'''
|
||||||
|
)
|
||||||
@click_config_file.configuration_option()
|
@click_config_file.configuration_option()
|
||||||
def __main__(**kwargs):
|
def __main__(**kwargs):
|
||||||
return GetYoutubeVideos(**kwargs)
|
return GetYoutubeVideos(**kwargs)
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,5 @@
|
||||||
click
|
click
|
||||||
click_config_file
|
click_config_file
|
||||||
|
requests
|
||||||
|
feedparser
|
||||||
|
yt_dlp
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue