Add summary
This commit is contained in:
parent
b106800f6c
commit
2f6e70ec32
1 changed files with 31 additions and 0 deletions
|
|
@ -9,6 +9,11 @@ import sys
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
|
from yaml import dump
|
||||||
|
try:
|
||||||
|
from yaml import CDumper as Dumper
|
||||||
|
except ImportError:
|
||||||
|
from yaml import Dumper
|
||||||
import logging
|
import logging
|
||||||
from logging.handlers import SysLogHandler
|
from logging.handlers import SysLogHandler
|
||||||
import click
|
import click
|
||||||
|
|
@ -29,6 +34,12 @@ class GetYoutubeVideos:
|
||||||
('min', 60),
|
('min', 60),
|
||||||
('sec', 1)
|
('sec', 1)
|
||||||
)
|
)
|
||||||
|
self.summary = {
|
||||||
|
'entries_count': 0,
|
||||||
|
'skipped_videos': 0,
|
||||||
|
'downloaded_videos': 0,
|
||||||
|
'error_videos': 0,
|
||||||
|
}
|
||||||
self.config = kwargs
|
self.config = kwargs
|
||||||
if 'log_file' not in kwargs or kwargs['log_file'] is None:
|
if 'log_file' not in kwargs or kwargs['log_file'] is None:
|
||||||
self.config['log_file'] = os.path.join(
|
self.config['log_file'] = os.path.join(
|
||||||
|
|
@ -50,10 +61,19 @@ class GetYoutubeVideos:
|
||||||
self.downloaded_items = []
|
self.downloaded_items = []
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
self._process_channels()
|
self._process_channels()
|
||||||
|
self._log.info(
|
||||||
|
dump(
|
||||||
|
{
|
||||||
|
"Summary": self.summary
|
||||||
|
},
|
||||||
|
Dumper=Dumper
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def _process_channels(self):
|
def _process_channels(self):
|
||||||
self.total_count = 0
|
self.total_count = 0
|
||||||
self.channels_count = 0
|
self.channels_count = 0
|
||||||
|
self.summary['total_channels'] = len(self.config['channels'])
|
||||||
for channel in self.config['channels']:
|
for channel in self.config['channels']:
|
||||||
self.channels_count += 1
|
self.channels_count += 1
|
||||||
self._log.debug(
|
self._log.debug(
|
||||||
|
|
@ -92,6 +112,7 @@ class GetYoutubeVideos:
|
||||||
"%s Skipping video.",
|
"%s Skipping video.",
|
||||||
error
|
error
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
break
|
break
|
||||||
video_info = ydl.sanitize_info(raw_video_info)
|
video_info = ydl.sanitize_info(raw_video_info)
|
||||||
info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
|
info_filename = os.path.join(self.config['download_dir'], f"{video_id}.json")
|
||||||
|
|
@ -106,6 +127,7 @@ class GetYoutubeVideos:
|
||||||
"Skipping video '%s' as it's a live video",
|
"Skipping video '%s' as it's a live video",
|
||||||
video_info.get('title', '?')
|
video_info.get('title', '?')
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
self.downloaded_items.append(video_id)
|
self.downloaded_items.append(video_id)
|
||||||
with open(
|
with open(
|
||||||
self.config['downloaded_database'], 'w', encoding='utf-8'
|
self.config['downloaded_database'], 'w', encoding='utf-8'
|
||||||
|
|
@ -118,6 +140,7 @@ class GetYoutubeVideos:
|
||||||
"Skipping video '%s' as it was a live video",
|
"Skipping video '%s' as it was a live video",
|
||||||
video_info.get('title', '?')
|
video_info.get('title', '?')
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
self._save_downloaded_items(video_id)
|
self._save_downloaded_items(video_id)
|
||||||
break
|
break
|
||||||
if ('duration' in video_info and
|
if ('duration' in video_info and
|
||||||
|
|
@ -127,6 +150,7 @@ class GetYoutubeVideos:
|
||||||
video_info.get('title', '?'),
|
video_info.get('title', '?'),
|
||||||
self._human_time_duration(self.config['max_length'])
|
self._human_time_duration(self.config['max_length'])
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
self._save_downloaded_items(video_id)
|
self._save_downloaded_items(video_id)
|
||||||
break
|
break
|
||||||
if 'duration' not in video_info:
|
if 'duration' not in video_info:
|
||||||
|
|
@ -134,6 +158,7 @@ class GetYoutubeVideos:
|
||||||
"Skipping video '%s' as there is no video duration",
|
"Skipping video '%s' as there is no video duration",
|
||||||
video_info.get('title', '?')
|
video_info.get('title', '?')
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
self._save_downloaded_items(video_id)
|
self._save_downloaded_items(video_id)
|
||||||
break
|
break
|
||||||
self._log.info(
|
self._log.info(
|
||||||
|
|
@ -182,17 +207,23 @@ class GetYoutubeVideos:
|
||||||
"Error getting video. %s",
|
"Error getting video. %s",
|
||||||
error
|
error
|
||||||
)
|
)
|
||||||
|
self.summary['error_videos'] += 1
|
||||||
break
|
break
|
||||||
|
self.summary['downloaded_videos'] += 1
|
||||||
else:
|
else:
|
||||||
self._log.debug(
|
self._log.debug(
|
||||||
"Video with ID '%s' has been already downloaded",
|
"Video with ID '%s' has been already downloaded",
|
||||||
video_id
|
video_id
|
||||||
)
|
)
|
||||||
|
self.summary['skipped_videos'] += 1
|
||||||
else:
|
else:
|
||||||
self._log.error(
|
self._log.error(
|
||||||
"Error! Video ID not found in URI '%s'",
|
"Error! Video ID not found in URI '%s'",
|
||||||
entry['link']
|
entry['link']
|
||||||
)
|
)
|
||||||
|
self.summary['error_videos'] += 1
|
||||||
|
self.summary['entries_count'] += self.entries_count
|
||||||
|
self.summary['processed_channels'] = self.channels_count
|
||||||
|
|
||||||
def _yt_progress_hook(self, data):
|
def _yt_progress_hook(self, data):
|
||||||
if data['status'] == 'finished':
|
if data['status'] == 'finished':
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue