Compare commits
No commits in common. "d9fd4400cd3040d38d90286f9efb5020ee046331" and "fffcc743af6f392e89aef0aba3d18086860ca46a" have entirely different histories.
d9fd4400cd
...
fffcc743af
1 changed files with 23 additions and 107 deletions
|
|
@ -8,12 +8,10 @@
|
|||
import sys
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import stat
|
||||
import json
|
||||
import logging
|
||||
from logging.handlers import SysLogHandler
|
||||
import yaml
|
||||
from yaml import dump
|
||||
try:
|
||||
from yaml import CDumper as Dumper
|
||||
|
|
@ -25,14 +23,6 @@ import requests
|
|||
import feedparser
|
||||
import yt_dlp
|
||||
|
||||
HOME_FOLDER = os.environ.get('HOME', os.environ.get('USERPROFILE', '/'))
|
||||
if HOME_FOLDER == '/':
|
||||
CACHE_FOLDER = '/var/cache'
|
||||
LOG_FOLDER = '/var/log/'
|
||||
else:
|
||||
CACHE_FOLDER = f"{HOME_FOLDER}/.local/"
|
||||
LOG_FOLDER = f"{HOME_FOLDER}/log/"
|
||||
|
||||
|
||||
class GetYoutubeVideos:
|
||||
'''Get YouTube videos from a series of channels'''
|
||||
|
|
@ -59,13 +49,6 @@ class GetYoutubeVideos:
|
|||
'get_youtube_videos.log'
|
||||
)
|
||||
self._init_log()
|
||||
|
||||
self._default_data = {
|
||||
"last_update": 0,
|
||||
"proxies": {},
|
||||
}
|
||||
self.data = self._read_cached_data()
|
||||
|
||||
self.summary = {
|
||||
'config': self.config,
|
||||
'entries_count': 0,
|
||||
|
|
@ -79,16 +62,9 @@ class GetYoutubeVideos:
|
|||
'downloaded_videos_titles': [],
|
||||
}
|
||||
if len(self.config['proxy']) > 0:
|
||||
for proxy in self.config['proxy']:
|
||||
if proxy not in self._default_data['proxies']:
|
||||
self._default_data['proxies'][proxy]['failures'] = 0
|
||||
less_failures = -1
|
||||
for proxy in self._default_data['proxies']:
|
||||
if less_failures == -1 or self._default_data['proxies'][proxy]['failures'] < less_failures:
|
||||
less_failures = self._default_data['proxies'][proxy]['failures']
|
||||
self.selected_proxy = proxy
|
||||
self.selected_proxy = self.config['proxy'][0]
|
||||
self.proxies = self.config['proxy']
|
||||
else:
|
||||
self._default_data['proxies'] = {}
|
||||
self.selected_proxy = ''
|
||||
self.proxy_index = 0
|
||||
if os.path.exists(self.config['downloaded_database']):
|
||||
|
|
@ -108,60 +84,6 @@ class GetYoutubeVideos:
|
|||
)
|
||||
self._save_metrics(self.summary)
|
||||
|
||||
def close(self):
|
||||
'''Close class and save data'''
|
||||
self._save_cached_data(self.data)
|
||||
|
||||
def _read_cached_data(self):
|
||||
if os.path.exists(self.config['cache_file']):
|
||||
with open(self.config['cache_file'], 'r', encoding='utf-8') as cache_file:
|
||||
try:
|
||||
cached_data = json.load(cache_file)
|
||||
if (
|
||||
'last_update' in cached_data and
|
||||
cached_data['last_update'] + self.config['max_cache_age'] > time.time()
|
||||
):
|
||||
cached_data = self._default_data
|
||||
except json.decoder.JSONDecodeError:
|
||||
cached_data = self._default_data
|
||||
return cached_data
|
||||
else:
|
||||
return self._default_data
|
||||
|
||||
def _save_cached_data(self, data):
|
||||
data['last_update'] = time.time()
|
||||
with open(self.config['cache_file'], 'w', encoding='utf-8') as cache_file:
|
||||
json.dump(data, cache_file, indent=2)
|
||||
self._debug(
|
||||
f"Saved cached data in '{self.config['cache_file']}'",
|
||||
)
|
||||
|
||||
def _output(self, message):
|
||||
if self.config['output_format'] == 'JSON':
|
||||
return json.dumps(message, indent=2)
|
||||
elif self.config['output_format'] == 'YAML':
|
||||
return yaml.dump(message, Dumper=yaml.Dumper)
|
||||
elif self.config['output_format'] == 'PLAIN':
|
||||
return f"{message}"
|
||||
else:
|
||||
self._log.warning(
|
||||
"Output format '%s' not supported",
|
||||
self.config['output_format']
|
||||
)
|
||||
return message
|
||||
|
||||
def _info(self, message):
|
||||
return self._log.info(self._output(message))
|
||||
|
||||
def _warning(self, message):
|
||||
return self._log.warning(self._output(message))
|
||||
|
||||
def _error(self, message):
|
||||
return self._log.error(self._output(message))
|
||||
|
||||
def _debug(self, message):
|
||||
return self._log.debug(self._output(message))
|
||||
|
||||
def _is_numeric(self, variable):
|
||||
if isinstance(variable, (int, float, complex)):
|
||||
return True
|
||||
|
|
@ -186,21 +108,28 @@ class GetYoutubeVideos:
|
|||
os.chmod('/var/lib/prometheus/node-exporter/get_youtube_videos.prom', stat.S_IROTH)
|
||||
|
||||
def _change_proxy(self, video_id):
|
||||
less_failures = -1
|
||||
for proxy in self._default_data['proxies']:
|
||||
if less_failures == -1 or self._default_data['proxies'][proxy]['failures'] < less_failures:
|
||||
less_failures = self._default_data['proxies'][proxy['failures']]
|
||||
self.selected_proxy = proxy
|
||||
if less_failures == -1:
|
||||
self.selected_proxy = list(self._default_data['proxies'].keys())[0]
|
||||
if len(self.proxies) > 0:
|
||||
if isinstance(self.proxies, list):
|
||||
self._log.debug(
|
||||
"Removing failing proxy '%s'",
|
||||
self.selected_proxy
|
||||
)
|
||||
self.proxies.pop(self.proxy_index)
|
||||
if self.proxy_index<len(self.proxies)-1:
|
||||
self.selected_proxy = self.proxies[self.proxy_index]
|
||||
self.proxy_index += 1
|
||||
self._log.warning(
|
||||
"Got an error fetching video information. Setting proxy to '%s'...",
|
||||
self.selected_proxy
|
||||
)
|
||||
return self._get_video_info(video_id)
|
||||
self._log.debug(
|
||||
"All proxies have the same number of errors, using first proxy of the list."
|
||||
"No more proxies to try. Resetting list."
|
||||
)
|
||||
self._log.warning(
|
||||
"Got an error fetching video information. Setting proxy to '%s'...",
|
||||
self.selected_proxy
|
||||
)
|
||||
return self._get_video_info(video_id)
|
||||
self.proxy_index = 0
|
||||
self.proxies = self.config['proxy']
|
||||
self.selected_proxy = self.proxies[0]
|
||||
return None
|
||||
|
||||
def _get_video_info(self, video_id):
|
||||
uri=f"https://www.youtube.com/watch?v={video_id}"
|
||||
|
|
@ -624,22 +553,9 @@ class GetYoutubeVideos:
|
|||
default=False,
|
||||
help='Skip live videos'
|
||||
)
|
||||
@click.option(
|
||||
'--cache-file',
|
||||
'-f',
|
||||
default=f"{CACHE_FOLDER}/get_youtube_videos.json",
|
||||
help='Cache file to store data from each run',
|
||||
)
|
||||
@click.option(
|
||||
'--max-cache-age',
|
||||
'-a',
|
||||
default=60*60*24*7,
|
||||
help='Max age in seconds for the cache'
|
||||
)
|
||||
@click_config_file.configuration_option()
|
||||
def __main__(**kwargs):
|
||||
obj = GetYoutubeVideos(**kwargs)
|
||||
obj.close()
|
||||
return GetYoutubeVideos(**kwargs)
|
||||
|
||||
if __name__ == "__main__":
|
||||
__main__()
|
||||
|
|
|
|||
Loading…
Reference in a new issue