From bffdd74e47a6ef157e28b1cfc9debbc84685e924 Mon Sep 17 00:00:00 2001 From: "Antonio J. Delgado" Date: Sun, 3 Nov 2024 14:29:14 +0200 Subject: [PATCH] Add option for reading the journal --- requirements.txt | 3 +- restic_exporter/restic_exporter.py | 154 ++++++++++++++++++++--------- 2 files changed, 107 insertions(+), 50 deletions(-) diff --git a/requirements.txt b/requirements.txt index 66bf966..32bcd31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ click -click_config_file \ No newline at end of file +click_config_file +systemd diff --git a/restic_exporter/restic_exporter.py b/restic_exporter/restic_exporter.py index f5ddff1..d42a180 100644 --- a/restic_exporter/restic_exporter.py +++ b/restic_exporter/restic_exporter.py @@ -9,68 +9,102 @@ import sys import os import logging import json -import click -import click_config_file from logging.handlers import SysLogHandler import time +from datetime import datetime, timedelta +import click +import click_config_file +from systemd import journal -class restic_exporter: +class ResticExporter: + '''Exporter of Restic data to Prometheus''' - def __init__(self, debug_level, log_file, json_file, job_name, extra_labels, metric_name, metric_description): + def __init__(self, **kwargs): ''' Initial function called when object is created ''' - self.config = dict() - self.config['debug_level'] = debug_level - if log_file is None: - log_file = os.path.join(os.environ.get('HOME', os.environ.get('USERPROFILE', os.getcwd())), 'log', 'restic_exporter.log') - self.config['log_file'] = log_file + self.config = kwargs + if 'log_file' not in kwargs or kwargs['log_file'] is None: + self.config['log_file'] = os.path.join( + os.environ.get( + 'HOME', + os.environ.get( + 'USERPROFILE', + os.getcwd() + ) + ), + 'log', + '__project_codename__.log' + ) self._init_log() - self.metric_name = metric_name.replace(' ', '_') - self.job_name = job_name.replace(' ', '_') - self.metric_description = metric_description - self.labels= {"job_name": job_name} + self.metric_name = self.config['metric_name'].replace(' ', '_') + self.job_name = self.config['job_name'].replace(' ', '_') + self.metric_description = self.config['metric_description'] + self.labels= {"job_name": self.config['job_name']} - if extra_labels: + if self.config['extra_labels']: self.labels = { **self.labels, - **self._read_extra_labels(extra_labels) + **self._read_extra_labels(self.config['extra_labels']) } - - self._read_summary_from_json(json_file) + if self.config['systemd']: + self._read_summary_from_json(self.config['json_file']) + else: + self._read_summary_from_systemd() self._show_metrics() + def _read_summary_from_systemd(self,): + self.summaries = () + journal_obj = journal.Reader() + journal_obj.seek_realtime(datetime.now() - timedelta(days=1)) + journal_obj.add_match('_EXE=/usr/local/bin/restic') + for entry in journal_obj: + if '"message_type":"summary"' in entry['MESSAGE']: + summary = json.loads(entry['MESSAGE']) + summary['metric_name'] = 'restic_summary' + summary['job_name'] = entry['_SYSTEMD_UNIT'].replace( + 'remote_backupjob_', + '').replace('.service', '') + self.summaries.append(summary) + def _show_metrics(self): - labels = self._convert_labels(self.labels) - counters = [ - 'files_new', - 'files_changed', - 'files_unmodified', - 'dirs_new', - 'dirs_changed', - 'dirs_unmodified', - 'data_blobs', - 'tree_blobs', - 'data_added', - 'total_files_processed', - 'total_bytes_processed', - 'total_duration' - ] - print(f"# HELP {self.metric_name}_{self.job_name} {self.metric_description}.") - print(f"# TYPE {self.metric_name}_{self.job_name} counter") - for counter in counters: - if counter in self.summary: - print(f"{self.metric_name}_{counter}{labels} {float(self.summary[counter])}") # {self.summary['timestamp']}") + for summary in self.summaries: + labels = self._convert_labels(self.labels) + counters = [ + 'files_new', + 'files_changed', + 'files_unmodified', + 'dirs_new', + 'dirs_changed', + 'dirs_unmodified', + 'data_blobs', + 'tree_blobs', + 'data_added', + 'total_files_processed', + 'total_bytes_processed', + 'total_duration' + ] + print( + f"# HELP {summary['metric_name']}_{summary['job_name']} {self.metric_description}." + ) + print(f"# TYPE {summary['metric_name']}_{summary['job_name']} counter") + for counter in counters: + if counter in summary: + print( + f"{summary['metric_name']}_{counter}{labels} {float(summary[counter])}" + ) def _read_summary_from_json(self, json_file): try: - with open(json_file, 'r') as file_pointer: + with open(json_file, 'r', encoding='utf-8') as file_pointer: content = file_pointer.readlines() except Exception as error: self._log.error(f"# Error reading file '{json_file}'. Check permissions. {error}") - self.summary = { - "timestamp": time.time() + summary = { + "timestamp": time.time(), + "metric_name": self.metric_name, + "job_name": self.job_name, } for line in content: try: @@ -84,9 +118,10 @@ class restic_exporter: fixed_line = line.replace('\n', '') self._log.error(f"# Error decoding line '{fixed_line}'. {error}") file_stats = os.stat(json_file) - self.summary['timestamp'] = round(file_stats.st_mtime * 1000) - self._log.debug(f"# Summary: {json.dumps(self.summary, indent=2)}") + summary['timestamp'] = round(file_stats.st_mtime * 1000) + self._log.debug(f"# Summary: {json.dumps(summary, indent=2)}") self._log.debug(f"# Labels: {self.labels}") + self.summaries = [ summary ] def _read_extra_labels(self, extra_labels): labels_ls = {} @@ -146,14 +181,35 @@ class restic_exporter: ), help='Set the debug level for the standard output.') @click.option('--log-file', '-l', help="File to store all debug messages.") @click.option("--json-file", "-j", required=True, help='JSON file containing the output of restic') -@click.option('--job-name', '-n', required=True, help='Restic job name to attach to the exported metrics') -@click.option('--extra-labels', '-a', required=False, default=None, help='Pairs key=value separated by commas with extra labels to add to the summary') -@click.option('--metric-name', '-m', default='restic_summary', help='Metric name. Spaces will be replaced with underscore (_).') -@click.option('--metric-description', '-d', default='Restic summary metrics', help='Metric description.') +@click.option( + '--job-name', '-n', + required=True, + help='Restic job name to attach to the exported metrics' +) +@click.option( + '--extra-labels', '-a', + required=False, + default=None, + help='Pairs key=value separated by commas with extra labels to add to the summary' +) +@click.option( + '--metric-name', '-m', + default='restic_summary', + help='Metric name. Spaces will be replaced with underscore (_).' +) +@click.option( + '--metric-description', '-d', + default='Restic summary metrics', + help='Metric description.' +) +@click.option( + '--systemd', '-s', + default=False, + help='Get JSON data from Systemd units', +) @click_config_file.configuration_option() -def __main__(debug_level, log_file, json_file, job_name, extra_labels, metric_name, metric_description): - return restic_exporter(debug_level, log_file, json_file, job_name, extra_labels, metric_name, metric_description) +def __main__(**kwargs): + return ResticExporter(**kwargs) if __name__ == "__main__": __main__() -