110 lines
4.7 KiB
Python
110 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
# -*- encoding: utf-8 -*-
|
|
#
|
|
# This script is licensed under GNU GPL version 2.0 or above
|
|
# (c) 2023 Antonio J. Delgado
|
|
# Given two directories, find files in first directory that are present in the second by checking hashes
|
|
|
|
import sys
|
|
import os
|
|
import logging
|
|
import click
|
|
import click_config_file
|
|
from logging.handlers import SysLogHandler
|
|
import zlib
|
|
|
|
class find_duplicate_files:
|
|
|
|
def __init__(self, debug_level, log_file, dummy, first_directory, second_directory):
|
|
''' Initial function called when object is created '''
|
|
self.config = dict()
|
|
self.config['debug_level'] = debug_level
|
|
if log_file is None:
|
|
log_file = os.path.join(os.environ.get('HOME', os.environ.get('USERPROFILE', os.getcwd())), 'log', 'find_duplicate_files.log')
|
|
self.config['log_file'] = log_file
|
|
self._init_log()
|
|
|
|
self.dummy = dummy
|
|
self.first_directory = first_directory
|
|
self.second_directory = second_directory
|
|
|
|
first_files = self.recursive_scandir(self.first_directory)
|
|
self._log.debug(f"Found {len(first_files)} files in first directory '{first_directory}'")
|
|
second_files = self.recursive_scandir(self.second_directory)
|
|
self._log.debug(f"Found {len(second_files)} files in second directory '{second_directory}'")
|
|
|
|
for hash in first_files:
|
|
if hash in second_files:
|
|
print(f"#File '{first_files[hash]}' is dupe with '{second_files[hash]}'.")
|
|
print(f"rm '{first_files[hash]}'")
|
|
|
|
def recursive_scandir(self, path, ignore_hidden_files=True):
|
|
''' Recursively scan a directory for files'''
|
|
files = dict()
|
|
try:
|
|
for file in os.scandir(path):
|
|
if not file.name.startswith('.'):
|
|
if file.is_file():
|
|
with open(file.path, 'rb') as file_pointer:
|
|
file_content = file_pointer.read()
|
|
hash = zlib.adler32(file_content)
|
|
files[hash] = file.path
|
|
elif file.is_dir(follow_symlinks=False):
|
|
more_files = self.recursive_scandir(
|
|
file.path,
|
|
ignore_hidden_files=ignore_hidden_files
|
|
)
|
|
if more_files:
|
|
files = { **files, **more_files }
|
|
except PermissionError as error:
|
|
self._log.warning(f"Permission denied accessing folder '{path}'")
|
|
return files
|
|
|
|
def _init_log(self):
|
|
''' Initialize log object '''
|
|
self._log = logging.getLogger("find_duplicate_files")
|
|
self._log.setLevel(logging.DEBUG)
|
|
|
|
sysloghandler = SysLogHandler()
|
|
sysloghandler.setLevel(logging.DEBUG)
|
|
self._log.addHandler(sysloghandler)
|
|
|
|
streamhandler = logging.StreamHandler(sys.stdout)
|
|
streamhandler.setLevel(logging.getLevelName(self.config.get("debug_level", 'INFO')))
|
|
self._log.addHandler(streamhandler)
|
|
|
|
if 'log_file' in self.config:
|
|
log_file = self.config['log_file']
|
|
else:
|
|
home_folder = os.environ.get('HOME', os.environ.get('USERPROFILE', ''))
|
|
log_folder = os.path.join(home_folder, "log")
|
|
log_file = os.path.join(log_folder, "find_duplicate_files.log")
|
|
|
|
if not os.path.exists(os.path.dirname(log_file)):
|
|
os.mkdir(os.path.dirname(log_file))
|
|
|
|
filehandler = logging.handlers.RotatingFileHandler(log_file, maxBytes=102400000)
|
|
# create formatter
|
|
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
|
|
filehandler.setFormatter(formatter)
|
|
filehandler.setLevel(logging.DEBUG)
|
|
self._log.addHandler(filehandler)
|
|
return True
|
|
|
|
@click.command()
|
|
@click.option("--debug-level", "-d", default="INFO",
|
|
type=click.Choice(
|
|
["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"],
|
|
case_sensitive=False,
|
|
), help='Set the debug level for the standard output.')
|
|
@click.option('--log-file', '-l', help="File to store all debug messages.")
|
|
@click.option("--dummy","-n", is_flag=True, help="Don't do anything, just show what would be done.") # Don't forget to add dummy to parameters of main function
|
|
@click.option('--first-directory', '-f', required=True, help='First directory to find files AND TO DELETE FILES FROM!!!')
|
|
@click.option('--second-directory', '-s', required=True, help='Second directory to find files')
|
|
@click_config_file.configuration_option()
|
|
def __main__(debug_level, log_file, dummy, first_directory, second_directory):
|
|
return find_duplicate_files(debug_level, log_file, dummy, first_directory, second_directory)
|
|
|
|
if __name__ == "__main__":
|
|
__main__()
|
|
|