adelgado/find_duplicate_files
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add exclude

Browse source
This commit is contained in:
Antonio J. Delgado 2023-01-25 17:04:15 +02:00
parent 2c8ef55f15
commit 2f5977a8fc
1 changed files with 33 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
find_duplicate_files/find_duplicate_files.py
View file

@ -13,10 +13,11 @@ import click_config_file
from logging.handlers import SysLogHandler from logging.handlers import SysLogHandler
import zlib import zlib
import sqlite3 import sqlite3
import re
class find_duplicate_files: class find_duplicate_files:
def __init__(self, debug_level, log_file, dummy, first_directory, second_directory): def __init__(self, debug_level, log_file, dummy, first_directory, second_directory, exclude):
''' Initial function called when object is created ''' ''' Initial function called when object is created '''
self.config = dict() self.config = dict()
self.config['debug_level'] = debug_level self.config['debug_level'] = debug_level
@ -28,6 +29,7 @@ class find_duplicate_files:
self.dummy = dummy self.dummy = dummy
self.first_directory = first_directory self.first_directory = first_directory
self.second_directory = second_directory self.second_directory = second_directory
self.exclude = exclude
self._init_db_cache() self._init_db_cache()
@ -73,29 +75,39 @@ class find_duplicate_files:
else: else:
return False return False
def _test_exclude(self, file_name):
""" Test if a filename match any of the exclusions """
for exclude in self.exclude:
match = re.search(exclude, file_name)
if match:
return True
else:
return False
def recursive_scandir(self, path, ignore_hidden_files=True): def recursive_scandir(self, path, ignore_hidden_files=True):
''' Recursively scan a directory for files''' ''' Recursively scan a directory for files'''
files = dict() files = dict()
try: try:
for file in os.scandir(path): for file in os.scandir(path):
if not file.name.startswith('.'): if not file.name.startswith('.'):
if file.is_file(): if not self._test_exclude(file.path):
check_cache = self._check_file_cache(file.path) if file.is_file():
if check_cache: check_cache = self._check_file_cache(file.path)
files[check_cache] = file.path if check_cache:
else: files[check_cache] = file.path
with open(file.path, 'rb') as file_pointer: else:
file_content = file_pointer.read() with open(file.path, 'rb') as file_pointer:
hash = zlib.adler32(file_content) file_content = file_pointer.read()
files[hash] = file.path hash = zlib.adler32(file_content)
self._cache_file(file.path, hash) files[hash] = file.path
elif file.is_dir(follow_symlinks=False): self._cache_file(file.path, hash)
more_files = self.recursive_scandir( elif file.is_dir(follow_symlinks=False):
file.path, more_files = self.recursive_scandir(
ignore_hidden_files=ignore_hidden_files file.path,
) ignore_hidden_files=ignore_hidden_files
if more_files: )
files = { **files, **more_files } if more_files:
files = { **files, **more_files }
except PermissionError as error: except PermissionError as error:
self._log.warning(f"Permission denied accessing folder '{path}'") self._log.warning(f"Permission denied accessing folder '{path}'")
self._log.debug(f"Found {len(files)} files. Cache contains {self._cache_size()} records.") self._log.debug(f"Found {len(files)} files. Cache contains {self._cache_size()} records.")
@ -142,9 +154,10 @@ class find_duplicate_files:
@click.option("--dummy","-n", is_flag=True, help="Don't do anything, just show what would be done.") # Don't forget to add dummy to parameters of main function @click.option("--dummy","-n", is_flag=True, help="Don't do anything, just show what would be done.") # Don't forget to add dummy to parameters of main function
@click.option('--first-directory', '-f', required=True, help='First directory to find files AND TO DELETE FILES FROM!!!') @click.option('--first-directory', '-f', required=True, help='First directory to find files AND TO DELETE FILES FROM!!!')
@click.option('--second-directory', '-s', required=True, help='Second directory to find files') @click.option('--second-directory', '-s', required=True, help='Second directory to find files')
@click.option('--exclude', '-e', multiple=True, help='Regular expression pattern to exclude from files and directories.')
@click_config_file.configuration_option() @click_config_file.configuration_option()
def __main__(debug_level, log_file, dummy, first_directory, second_directory): def __main__(debug_level, log_file, dummy, first_directory, second_directory, exclude):
return find_duplicate_files(debug_level, log_file, dummy, first_directory, second_directory) return find_duplicate_files(debug_level, log_file, dummy, first_directory, second_directory, exclude)
if __name__ == "__main__": if __name__ == "__main__":
__main__() __main__()