From ff5b83e48f81c34836cb2c3f7f6acc1a19dc1195 Mon Sep 17 00:00:00 2001 From: Alexander Bushnev Date: Fri, 11 May 2018 16:10:03 -0400 Subject: [PATCH] Importer added. --- config.py | 1 + fileprop.py | 19 ++++++-- importer.py | 100 ++++++++++++++++++++++++++++++++++++++++ log.py | 36 +++++++++++++++ rotate.py => rotator.py | 10 ++-- 5 files changed, 158 insertions(+), 8 deletions(-) create mode 100755 importer.py create mode 100644 log.py rename rotate.py => rotator.py (91%) diff --git a/config.py b/config.py index 86f5af5..6434c1d 100755 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ class Config(object): 'out_time_format': '%%Y-%%m-%%d_%%H-%%M-%%S', 'time_src': 'exif,name,attr', 'remove_garbage': True, + 'move_mode': False, 'threads_count': 2, } } diff --git a/fileprop.py b/fileprop.py index ac70bf0..d844a66 100755 --- a/fileprop.py +++ b/fileprop.py @@ -49,10 +49,10 @@ class FileProp(object): def __init__(self, config, fullname): self.__config = config - path, fname_ext = os.path.split(fullname) - fname, ext = os.path.splitext(fname_ext) + self.__path, fname_ext = os.path.split(fullname) + fname, self.__ext = os.path.splitext(fname_ext) - self.__type = self.__type_by_ext(ext) + self.__type = self.__type_by_ext(self.__ext) self.__time = self.__time(fullname, fname) @@ -133,6 +133,19 @@ class FileProp(object): def ok(self): return self.__ok + def path(self): + return self.__path + + def ext(self): + return self.__ext + + def out_name_full(self, path=None): + if path is None: + path = self.__path + + return os.path.join(path, self.out_name()) + self.ext() + + if __name__ == '__main__': import sys diff --git a/importer.py b/importer.py new file mode 100755 index 0000000..a205f5f --- /dev/null +++ b/importer.py @@ -0,0 +1,100 @@ +#!/usr/bin/python3 + +import os +import logging +import threading + +import config +import rotator +import fileprop + + +class Importer(threading.Thread): + def __init__(self, config, input_path, output_path): + threading.Thread.__init__(self) + self.__config = config + self.__input_path = input_path + self.__output_path = output_path + self.__move_mode = bool(config['main']['move_mode']) + self.__remove_garbage = bool(config['main']['remove_garbage']) + self.__rot = None + self.__stat = {} + + def run(self): + logging.info( + 'Start: %s -> %s' % + (self.__input_path, self.__output_path)) + + filenames = self.__scan_files(self.__input_path) + logging.info('Found %s files' % len(filenames)) + + new_filenames = self.__move_files(filenames) + logging.info('Moved %s files' % len(new_filenames)) + + self.__rotate_files(new_filenames) + logging.info('Done') + + def __scan_files(self, input_path): + res = [] + for root, dirs, files in os.walk(input_path): + for fname in files: + res.append(os.path.join(root, fname)) + + self.__stat['total'] = len(res) + return res + + def __move_files(self, filenames): + self.__stat['moved'] = 0 + self.__stat['removed'] = 0 + self.__stat['skipped'] = 0 + res = [] + for fname in filenames: + prop = fileprop.FileProp(self.__config, fname) + + if prop.type() == prop.GARBAGE: + if self.__remove_garbage: + os.remove(fname) + self.__stat['removed'] += 1 + else: + self.__stat['skipped'] += 1 + continue + + if prop.type() == prop.OTHER: + self.__stat['skipped'] += 1 + continue + + if self.__output_path: + pass + else: + if prop.ok(): + res.append(fname) + else: + new_fname = prop.out_name_full() + os.rename(fname, new_fname) # TODO: CHECK DUPLICATES!!!! + res.append(new_fname) + + self.__stat['moved'] += 1 + + return res + + def __rotate_files(self, filenames): + self.__rot = rotator.Rotator(self.__config, filenames) + self.__rot.run() + + def status(self): + if self.__rot: + self.__stat['rotation'] = self.__rot.status() + return self.__stat + + +if __name__ == '__main__': + import sys + import log + + log.initLogger() + + imp = Importer(config.Config(False), sys.argv[1], sys.argv[2]) + imp.start() + imp.join() + + print(imp.status()) diff --git a/log.py b/log.py new file mode 100644 index 0000000..4fb0aeb --- /dev/null +++ b/log.py @@ -0,0 +1,36 @@ +import os +import sys +import time +import logging + + +def calcLogName(): + defpath = '/var/log/photo-importer' + + fname = time.strftime('%Y-%m-%d_%H-%M-%S_', time.localtime()) + '.log' + + return os.path.join(defpath, fname) + + +def initLogger(filename=None): + if filename is not None: + try: + os.makedirs(os.path.split(filename)[0]) + except OSError: + pass + #init file logger and console + fh = logging.FileHandler(filename, 'a') + else: + #init only console + fh = logging.StreamHandler() + + form = '[%(asctime)s] [%(levelname)s] %(message)s' + datefmt = '%Y-%m-%d %H:%M:%S' + fmt = logging.Formatter(form, datefmt) + fh.setFormatter(fmt) + logging.getLogger().addHandler(fh) + + logging.getLogger().setLevel(logging.INFO) + + logging.info('Log file: ' + str(filename)) + logging.debug(str(sys.argv)) diff --git a/rotate.py b/rotator.py similarity index 91% rename from rotate.py rename to rotator.py index b127556..5cb4c41 100755 --- a/rotate.py +++ b/rotator.py @@ -61,11 +61,11 @@ class Rotator(object): return ok def status(self): - return ( - len(self.__filenames), - self.__processed, - self.__good, - self.__errors) + return { + 'total': len(self.__filenames), + 'processed': self.__processed, + 'good': self.__good, + 'errors': self.__errors} if __name__ == '__main__':