Extensions moved to config. Fileprop optimization. Dryrun mode added.

config.py

@@ -14,8 +14,13 @@ class Config(object):
             'out_subdir_video': 'Video',
             'out_subdir_audio': 'Audio',
             'time_src_image': 'exif,name',
-            'time_src_video': 'exif,name,attr',
-            'time_src_audio': 'exif,name,attr',
+            'time_src_video': 'name,attr',
+            'time_src_audio': 'name,attr',
+            'file_ext_image': 'jpeg,jpg',
+            'file_ext_video': 'mp4,mpg,mpeg,mov,avi',
+            'file_ext_audio': 'mp3,3gpp,m4a,wav',
+            'file_ext_garbage': 'thm,ctg',
+            'file_ext_ignore': 'ini,zip,db',
             'remove_garbage': 1,
             'remove_empty_dirs': 1,
             'move_mode': 0,

default.cfg

@@ -1,8 +1,8 @@
 [main]
 # time source order
 time_src_image = exif,name
-time_src_video = exif,name,attr
-time_src_audio = exif,name,attr
+time_src_video = name,attr
+time_src_audio = name,attr
 
 # Date/Time formats
 out_date_format = %%Y-%%m-%%d
@@ -13,6 +13,13 @@ out_subdir_image = Foto
 out_subdir_video = Video
 out_subdir_audio = Audio
 
+# File extensions  
+file_ext_image = jpeg,jpg
+file_ext_video = mp4,mpg,mpeg,mov,avi
+file_ext_audio = mp3,3gpp,m4a,wav
+file_ext_garbage = thm,ctg
+file_ext_ignore = ini,zip,db
+
 # Thread count
 threads_count = 2
 

fileprop.py

@@ -11,27 +11,14 @@ import datetime
 import config
 
 
+IGNORE = 0 
+IMAGE = 1
+VIDEO = 2
+AUDIO = 3
+GARBAGE = 4
+
+
 class FileProp(object):
-    OTHER = 0
-    IMAGE = 1
-    VIDEO = 2
-    AUDIO = 3
-    GARBAGE = 4
-
-    EXT_TO_TYPE = {
-        '.jpeg': IMAGE,
-        '.jpg': IMAGE,
-        '.mp4': VIDEO,
-        '.mpg': VIDEO,
-        '.mpeg': VIDEO,
-        '.avi': VIDEO,
-        '.mp3': AUDIO,
-        '.3gpp': AUDIO,
-        '.m4a': AUDIO,
-        '.thm': GARBAGE,
-        '.ctg': GARBAGE,
-    }
-
     DATE_REX = [
         (re.compile('\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}'),
             '%Y-%m-%d_%H-%M-%S'),
@@ -52,31 +39,39 @@ class FileProp(object):
         AUDIO: 'time_src_audio',
     }
 
-    def __init__(self, config, fullname):
+    FILE_EXT_CFG = {
+        IMAGE: 'file_ext_image',
+        VIDEO: 'file_ext_video',
+        AUDIO: 'file_ext_audio',
+        GARBAGE: 'file_ext_garbage',
+        IGNORE: 'file_ext_ignore',
+    }
+
+    EXT_TO_TYPE = {}
+
+    def __init__(self, config):
         self.__config = config
+        self.__prepare_ext_to_type()
+        self.__out_list = set()
 
-        self.__path, fname_ext = os.path.split(fullname)
-        fname, self.__ext = os.path.splitext(fname_ext)
-
-        self.__type = self.__type_by_ext(self.__ext)
-
-        self.__time = self.__time(fullname, fname, self.__type)
-
-        out_name = self.out_name()
-        if out_name:
-            self.__ok = fname[0:len(out_name)] == out_name
-        else:
-            self.__ok = False
+    def __prepare_ext_to_type(self):
+        self.EXT_TO_TYPE = {}
+        for tp, cfg in self.FILE_EXT_CFG.items():
+            for ext in self.__config['main'][cfg].split(','):
+                ext = '.' + ext.lower()
+                if ext in self.EXT_TO_TYPE:
+                    logging.fatal('Double ext: ' + ext)
+                self.EXT_TO_TYPE[ext] = tp
 
     def __type_by_ext(self, ext):
         try:
             return self.EXT_TO_TYPE[ext.lower()]
         except KeyError:
             logging.warning('Unknown ext: ' + ext)
-            return self.OTHER
+            return IGNORE 
 
     def __time(self, fullname, name, tp):
-        if self.__type not in (self.IMAGE, self.VIDEO, self.AUDIO):
+        if tp not in (IMAGE, VIDEO, AUDIO):
             return None
 
         for src in self.__config['main'][self.TIME_SRC_CFG[tp]].split(','):
@@ -93,6 +88,8 @@ class FileProp(object):
             if time:
                 return time
 
+        return None
+
     def __time_by_name(self, fname):
         for exp, fs in self.DATE_REX:
             mat = exp.findall(fname)
@@ -105,13 +102,9 @@ class FileProp(object):
                     return time
                 except ValueError:
                     pass
-
         return None
 
     def __time_by_exif(self, fullname):
-        if self.__type != self.IMAGE:
-            return None
-
         try:
             with open(fullname, 'rb') as f:
                 tags = exifread.process_file(f)
@@ -127,12 +120,49 @@ class FileProp(object):
         except (FileNotFoundError, KeyError) as ex:
             logging.warning('time by attr (%s) exception: %s' % (fullname, ex))
 
-    def out_name(self):
-        if self.__time:
-            return self.__time.strftime(
+    def _out_name_full(self, path, out_name, ext):
+        res = os.path.join(path, out_name) + ext
+
+        i = 1
+        while os.path.isfile(res) or res in self.__out_list:
+            i += 1
+            res = os.path.join(path, out_name + '_' + str(i) + ext)
+
+        self.__out_list.add(res)
+
+        return res
+
+    def get(self, fullname):
+        path, fname_ext = os.path.split(fullname)
+        fname, ext = os.path.splitext(fname_ext)
+
+        tp = self.__type_by_ext(ext)
+
+        ftime = self.__time(fullname, fname, tp)
+
+        if ftime:
+            out_name = ftime.strftime(
                 self.__config['main']['out_time_format'])
         else:
-            return None
+            out_name = None
+
+        if out_name:
+            ok = fname[0:len(out_name)] == out_name
+        else:
+            ok = False
+
+        return FilePropRes(self, tp, ftime, path, ext, out_name, ok)
+
+
+class FilePropRes(object):
+    def __init__(self, prop_ptr, tp, time, path, ext, out_name, ok):
+        self.__prop_ptr = prop_ptr
+        self.__type = tp
+        self.__time = time
+        self.__path = path
+        self.__ext = ext
+        self.__out_name = out_name
+        self.__ok = ok
 
     def type(self):
         return self.__type
@@ -149,21 +179,15 @@ class FileProp(object):
     def ext(self):
         return self.__ext
 
+    def out_name(self):
+        return self.__out_name
+
     def out_name_full(self, path=None):
         if path is None:
             path = self.__path
 
-        out_name = self.out_name()
-
-        res = os.path.join(path, out_name) + self.ext()
-
-        i = 1
-        while (os.path.isfile(res)):
-            i += 1
-            res = os.path.join(path, out_name + '_' + str(i) + self.ext())
-
-        return res
-
+        return self.__prop_ptr._out_name_full(
+            path, self.__out_name, self.__ext)
 
 if __name__ == '__main__':
     import sys

fileprop_test.py

@@ -9,130 +9,129 @@ import datetime
 class TestFileProp(unittest.TestCase):
     def setUp(self):
         self.conf = config.Config()
+        self.fp = fileprop.FileProp(self.conf)
 
     # photo
     def test_camera_photo(self):
-        fp = fileprop.FileProp(self.conf, 'DSC_0846.JPG')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('DSC_0846.JPG')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), None)
         self.assertEqual(fp.ok(), False)
 
     def test_phone1_photo(self):
-        fp = fileprop.FileProp(self.conf, 'IMG_20180413_173249204.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('IMG_20180413_173249204.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2018, 4, 13, 17, 32, 49))
         self.assertEqual(fp.ok(), False)
 
     def test_phone2_photo(self):
-        fp = fileprop.FileProp(self.conf, '2018-02-26-18-30-36-816.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('2018-02-26-18-30-36-816.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2018, 2, 26, 18, 30, 36))
         self.assertEqual(fp.ok(), False)
 
     def test_phone3_photo(self):
-        fp = fileprop.FileProp(self.conf, '20180306124843.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('20180306124843.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2018, 3, 6, 12, 48, 43))
         self.assertEqual(fp.ok(), False)
 
     def test_phone4_photo(self):
-        fp = fileprop.FileProp(self.conf, 'P_20170818_191317_1.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('P_20170818_191317_1.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2017, 8, 18, 19, 13, 17))
         self.assertEqual(fp.ok(), False)
 
     def test_phone5_photo(self):
-        fp = fileprop.FileProp(self.conf, 'P_20171010_083339_BF.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('P_20171010_083339_BF.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2017, 10, 10, 8, 33, 39))
         self.assertEqual(fp.ok(), False)
 
     def test_phone6_photo(self):
-        fp = fileprop.FileProp(self.conf, 'zcamera-20171115_054429.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('zcamera-20171115_054429.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2017, 11, 15, 5, 44, 29))
         self.assertEqual(fp.ok(), False)
 
     def test_phone7_photo(self):
-        fp = fileprop.FileProp(self.conf, 'IMG-20171205-WA0006.jpeg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('IMG-20171205-WA0006.jpeg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2017, 12, 5))
         self.assertEqual(fp.ok(), False)
 
     def test_phone8_photo(self):
-        fp = fileprop.FileProp(self.conf, 'DSC_0001_1523811900639.JPG')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('DSC_0001_1523811900639.JPG')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), None)
         self.assertEqual(fp.ok(), False)
 
     def test_phone9_photo(self):
-        fp = fileprop.FileProp(self.conf, 'DSC_0119a.JPG')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('DSC_0119a.JPG')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), None)
         self.assertEqual(fp.ok(), False)
 
     def test_valid1_photo(self):
-        fp = fileprop.FileProp(self.conf, '2018-02-26_18-30-36.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('2018-02-26_18-30-36.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2018, 2, 26, 18, 30, 36))
         self.assertEqual(fp.ok(), True)
 
     def test_valid2_photo(self):
-        fp = fileprop.FileProp(self.conf, '2018-02-26_18-30-36_9.jpg')
-        self.assertEqual(fp.type(), fp.IMAGE)
+        fp = self.fp.get('2018-02-26_18-30-36_9.jpg')
+        self.assertEqual(fp.type(), fileprop.IMAGE)
         self.assertEqual(fp.time(), datetime.datetime(2018, 2, 26, 18, 30, 36))
         self.assertEqual(fp.ok(), True)
 
     # video
     def test_camera_video(self):
-        fp = fileprop.FileProp(self.conf, 'MOV_1422.mp4')
-        self.assertEqual(fp.type(), fp.VIDEO)
+        fp = self.fp.get('MOV_1422.mp4')
+        self.assertEqual(fp.type(), fileprop.VIDEO)
         self.assertEqual(fp.time(), None)
         self.assertEqual(fp.ok(), False)
 
     def test_videoshow_video(self):
-        fp = fileprop.FileProp(
-            self.conf,
-            'Video_20171107123943353_by_videoshow.mp4')
-        self.assertEqual(fp.type(), fp.VIDEO)
+        fp = self.fp.get('Video_20171107123943353_by_videoshow.mp4')
+        self.assertEqual(fp.type(), fileprop.VIDEO)
         self.assertEqual(fp.time(), datetime.datetime(2017, 11, 7, 12, 39, 43))
         self.assertEqual(fp.ok(), False)
 
     def test_phone1_video(self):
-        fp = fileprop.FileProp(self.conf, 'VID-20180407-WA0000_0811.mp4')
-        self.assertEqual(fp.type(), fp.VIDEO)
+        fp = self.fp.get('VID-20180407-WA0000_0811.mp4')
+        self.assertEqual(fp.type(), fileprop.VIDEO)
         self.assertEqual(fp.time(), datetime.datetime(2018, 4, 7))
         self.assertEqual(fp.ok(), False)
 
     def test_phone2_video(self):
-        fp = fileprop.FileProp(self.conf, 'video_2017-12-28T22.13.03.mp4')
-        self.assertEqual(fp.type(), fp.VIDEO)
+        fp = self.fp.get('video_2017-12-28T22.13.03.mp4')
+        self.assertEqual(fp.type(), fileprop.VIDEO)
         self.assertEqual(fp.time(), datetime.datetime(2017, 12, 28, 22, 13, 3))
         self.assertEqual(fp.ok(), False)
 
     # audio
     def test_phone1_audio(self):
-        fp = fileprop.FileProp(self.conf, 'AUD-20170924-WA0002.3gpp')
-        self.assertEqual(fp.type(), fp.AUDIO)
+        fp = self.fp.get('AUD-20170924-WA0002.3gpp')
+        self.assertEqual(fp.type(), fileprop.AUDIO)
         self.assertEqual(fp.time(), datetime.datetime(2017, 9, 24))
         self.assertEqual(fp.ok(), False)
 
     def test_phone2_audio(self):
-        fp = fileprop.FileProp(self.conf, 'AUD-20170924-WA0001.mp3')
-        self.assertEqual(fp.type(), fp.AUDIO)
+        fp = self.fp.get('AUD-20170924-WA0001.mp3')
+        self.assertEqual(fp.type(), fileprop.AUDIO)
         self.assertEqual(fp.time(), datetime.datetime(2017, 9, 24))
         self.assertEqual(fp.ok(), False)
 
     def test_phone3_audio(self):
-        fp = fileprop.FileProp(self.conf, 'AUD-20171122-WA0000.m4a')
-        self.assertEqual(fp.type(), fp.AUDIO)
+        fp = self.fp.get('AUD-20171122-WA0000.m4a')
+        self.assertEqual(fp.type(), fileprop.AUDIO)
         self.assertEqual(fp.time(), datetime.datetime(2017, 11, 22))
         self.assertEqual(fp.ok(), False)
 
     # garbage
     def test_garbage(self):
-        fp = fileprop.FileProp(self.conf, 'M0101.CTG')
-        self.assertEqual(fp.type(), fp.GARBAGE)
+        fp = self.fp.get('M0101.CTG')
+        self.assertEqual(fp.type(), fileprop.GARBAGE)
         self.assertEqual(fp.time(), None)
         self.assertEqual(fp.ok(), False)
 

importer.py

@@ -7,26 +7,28 @@ import threading
 import mover
 import config
 import rotator
+import fileprop
 
 
 class Importer(threading.Thread):
-    def __init__(self, config, input_path, output_path):
+    def __init__(self, config, input_path, output_path, dryrun):
         threading.Thread.__init__(self)
         self.__config = config
         self.__input_path = input_path
         self.__output_path = output_path
+        self.__dryrun = dryrun
         self.__mov = None
         self.__rot = None
         self.__stat = {'stage': ''}
 
     def run(self):
         logging.info(
-            'Start: %s -> %s' %
-            (self.__input_path, self.__output_path))
+            'Start: %s -> %s (dryrun: %s)' %
+            (self.__input_path, self.__output_path, self.__dryrun))
 
         filenames, dirs = self.__scan_files(self.__input_path)
 
-        new_filenames = self.__move_files(filenames)
+        new_filenames = self.__image_filenames(self.__move_files(filenames))
 
         if self.__config['main']['remove_empty_dirs']:
             self.__remove_empty_dirs(dirs)
@@ -50,6 +52,8 @@ class Importer(threading.Thread):
                 res_dir.append(os.path.join(root, dname))
 
         self.__stat['total'] = len(res)
+        res.sort()
+        res_dir.sort()
         logging.info('Found %i files and %i dirs' % (len(res), len(res_dir)))
         return res, res_dir
 
@@ -62,18 +66,27 @@ class Importer(threading.Thread):
             self.__config,
             self.__input_path,
             self.__output_path,
-            filenames)
+            filenames,
+            self.__dryrun)
         self.__stat['stage'] = 'move'
 
         res = self.__mov.run()
         logging.info('Processed %s files' % len(res))
         return res
 
+    def __image_filenames(self, move_result):
+        res = []
+        for old, new, prop in move_result:
+            if prop.type() == fileprop.IMAGE:
+                res.append(new)
+        return res
+
     def __rotate_files(self, filenames):
         logging.info('Rotating')
         self.__rot = rotator.Rotator(
             self.__config,
-            filenames)
+            filenames,
+            self.__dryrun)
         self.__stat['stage'] = 'rotate'
 
         self.__rot.run()

mover.py

@@ -9,19 +9,21 @@ import fileprop
 
 class Mover(object):
     OUT_SUBDIR_CFG = {
-        fileprop.FileProp.IMAGE: 'out_subdir_image',
-        fileprop.FileProp.VIDEO: 'out_subdir_video',
-        fileprop.FileProp.AUDIO: 'out_subdir_audio',
+        fileprop.IMAGE: 'out_subdir_image',
+        fileprop.VIDEO: 'out_subdir_video',
+        fileprop.AUDIO: 'out_subdir_audio',
     }
 
-    def __init__(self, config, input_path, output_path, filenames):
+    def __init__(self, config, input_path, output_path, filenames, dryrun):
         self.__config = config
         self.__input_path = input_path
         self.__output_path = output_path
         self.__filenames = filenames
+        self.__dryrun = dryrun
         self.__move_mode = int(config['main']['move_mode'])
         self.__remove_garbage = int(config['main']['remove_garbage'])
         self.__stat = {'total': len(filenames)}
+        self.__file_prop = fileprop.FileProp(self.__config)
 
     def run(self):
         self.__stat['moved'] = 0
@@ -33,29 +35,30 @@ class Mover(object):
         res = []
         for fname in self.__filenames:
             try:
-                new_fname = self.__move_file(fname)
+                prop = self.__file_prop.get(fname)
+                new_fname = self.__move_file(fname, prop)
                 if new_fname:
-                    res.append(new_fname)
+                    res.append((fname, new_fname, prop))
             except Exception as ex:
                 logging.error('Move files exception: %s' % ex)
                 self.__stat['errors'] += 1
 
             self.__stat['processed'] += 1
+
         return res
 
-    def __move_file(self, fname):
-        prop = fileprop.FileProp(self.__config, fname)
-
-        if prop.type() == prop.GARBAGE:
+    def __move_file(self, fname, prop):
+        if prop.type() == fileprop.GARBAGE:
             if self.__remove_garbage:
-                os.remove(fname)
+                if not self.__dryrun:
+                    os.remove(fname)
                 logging.info('removed "%s"' % fname)
                 self.__stat['removed'] += 1
             else:
                 self.__stat['skipped'] += 1
             return None
 
-        if prop.type() == prop.OTHER or prop.time() is None:
+        if prop.type() == fileprop.IGNORE or prop.time() is None:
             self.__stat['skipped'] += 1
             return None
 
@@ -68,15 +71,19 @@ class Mover(object):
 
             path = os.path.join(self.__output_path, type_subdir, date_subdir)
             if not os.path.isdir(path):
-                os.makedirs(path)
+                if not self.__dryrun:
+                    os.makedirs(path)
+                logging.info('dir "%s" created' % path)
 
             fullname = prop.out_name_full(path)
             if self.__move_mode:
-                shutil.move(fname, fullname)
+                if not self.__dryrun:
+                    shutil.move(fname, fullname)
                 logging.info('"%s" moved "%s"' % (fname, fullname))
                 self.__stat['moved'] += 1
             else:
-                shutil.copy2(fname, fullname)
+                if not self.__dryrun:
+                    shutil.copy2(fname, fullname)
                 logging.info('"%s" copied "%s"' % (fname, fullname))
                 self.__stat['copied'] += 1
 
@@ -87,7 +94,8 @@ class Mover(object):
                 return None
             else:
                 new_fname = prop.out_name_full()
-                os.rename(fname, new_fname)
+                if not self.__dryrun:
+                    os.rename(fname, new_fname)
                 logging.info('"%s" renamed "%s"' % (fname, new_fname))
                 self.__stat['moved'] += 1
                 return new_fname

rotator.py

@@ -8,9 +8,10 @@ import config
 
 
 class Rotator(object):
-    def __init__(self, config, filenames):
+    def __init__(self, config, filenames, dryrun):
         self.__config = config
         self.__filenames = filenames
+        self.__dryrun = dryrun
         self.__processed = 0
         self.__good = 0
         self.__errors = 0
@@ -34,6 +35,11 @@ class Rotator(object):
         ok = False
         try:
             cmd = 'exiftran -aip "%s"' % filename
+            logging.info('rotate: %s' % cmd)
+
+            if self.__dryrun:
+                return True
+
             p = subprocess.Popen(
                 cmd,
                 shell=True,

run.py

@@ -17,6 +17,9 @@ class ProgressBar(threading.Thread):
         self.__pbar = None
 
     def __create(self, name, count):
+        if count == 0:
+            return
+
         if self.__pbar:
             self.__pbar.finish()
             self.__pbar = None
@@ -57,10 +60,11 @@ class ProgressBar(threading.Thread):
 
 def args_parse():
     parser = argparse.ArgumentParser()
-    parser.add_argument('in_path', help="Input path")
-    parser.add_argument('out_path', help="Output path", nargs='?')
-    parser.add_argument('-c', '--config', help="Config file")
-    parser.add_argument('-l', '--logfile', help="Log file", default='log.txt')
+    parser.add_argument('in_path', help='Input path')
+    parser.add_argument('out_path', help='Output path', nargs='?')
+    parser.add_argument('-c', '--config', help='Config file')
+    parser.add_argument('-l', '--logfile', help='Log file', default='log.txt')
+    parser.add_argument('-d', '--dryrun', help='Dry run', action='store_true')
     return parser.parse_args()
 
 
@@ -74,7 +78,8 @@ def main():
     imp = importer.Importer(
         cfg,
         args.in_path,
-        args.out_path)
+        args.out_path,
+        args.dryrun)
 
     pbar = ProgressBar(imp)
     imp.start()