python/photo-importer
1
0
Fork 0
mirror of https://github.com/sashacmc/photo-importer.git synced 2025-02-22 18:42:16 +02:00
Code Issues Releases Activity

Merge pull request #13 from sashacmc/structure-upgrade

Browse Source 
Structure upgrade and refactoring
This commit is contained in:
Alexander 2024-07-09 01:04:31 +02:00 committed by GitHub
commit a2f82fc795
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 241 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
.github/workflows/build-debian-package.yml vendored Normal file
View File

@ -0,0 +1,56 @@
name: Build Debian Package
on:
push:
branches:
- main
- master
pull_request:
branches:
- main
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build Debian package
run: |
sudo apt-get update
sudo apt-get install -y devscripts debhelper curl
# Ensure you have the necessary build dependencies
sudo apt-get build-dep .
# Build the Debian package
debuild -us -uc -b
- name: Create artifact directory
run: mkdir -p artifacts
- name: Move Debian package to artifact directory
run: mv ../*.deb artifacts/
- name: Test Debian package installation
run: |
sudo apt install -y pip python3-exif python3-progressbar exiftran python3-psutil
sudo pip install PyExifTool
sudo dpkg -i artifacts/*.deb
sudo systemctl enable photo-importer.service
sudo systemctl restart photo-importer.service
sudo systemctl status photo-importer.service
photo-importer -h
- name: Upload Debian package as artifact
uses: actions/upload-artifact@v2
with:
name: debian-package
path: artifacts/*.deb

24
.github/workflows/pylint.yml vendored Normal file
View File

@ -0,0 +1,24 @@
name: Pylint
on: [push]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pylint
pip install .
- name: Analysing the code with pylint
run: |
pylint $(git ls-files '*.py') --disable missing-function-docstring,missing-module-docstring,missing-class-docstring,broad-exception-caught

4
README.md
View File

@ -50,7 +50,9 @@ sudo pip install photo-importer
#### Installing as debian package
```bash
debuild -b
sudo dpkg -i ../photo-importer_1.2.0_all.deb
sudo apt install pip python3-exif python3-progressbar exiftran python3-psutil
sudo pip install PyExifTool
sudo dpkg -i ../photo-importer_1.2.5_all.deb
```
#### Installing via setup.py
```bash

7
debian/changelog vendored
View File

@ -1,3 +1,10 @@
photo-importer (1.2.5) stable; urgency=medium
* Package refactoring
* Code cleanup
-- Alexander Bushnev <Alexander@Bushnev.pro> Tue, 09 Jul 2024 00:24:52 +0200
photo-importer (1.2.4) stable; urgency=medium
* Add time_shift options

2
debian/compat vendored
View File

@ -1 +1 @@
9
10

4
photo_importer/config.py
View File

@ -4,7 +4,7 @@ import os
import configparser
class Config(object):
class Config:
DEFAULT_CONFIG_FILES = (
os.path.expanduser('~/.photo-importer.cfg'),
'/etc/photo-importer.cfg',
@ -66,7 +66,7 @@ class Config(object):
if os.path.exists(self.DEFAULT_CONFIG_FILES[0]):
return
with open(self.DEFAULT_CONFIG_FILES[0], 'w') as conffile:
with open(self.DEFAULT_CONFIG_FILES[0], 'w', encoding='utf-8') as conffile:
self.__config.write(conffile)
def __getitem__(self, sect):

73
photo_importer/fileprop.py
View File

@ -1,12 +1,13 @@
#!/usr/bin/python3
# pylint: disable=too-many-arguments
import os
import re
import stat
import time
import logging
import exiftool
import datetime
import exiftool
IGNORE = 0
@ -16,7 +17,7 @@ AUDIO = 3
GARBAGE = 4
class FileProp(object):
class FileProp:
DATE_REGEX = [
(
re.compile(r'\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}'),
@ -59,8 +60,6 @@ class FileProp(object):
IGNORE: 'file_ext_ignore',
}
EXT_TO_TYPE = {}
DATE_TAGS = [
'EXIF:DateTimeOriginal',
'H264:DateTimeOriginal',
@ -70,8 +69,8 @@ class FileProp(object):
'EXIF:CreateDate',
]
def __init__(self, config):
self.__config = config
def __init__(self, conf):
self.__config = conf
self.__prepare_ext_to_type()
self.__out_list = set()
self.__exiftool = exiftool.ExifToolHelper()
@ -80,19 +79,19 @@ class FileProp(object):
self.__exiftool.terminate()
def __prepare_ext_to_type(self):
self.EXT_TO_TYPE = {}
self.ext_to_type = {}
for tp, cfg in self.FILE_EXT_CFG.items():
for ext in self.__config['main'][cfg].split(','):
ext = '.' + ext.lower()
if ext in self.EXT_TO_TYPE:
logging.fatal('Double ext: ' + ext)
self.EXT_TO_TYPE[ext] = tp
if ext in self.ext_to_type:
logging.fatal('Double ext: %s', ext)
self.ext_to_type[ext] = tp
def __type_by_ext(self, ext):
try:
return self.EXT_TO_TYPE[ext]
return self.ext_to_type[ext]
except KeyError:
logging.warning('Unknown ext: ' + ext)
logging.warning('Unknown ext: %s', ext)
return IGNORE
def __time(self, fullname, name, tp):
@ -100,18 +99,18 @@ class FileProp(object):
return None
for src in self.__config['main'][self.TIME_SRC_CFG[tp]].split(','):
time = None
ftime = None
if src == 'exif':
time = self.__time_by_exif(fullname)
ftime = self.__time_by_exif(fullname)
elif src == 'name':
time = self.__time_by_name(name)
ftime = self.__time_by_name(name)
elif src == 'attr':
time = self.__time_by_attr(fullname)
ftime = self.__time_by_attr(fullname)
else:
raise Exception('Wrong time_src: ' + src)
raise UserWarning(f'Wrong time_src: {src}')
if time:
return time
if ftime:
return ftime
return None
@ -120,11 +119,11 @@ class FileProp(object):
mat = exp.findall(fname)
if len(mat):
try:
time = datetime.datetime.strptime(mat[0], fs)
if time.year < 1990 or time.year > 2100:
ftime = datetime.datetime.strptime(mat[0], fs)
if ftime.year < 1990 or ftime.year > 2100:
continue
return time
return ftime
except ValueError:
pass
return None
@ -140,15 +139,12 @@ class FileProp(object):
md = md[0:pos]
return datetime.datetime.strptime(md, '%Y:%m:%d %H:%M:%S')
logging.warning(
'time by exif (%s) not found tags count: %s'
% (fullname, len(metadata))
)
logging.warning('time by exif (%s) not found tags count: %s', fullname, len(metadata))
for tag, val in metadata.items():
logging.debug('%s: %s' % (tag, val))
return None
logging.debug('%s: %s', tag, val)
except Exception as ex:
logging.warning('time by exif (%s) exception: %s' % (fullname, ex))
logging.warning('time by exif (%s) exception: %s', fullname, ex)
return None
def __time_by_attr(self, fullname):
try:
@ -156,18 +152,19 @@ class FileProp(object):
time.mktime(time.localtime(os.stat(fullname)[stat.ST_MTIME]))
)
except (FileNotFoundError, KeyError) as ex:
logging.warning('time by attr (%s) exception: %s' % (fullname, ex))
logging.warning('time by attr (%s) exception: %s', fullname, ex)
return None
def __calc_orig_name(self, fname):
if not int(self.__config['main']['add_orig_name']):
return ''
for exp, fs in self.DATE_REGEX:
for exp, _ in self.DATE_REGEX:
mat = exp.findall(fname)
if len(mat):
return ''
return '_' + self.SPACE_REGEX.sub('_', fname)
def _out_name_full(self, path, out_name, ext):
def out_name_full(self, path, out_name, ext):
res = os.path.join(path, out_name) + ext
i = 1
@ -206,11 +203,11 @@ class FileProp(object):
return FilePropRes(self, tp, ftime, path, ext, out_name, ok)
class FilePropRes(object):
def __init__(self, prop_ptr, tp, time, path, ext, out_name, ok):
class FilePropRes:
def __init__(self, prop_ptr, tp, ftime, path, ext, out_name, ok):
self.__prop_ptr = prop_ptr
self.__type = tp
self.__time = time
self.__time = ftime
self.__path = path
self.__ext = ext
self.__out_name = out_name
@ -238,9 +235,7 @@ class FilePropRes(object):
if path is None:
path = self.__path
return self.__prop_ptr._out_name_full(
path, self.__out_name, self.__ext
)
return self.__prop_ptr.out_name_full(path, self.__out_name, self.__ext)
if __name__ == '__main__':
@ -251,7 +246,7 @@ if __name__ == '__main__':
from photo_importer import log
from photo_importer import config
log.initLogger(None, logging.DEBUG)
log.init_logger(None, logging.DEBUG)
fp = FileProp(config.Config()).get(sys.argv[1])
print(fp.type(), fp.time(), fp.ok())

5
photo_importer/fileprop_test.py
View File

@ -1,10 +1,11 @@
#!/usr/bin/python3
# pylint: disable=too-many-public-methods
import unittest
import datetime
from . import config
from . import fileprop
from photo_importer import config
from photo_importer import fileprop
class TestFileProp(unittest.TestCase):

28
photo_importer/importer.py
View File

@ -1,13 +1,14 @@
#!/usr/bin/python3
# pylint: disable=too-many-instance-attributes
import os
import logging
import threading
from . import log
from . import mover
from . import rotator
from . import fileprop
from photo_importer import log
from photo_importer import mover
from photo_importer import rotator
from photo_importer import fileprop
class Importer(threading.Thread):
@ -24,8 +25,7 @@ class Importer(threading.Thread):
def run(self):
logging.info(
'Start: %s -> %s (dryrun: %s)'
% (self.__input_path, self.__output_path, self.__dryrun)
'Start: %s -> %s (dryrun: %s)', self.__input_path, self.__output_path, self.__dryrun
)
filenames, dirs = self.__scan_files(self.__input_path)
@ -38,16 +38,13 @@ class Importer(threading.Thread):
self.__rotate_files(new_filenames)
self.__stat['stage'] = 'done'
logging.info('Done: %s' % str(self.status()))
logging.info('Done: %s', str(self.status()))
def __scan_files(self, input_path):
self.__stat['stage'] = 'scan'
res_dir = []
res = []
for root, dirs, files in os.walk(
input_path, onerror=self.__on_walk_error
):
for root, dirs, files in os.walk(input_path, onerror=self.__on_walk_error):
for fname in files:
res.append(os.path.join(root, fname))
@ -57,17 +54,16 @@ class Importer(threading.Thread):
self.__stat['total'] = len(res)
res.sort()
res_dir.sort()
logging.info('Found %i files and %i dirs' % (len(res), len(res_dir)))
logging.info('Found %i files and %i dirs', len(res), len(res_dir))
return res, res_dir
def __on_walk_error(self, err):
logging.error('Scan files error: %s' % err)
logging.error('Scan files error: %s', err)
def __move_files(self, filenames):
logging.info('Moving')
self.__mov = mover.Mover(
self.__config,
self.__input_path,
self.__output_path,
filenames,
self.__dryrun,
@ -75,12 +71,12 @@ class Importer(threading.Thread):
self.__stat['stage'] = 'move'
res = self.__mov.run()
logging.info('Processed %s files' % len(res))
logging.info('Processed %s files', len(res))
return res
def __image_filenames(self, move_result):
res = []
for old, new, prop in move_result:
for _, new, prop in move_result:
if prop.type() == fileprop.IMAGE:
res.append(new)
return res

14
photo_importer/importer_test.py
View File

@ -4,8 +4,8 @@ import os
import unittest
import tempfile
from . import config
from . import importer
from photo_importer import config
from photo_importer import importer
class TestImporter(unittest.TestCase):
@ -46,7 +46,7 @@ class TestImporter(unittest.TestCase):
)
files = []
for path, cd, fs in os.walk(tmpdirname):
for path, _, fs in os.walk(tmpdirname):
for f in fs:
print(os.path.join(path, f))
files.append(os.path.join(path, f))
@ -55,13 +55,9 @@ class TestImporter(unittest.TestCase):
self.assertEqual(len(files), 2)
self.assertEqual(
files[0],
os.path.join(
tmpdirname, 'Foto/2021/2021-12-19/2021-12-19_13-11-36.jpeg'
),
os.path.join(tmpdirname, 'Foto/2021/2021-12-19/2021-12-19_13-11-36.jpeg'),
)
self.assertEqual(
files[1],
os.path.join(
tmpdirname, 'Foto/2022/2022-11-21/2022-11-21_00-42-07.jpg'
),
os.path.join(tmpdirname, 'Foto/2022/2022-11-21/2022-11-21_00-42-07.jpg'),
)

19
photo_importer/log.py
View File

@ -1,22 +1,13 @@
import io
import os
import sys
import time
import logging
LOGFMT = '[%(asctime)s] [%(levelname)s] %(message)s'
DATEFMT = '%Y-%m-%d %H:%M:%S'
def calcLogName():
defpath = '/var/log/photo-importer'
fname = time.strftime('%Y-%m-%d_%H-%M-%S_', time.localtime()) + '.log'
return os.path.join(defpath, fname)
def initLogger(filename=None, level=logging.INFO):
def init_logger(filename=None, level=logging.INFO):
if filename is not None:
try:
os.makedirs(os.path.split(filename)[0])
@ -33,11 +24,11 @@ def initLogger(filename=None, level=logging.INFO):
logging.getLogger().setLevel(level)
logging.info('Log file: ' + str(filename))
logging.info('Log file: %s', filename)
logging.debug(str(sys.argv))
class MemLogger(object):
class MemLogger:
def __init__(self, name):
self.__name = name
fmt = logging.Formatter(LOGFMT, DATEFMT)
@ -45,11 +36,11 @@ class MemLogger(object):
self.__handler = logging.StreamHandler(self.__stream)
self.__handler.setFormatter(fmt)
logging.getLogger().addHandler(self.__handler)
logging.info("MemLogger " + self.__name + " started")
logging.info("MemLogger %s started", self.__name)
def __del__(self):
logging.getLogger().removeHandler(self.__handler)
logging.info("MemLogger " + self.__name + " finished")
logging.info("MemLogger %s finished", self.__name)
def get_text(self):
return self.__stream.getvalue()

52
photo_importer/mover.py
View File

@ -1,23 +1,23 @@
#!/usr/bin/python3
# pylint: disable=too-many-instance-attributes,too-many-arguments
import os
import shutil
import logging
import subprocess
from . import fileprop
from photo_importer import fileprop
class Mover(object):
class Mover:
OUT_SUBDIR_CFG = {
fileprop.IMAGE: 'out_subdir_image',
fileprop.VIDEO: 'out_subdir_video',
fileprop.AUDIO: 'out_subdir_audio',
}
def __init__(self, config, input_path, output_path, filenames, dryrun):
def __init__(self, config, output_path, filenames, dryrun):
self.__config = config
self.__input_path = input_path
self.__output_path = output_path
self.__filenames = filenames
self.__dryrun = dryrun
@ -44,7 +44,7 @@ class Mover(object):
if new_fname:
res.append((fname, new_fname, prop))
except Exception as ex:
logging.error('Move files exception: %s' % ex)
logging.error('Move files exception: %s', ex)
self.__stat['errors'] += 1
self.__stat['processed'] += 1
@ -56,7 +56,7 @@ class Mover(object):
if self.__remove_garbage:
if not self.__dryrun:
os.remove(fname)
logging.info('removed "%s"' % fname)
logging.info('removed "%s"', fname)
self.__stat['removed'] += 1
else:
self.__stat['skipped'] += 1
@ -78,52 +78,50 @@ class Mover(object):
if not os.path.isdir(path):
if not self.__dryrun:
os.makedirs(path)
logging.info('dir "%s" created' % path)
logging.info('dir "%s" created', path)
fullname = prop.out_name_full(path)
if self.__move_mode:
self.__move(fname, fullname)
logging.info('"%s" moved "%s"' % (fname, fullname))
logging.info('"%s" moved "%s"', fname, fullname)
self.__stat['moved'] += 1
else:
self.__copy(fname, fullname)
logging.info('"%s" copied "%s"' % (fname, fullname))
logging.info('"%s" copied "%s"', fname, fullname)
self.__stat['copied'] += 1
return fullname
else:
if prop.ok():
self.__stat['skipped'] += 1
return None
else:
new_fname = prop.out_name_full()
if not self.__dryrun:
os.rename(fname, new_fname)
logging.info('"%s" renamed "%s"' % (fname, new_fname))
self.__stat['moved'] += 1
return new_fname
if prop.ok():
self.__stat['skipped'] += 1
return None
new_fname = prop.out_name_full()
if not self.__dryrun:
os.rename(fname, new_fname)
logging.info('"%s" renamed "%s"', fname, new_fname)
self.__stat['moved'] += 1
return new_fname
def __move(self, src, dst):
if self.__use_shutil:
shutil.move(src, dst)
else:
if not self.__run(["mv", src, dst]):
raise SystemError('mv "%s" "%s" failed' % (src, dst))
raise SystemError(f'mv "{src}" "{dst}" failed')
def __copy(self, src, dst):
if self.__use_shutil:
shutil.copy2(src, dst)
else:
if not self.__run(["cp", "-a", src, dst]):
raise SystemError('mv "%s" "%s" failed' % (src, dst))
raise SystemError(f'cp "{src}" "{dst}" failed')
def __run(self, args):
if self.__dryrun:
return True
with subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as proc:
with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
proc.wait()
info = proc.stdout.read().strip()
if info:
@ -132,9 +130,7 @@ class Mover(object):
if err:
logging.error(err.decode("utf-8"))
elif proc.returncode != 0:
logging.error(
'%s failed with code %i' % (args[0], proc.returncode)
)
logging.error('%s failed with code %i', args[0], proc.returncode)
return proc.returncode == 0
def status(self):

40
photo_importer/rotator.py
View File

@ -2,10 +2,10 @@
import os
import logging
import exiftool
import tempfile
import subprocess
import concurrent.futures
import exiftool
JPEGTRAN_COMMAND = {
@ -23,7 +23,7 @@ JPEGTRAN_COMMAND = {
ORIENTATION_TAG = 'EXIF:Orientation'
class Rotator(object):
class Rotator:
def __init__(self, config, filenames, dryrun):
self.__config = config
self.__filenames = filenames
@ -31,6 +31,7 @@ class Rotator(object):
self.__processed = 0
self.__good = 0
self.__errors = 0
self.__exiftool = None
def run(self):
os.umask(int(self.__config['main']['umask'], 8))
@ -43,10 +44,7 @@ class Rotator(object):
tc = 1
with concurrent.futures.ThreadPoolExecutor(max_workers=tc) as executor:
futures = {
executor.submit(processor, fn): fn for fn in self.__filenames
}
futures = {executor.submit(processor, fn): fn for fn in self.__filenames}
for future in concurrent.futures.as_completed(futures):
self.__processed += 1
@ -61,8 +59,8 @@ class Rotator(object):
def __process_exiftran(self, filename):
ok = False
try:
cmd = 'exiftran -aip "%s"' % filename
logging.debug('rotate: %s' % cmd)
cmd = f'exiftran -aip "{filename}"'
logging.debug('rotate: %s', cmd)
if self.__dryrun:
return True
@ -87,10 +85,10 @@ class Rotator(object):
error += line
if error != '':
logging.error('exiftran (%s) error: %s' % (filename, error))
logging.error('exiftran (%s) error: %s', filename, error)
except Exception as ex:
logging.error('Rotator exception (%s): %s' % (filename, ex))
logging.error('Rotator exception (%s): %s', filename, ex)
return ok
@ -100,9 +98,7 @@ class Rotator(object):
if orientation_cmd is None:
return True
logging.debug(
'rotate: jpegtran %s %s' % (orientation_cmd, filename)
)
logging.debug('rotate: jpegtran %s %s', orientation_cmd, filename)
if self.__dryrun:
return True
@ -110,11 +106,7 @@ class Rotator(object):
handle, tmpfile = tempfile.mkstemp(dir=os.path.dirname(filename))
os.close(handle)
cmd = 'jpegtran -copy all -outfile %s %s %s' % (
tmpfile,
orientation_cmd,
filename,
)
cmd = 'jpegtran -copy all -outfile {tmpfile} {orientation_cmd} {filename}'
with subprocess.Popen(
cmd,
@ -125,9 +117,7 @@ class Rotator(object):
) as p:
line = p.stderr.readline()
if line:
logging.error(
'jpegtran (%s) failed: %s' % (filename, line)
)
logging.error('jpegtran (%s) failed: %s', filename, line)
return False
self.__clear_orientation_tag(tmpfile)
@ -137,7 +127,7 @@ class Rotator(object):
return True
except Exception as ex:
logging.error('Rotator exception (%s): %s' % (filename, ex))
logging.error('Rotator exception (%s): %s', filename, ex)
return False
def __get_orientation_cmd(self, fullname):
@ -145,10 +135,10 @@ class Rotator(object):
if ORIENTATION_TAG not in tags:
return None
orientation = tags[ORIENTATION_TAG]
if 0 <= orientation and orientation < len(JPEGTRAN_COMMAND):
if 0 <= orientation < len(JPEGTRAN_COMMAND):
return JPEGTRAN_COMMAND[orientation]
else:
return None
return None
def __clear_orientation_tag(self, fullname):
self.__exiftool.set_tags(fullname, {ORIENTATION_TAG: 1})

16
photo_importer/run.py
View File

@ -5,9 +5,9 @@ import argparse
import threading
import progressbar
from . import log
from . import config
from . import importer
from photo_importer import log
from photo_importer import config
from photo_importer import importer
class ProgressBar(threading.Thread):
@ -48,7 +48,7 @@ class ProgressBar(threading.Thread):
print('Scan... ', end='', flush=True)
continue
if stage == 'move':
print('Done. Found %i files' % stat['total'])
print(f'Done. Found {stat["total"]} files')
self.__create('Import:', stat['total'])
continue
if stage == 'rotate':
@ -59,9 +59,7 @@ class ProgressBar(threading.Thread):
self.__pbar.finish()
break
if (
stage == 'move' or stage == 'rotate'
) and self.__pbar is not None:
if stage in ('move', 'rotate') and self.__pbar is not None:
self.__pbar.update(stat[stage]['processed'])
@ -80,7 +78,7 @@ def main():
cfg = config.Config(args.config)
log.initLogger(args.logfile)
log.init_logger(args.logfile)
imp = importer.Importer(cfg, args.in_path, args.out_path, args.dryrun)
@ -91,7 +89,7 @@ def main():
pbar.join()
status = imp.status()
logging.info('status: %s' % str(status))
logging.info('status: %s', str(status))
if status['move']['errors'] != 0 or status['rotate']['errors'] != 0:
print('Some errors found. Please check log file.')

104
photo_importer/server.py
View File

@ -1,10 +1,10 @@
#!/usr/bin/python3
# pylint: disable=import-outside-toplevel,import-error,invalid-name,duplicate-code
import os
import re
import glob
import json
import psutil
import urllib
import logging
import argparse
@ -12,9 +12,11 @@ import subprocess
import http.server
from http import HTTPStatus
from . import log
from . import config
from . import importer
import psutil
from photo_importer import log
from photo_importer import config
from photo_importer import importer
FIXED_IN_PATH_NAME = 'none'
@ -64,15 +66,15 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
mount_list = self.__get_mounted_list()
res = {}
for path in glob.glob('/sys/block/*/device'):
dev = re.sub('.*/(.*?)/device', '\g<1>', path)
with open('/sys/block/%s/removable' % (dev,)) as f:
dev = re.sub(r'.*/(.*?)/device', r'\g<1>', path)
with open(f'/sys/block/{dev}/removable', encoding='utf-8') as f:
if f.read(1) != '1':
continue
read_only = False
with open('/sys/block/%s/ro' % (dev,)) as f:
with open(f'/sys/block/{dev}/ro', encoding='utf-8') as f:
if f.read(1) == '1':
read_only = True
for ppath in glob.glob('/sys/block/%s/%s*' % (dev, dev)):
for ppath in glob.glob(f'/sys/block/{dev}/{dev}*'):
pdev = os.path.split(ppath)[1]
if pdev in mount_list:
res[pdev] = {
@ -112,15 +114,13 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
elif os.name == 'posix':
res = self.__get_removable_devices_posix()
else:
raise Exception('Unsupported os: %s' % os.name)
raise UserWarning(f'Unsupported os: {os.name}')
if self.server.fixed_in_path() != '':
res[FIXED_IN_PATH_NAME] = {
'dev_path': FIXED_IN_PATH_NAME,
'mount_path': self.server.fixed_in_path(),
'read_only': not os.access(
self.server.fixed_in_path(), os.W_OK
),
'read_only': not os.access(self.server.fixed_in_path(), os.W_OK),
}
return res
@ -143,30 +143,22 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
r['path'] = info['mount_path']
r['progress'] = 0
r['read_only'] = info['read_only']
r['allow_start'] = not (
info['read_only'] and self.server.move_mode()
)
r['allow_start'] = not (info['read_only'] and self.server.move_mode())
if r['path']:
stat = self.server.import_status(r['path'])
du = psutil.disk_usage(r['path'])
if dev == FIXED_IN_PATH_NAME:
r['size'] = self.__bytes_to_gbytes(
self.__folder_size(r['path'])
)
r['size'] = self.__bytes_to_gbytes(self.__folder_size(r['path']))
else:
r['size'] = self.__bytes_to_gbytes(du.total)
r['usage'] = du.percent
if stat:
stage = stat['stage']
r['state'] = stage
if stage == 'move' or stage == 'rotate':
r['progress'] = round(
100.0 * stat[stage]['processed'] / stat['total']
)
if stage in ('move', 'rotate'):
r['progress'] = round(100.0 * stat[stage]['processed'] / stat['total'])
elif stage == 'done':
cerr = (
stat['move']['errors'] + stat['rotate']['errors']
)
cerr = stat['move']['errors'] + stat['rotate']['errors']
if cerr != 0:
r['state'] = 'error'
r['total'] = cerr
@ -187,7 +179,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
raise HTTPError(HTTPStatus.BAD_REQUEST, 'empty "d" param')
dev_list = self.__get_removable_devices()
if dev not in dev_list:
raise HTTPError(HTTPStatus.BAD_REQUEST, 'wrong device: %s' % dev)
raise HTTPError(HTTPStatus.BAD_REQUEST, f'wrong device: {dev}')
device = dev_list[dev]
if device['mount_path']:
self.server.import_done(device['mount_path'])
@ -196,7 +188,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
def __run_cmd(self, cmd):
error = ''
try:
logging.info('run cmd: %s' % cmd)
logging.info('run cmd: %s', cmd)
with subprocess.Popen(
cmd,
@ -211,7 +203,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
error += line
if error != '':
logging.error('cmd run error: %s' % error.strip())
logging.error('cmd run error: %s', error.strip())
except Exception:
logging.exception('cmd run exception')
@ -223,18 +215,18 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
def __mount_mount(self, dev):
dev_path = self.__check_dev_for_mount(dev)
return self.__run_cmd('pmount --umask=000 %s' % dev_path)
return self.__run_cmd(f'pmount --umask=000 {dev_path}')
def __mount_umount(self, dev):
dev_path = self.__check_dev_for_mount(dev)
return self.__run_cmd('pumount %s' % dev_path)
return self.__run_cmd(f'pumount {dev_path}')
def __mount_request(self, params):
try:
action = params['a'][0]
except Exception as ex:
logging.exception(ex)
raise HTTPError(HTTPStatus.BAD_REQUEST, str(ex))
raise HTTPError(HTTPStatus.BAD_REQUEST, str(ex)) from ex
try:
dev = params['d'][0]
@ -250,9 +242,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
elif action == 'umount':
result = self.__mount_umount(dev)
else:
raise HTTPError(
HTTPStatus.BAD_REQUEST, 'unknown action %s' % action
)
raise HTTPError(HTTPStatus.BAD_REQUEST, f'unknown action {action}')
self.__ok_response(result)
@ -261,7 +251,8 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
return True
def __import_stop(self, dev):
pass
logging.warning('import stop not implemented: %s', dev)
return False
def __import_get_log(self, in_path):
return self.server.get_log(in_path)
@ -272,7 +263,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
in_path = params['p'][0]
except Exception as ex:
logging.exception(ex)
raise HTTPError(HTTPStatus.BAD_REQUEST, str(ex))
raise HTTPError(HTTPStatus.BAD_REQUEST, str(ex)) from ex
try:
out_path = params['o'][0]
@ -291,9 +282,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
result = self.__import_get_log(in_path)
self.__text_response(result)
else:
raise HTTPError(
HTTPStatus.BAD_REQUEST, 'unknown action %s' % action
)
raise HTTPError(HTTPStatus.BAD_REQUEST, f'unknown action {action}')
def __sysinfo_request(self, params):
try:
@ -314,11 +303,9 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
try:
if (path[0]) == '/':
path = path[1:]
fname = os.path.normpath(
os.path.join(self.server.web_path(), path)
)
fname = os.path.normpath(os.path.join(self.server.web_path(), path))
if not fname.startswith(self.server.web_path()):
logging.warning('incorrect path: ' + path)
logging.warning('incorrect path: %s', path)
raise HTTPError(HTTPStatus.NOT_FOUND, path)
ext = os.path.splitext(fname)[1]
cont = ''
@ -337,17 +324,16 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(bytearray(f.read()))
except IOError as ex:
logging.exception(ex)
raise HTTPError(HTTPStatus.NOT_FOUND, path)
raise HTTPError(HTTPStatus.NOT_FOUND, path) from ex
def __path_params(self):
path_params = self.path.split('?')
if len(path_params) > 1:
return path_params[0], urllib.parse.parse_qs(path_params[1])
else:
return path_params[0], {}
return path_params[0], {}
def do_GET(self):
logging.debug('do_GET: ' + self.path)
logging.debug('do_GET: %s', self.path)
try:
path, params = self.__path_params()
if path == '/mount':
@ -370,18 +356,16 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
self.__file_request(path)
return
logging.warning('Wrong path: ' + path)
logging.warning('Wrong path: %s', path)
raise HTTPError(HTTPStatus.NOT_FOUND, path)
except HTTPError as ex:
self.__error_response_get(ex.code, ex.reason)
except Exception as ex:
self.__error_response_get(
HTTPStatus.INTERNAL_SERVER_ERROR, str(ex)
)
self.__error_response_get(HTTPStatus.INTERNAL_SERVER_ERROR, str(ex))
logging.exception(ex)
def do_POST(self):
logging.debug('do_POST: ' + self.path)
logging.debug('do_POST: %s', self.path)
try:
path, params = self.__path_params()
@ -395,9 +379,7 @@ class PhotoImporterHandler(http.server.BaseHTTPRequestHandler):
except HTTPError as ex:
self.__error_response_post(ex.code, ex.reason)
except Exception as ex:
self.__error_response_post(
HTTPStatus.INTERNAL_SERVER_ERROR, str(ex)
)
self.__error_response_post(HTTPStatus.INTERNAL_SERVER_ERROR, str(ex))
logging.exception(ex)
@ -427,17 +409,14 @@ class PhotoImporterServer(http.server.HTTPServer):
def import_start(self, in_path, out_path):
logging.info('import_start: %s', in_path)
self.__importers[in_path] = importer.Importer(
self.__cfg, in_path, out_path, False
)
self.__importers[in_path] = importer.Importer(self.__cfg, in_path, out_path, False)
self.__importers[in_path].start()
def import_status(self, in_path):
if in_path in self.__importers:
return self.__importers[in_path].status()
else:
return None
return None
def import_done(self, in_path):
logging.info('import_done: %s', in_path)
@ -447,8 +426,7 @@ class PhotoImporterServer(http.server.HTTPServer):
def get_log(self, in_path):
if in_path in self.__importers:
return self.__importers[in_path].log_text()
else:
return ''
return ''
def args_parse():
@ -468,7 +446,7 @@ def main():
else:
logfile = cfg['server']['log_file']
log.initLogger(logfile)
log.init_logger(logfile)
try:
server = PhotoImporterServer(cfg)

4
setup.py
View File

@ -1,5 +1,5 @@
from setuptools import setup
import os
from setuptools import setup
def get_long_description():
@ -12,7 +12,7 @@ def get_long_description():
setup(
name='photo-importer',
version='1.2.4',
version='1.2.5',
description='Photo importer tool',
author='Alexander Bushnev',
author_email='Alexander@Bushnev.pro',