diff --git a/cps/__init__.py b/cps/__init__.py index 1200c2e1..fead0d2f 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -56,23 +56,26 @@ except ImportError: mimetypes.init() mimetypes.add_type('application/xhtml+xml', '.xhtml') mimetypes.add_type('application/epub+zip', '.epub') -mimetypes.add_type('application/fb2+zip', '.fb2') -mimetypes.add_type('application/x-mobipocket-ebook', '.mobi') -mimetypes.add_type('application/x-mobipocket-ebook', '.prc') +mimetypes.add_type('application/epub+zip', '.kepub') +mimetypes.add_type('text/xml', '.fb2') +mimetypes.add_type('application/octet-stream', '.mobi') +mimetypes.add_type('application/octet-stream', '.prc') mimetypes.add_type('application/vnd.amazon.ebook', '.azw') mimetypes.add_type('application/x-mobi8-ebook', '.azw3') -mimetypes.add_type('application/x-cbr', '.cbr') -mimetypes.add_type('application/x-cbz', '.cbz') -mimetypes.add_type('application/x-cbt', '.cbt') -mimetypes.add_type('application/x-cb7', '.cb7') +mimetypes.add_type('application/x-rar', '.cbr') +mimetypes.add_type('application/zip', '.cbz') +mimetypes.add_type('application/x-tar', '.cbt') +mimetypes.add_type('application/x-7z-compressed', '.cb7') mimetypes.add_type('image/vnd.djv', '.djv') +mimetypes.add_type('image/vnd.djv', '.djvu') mimetypes.add_type('application/mpeg', '.mpeg') -mimetypes.add_type('application/mpeg', '.mp3') +mimetypes.add_type('audio/mpeg', '.mp3') mimetypes.add_type('application/mp4', '.m4a') mimetypes.add_type('application/mp4', '.m4b') -mimetypes.add_type('application/ogg', '.ogg') +mimetypes.add_type('audio/ogg', '.ogg') mimetypes.add_type('application/ogg', '.oga') mimetypes.add_type('text/css', '.css') +mimetypes.add_type('application/x-ms-reader', '.lit') mimetypes.add_type('text/javascript; charset=UTF-8', '.js') log = logger.create() diff --git a/cps/admin.py b/cps/admin.py index a39b4342..f5b9e6c3 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -1777,7 +1777,7 @@ def _configuration_update_helper(): to_save["config_upload_formats"] = ','.join( helper.uniq([x.lstrip().rstrip().lower() for x in to_save["config_upload_formats"].split(',')])) _config_string(to_save, "config_upload_formats") - constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') + # constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") _config_string(to_save, "config_binariesdir") @@ -1827,6 +1827,7 @@ def _configuration_update_helper(): reboot_required |= reboot # security configuration + _config_checkbox(to_save, "config_check_extensions") _config_checkbox(to_save, "config_password_policy") _config_checkbox(to_save, "config_password_number") _config_checkbox(to_save, "config_password_lower") diff --git a/cps/config_sql.py b/cps/config_sql.py index 33dba623..044c12b5 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -172,6 +172,7 @@ class _Settings(_Base): config_ratelimiter = Column(Boolean, default=True) config_limiter_uri = Column(String, default="") config_limiter_options = Column(String, default="") + config_check_extensions = Column(Boolean, default=True) def __repr__(self): return self.__class__.__name__ @@ -371,7 +372,7 @@ class ConfigSQL(object): db_file = os.path.join(self.config_calibre_dir, 'metadata.db') have_metadata_db = os.path.isfile(db_file) self.db_configured = have_metadata_db - constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')] + # constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')] from . import cli_param if os.environ.get('FLASK_DEBUG'): logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG) diff --git a/cps/editbooks.py b/cps/editbooks.py index 802c3b09..957e97dc 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -23,7 +23,7 @@ import os from datetime import datetime import json -from shutil import copyfile +from shutil import copyfile, move from uuid import uuid4 from markupsafe import escape, Markup # dependency of flask from functools import wraps @@ -46,7 +46,7 @@ from .render_template import render_title_template from .usermanagement import login_required_if_no_ano from .kobo_sync_status import change_archived_books from .redirect import get_redirect_location - +from .file_helper import validate_mime_type editbook = Blueprint('edit-book', __name__) log = logger.create() @@ -118,14 +118,13 @@ def edit_book(book_id): # handle book title change title_change = handle_title_on_edit(book, to_save["book_title"]) # handle book author change - input_authors, author_change, renamed = handle_author_on_edit(book, to_save["author_name"]) + input_authors, author_change = handle_author_on_edit(book, to_save["author_name"]) if author_change or title_change: edited_books_id = book.id modify_date = True title_author_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), - input_authors[0], - renamed_author=renamed) + input_authors[0]) if title_author_error: flash(title_author_error, category="error") calibre_db.session.rollback() @@ -251,7 +250,7 @@ def upload(): if error: return error - db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta) + db_book, input_authors, title_dir = create_book_on_upload(modify_date, meta) # Comments need book id therefore only possible after flush modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) @@ -261,7 +260,6 @@ def upload(): if config.config_use_google_drive: helper.upload_new_file_gdrive(book_id, input_authors[0], - renamed_authors, title, title_dir, meta.file_path, @@ -271,8 +269,7 @@ def upload(): config.get_book_path(), input_authors[0], meta.file_path, - title_dir + meta.extension.lower(), - renamed_author=renamed_authors) + title_dir + meta.extension.lower()) move_coverfile(meta, db_book) @@ -405,9 +402,8 @@ def edit_list_book(param): ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}), mimetype='application/json') elif param == 'authors': - input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") - rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0], - renamed_author=renamed) + input_authors, __ = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") + rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0]) if not rename_error: ret = Response(json.dumps({ 'success': True, @@ -543,7 +539,7 @@ def table_xchange_author_title(): author_names.append(authr.name.replace('|', ',')) title_change = handle_title_on_edit(book, " ".join(author_names)) - input_authors, author_change, renamed = handle_author_on_edit(book, authors) + input_authors, author_change = handle_author_on_edit(book, authors) if author_change or title_change: edited_books_id = book.id modify_date = True @@ -553,8 +549,7 @@ def table_xchange_author_title(): if edited_books_id: # toDo: Handle error - edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0], - renamed_author=renamed) + edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0]) if modify_date: book.last_modified = datetime.utcnow() calibre_db.set_metadata_dirty(book.id) @@ -602,7 +597,9 @@ def identifier_list(to_save, book): return result -def prepare_authors(authr): +def prepare_authors(authr, calibre_path, gdrive=False): + if gdrive: + calibre_path = "" # handle authors input_authors = authr.split('&') # handle_authors(input_authors) @@ -614,18 +611,44 @@ def prepare_authors(authr): if input_authors == ['']: input_authors = [_('Unknown')] # prevent empty Author - renamed = list() for in_aut in input_authors: - renamed_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == in_aut).first() + renamed_author = calibre_db.session.query(db.Authors).filter(func.lower(db.Authors.name).ilike(in_aut)).first() if renamed_author and in_aut != renamed_author.name: - renamed.append(renamed_author.name) + old_author_name = renamed_author.name + # rename author in Database + create_objects_for_addition(renamed_author, in_aut,"author") + # rename all Books with this author as first author: + # rename all book author_sort strings with the new author name all_books = calibre_db.session.query(db.Books) \ .filter(db.Books.authors.any(db.Authors.name == renamed_author.name)).all() - sorted_renamed_author = helper.get_sorted_author(renamed_author.name) - sorted_old_author = helper.get_sorted_author(in_aut) for one_book in all_books: - one_book.author_sort = one_book.author_sort.replace(sorted_renamed_author, sorted_old_author) - return input_authors, renamed + # ToDo: check + sorted_old_author = helper.get_sorted_author(old_author_name) + sorted_renamed_author = helper.get_sorted_author(in_aut) + # change author sort path + try: + author_index = one_book.author_sort.index(sorted_old_author) + one_book.author_sort = one_book.author_sort.replace(sorted_old_author, sorted_renamed_author) + except ValueError: + log.error("Sorted author {} not found in database".format(sorted_old_author)) + author_index = -1 + # change book path if changed author is first author -> match on first position + if author_index == 0: + one_titledir = one_book.path.split('/')[1] + one_old_authordir = one_book.path.split('/')[0] + # rename author path only once per renamed author -> search all books with author name in book.path + # das muss einmal geschehen aber pro Buch geprüft werden ansonsten habe ich das Problem das vlt. 2 gleiche Ordner bis auf Groß/Kleinschreibung vorhanden sind im Umzug + new_author_dir = helper.rename_author_path(in_aut, one_old_authordir, renamed_author.name, calibre_path, gdrive) + one_book.path = os.path.join(new_author_dir, one_titledir).replace('\\', '/') + # rename all books in book data with the new author name and move corresponding files to new locations + # old_path = os.path.join(calibre_path, new_author_dir, one_titledir) + new_path = os.path.join(calibre_path, new_author_dir, one_titledir) + all_new_name = helper.get_valid_filename(one_book.title, chars=42) + ' - ' \ + + helper.get_valid_filename(renamed_author.name, chars=42) + # change location in database to new author/title path + helper.rename_all_files_on_change(one_book, new_path, new_path, all_new_name, gdrive) + + return input_authors def prepare_authors_on_upload(title, authr): @@ -636,12 +659,13 @@ def prepare_authors_on_upload(title, authr): flash(_("Uploaded book probably exists in the library, consider to change before upload new: ") + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning") - input_authors, renamed = prepare_authors(authr) + input_authors = prepare_authors(authr, config.get_book_path(), config.config_use_google_drive) sort_authors_list = list() db_author = None for inp in input_authors: - stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() + # stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() + stored_author = calibre_db.session.query(db.Authors).filter(func.lower(db.Authors.name).ilike(inp)).first() if not stored_author: if not db_author: db_author = db.Authors(inp, helper.get_sorted_author(inp), "") @@ -654,13 +678,13 @@ def prepare_authors_on_upload(title, authr): sort_author = stored_author.sort sort_authors_list.append(sort_author) sort_authors = ' & '.join(sort_authors_list) - return sort_authors, input_authors, db_author, renamed + return sort_authors, input_authors, db_author def create_book_on_upload(modify_date, meta): title = meta.title authr = meta.author - sort_authors, input_authors, db_author, renamed_authors = prepare_authors_on_upload(title, authr) + sort_authors, input_authors, db_author = prepare_authors_on_upload(title, authr) title_dir = helper.get_valid_filename(title, chars=96) author_dir = helper.get_valid_filename(db_author.name, chars=96) @@ -717,14 +741,20 @@ def create_book_on_upload(modify_date, meta): flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning") modify_date |= modification - return db_book, input_authors, title_dir, renamed_authors + return db_book, input_authors, title_dir def file_handling_on_upload(requested_file): # check if file extension is correct + allowed_extensions = config.config_upload_formats.split(',') + if requested_file: + if config.config_check_extensions and allowed_extensions != ['']: + if not validate_mime_type(requested_file, allowed_extensions): + flash(_("File type isn't allowed to be uploaded to this server"), category="error") + return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') if '.' in requested_file.filename: file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() - if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD: + if file_ext not in allowed_extensions and '' not in allowed_extensions: flash( _("File extension '%(ext)s' is not allowed to be uploaded to this server", ext=file_ext), category="error") @@ -1152,7 +1182,12 @@ def edit_cc_data(book_id, book, to_save, cc): def upload_single_file(file_request, book, book_id): # Check and handle Uploaded file requested_file = file_request.files.get('btn-upload-format', None) + allowed_extensions = config.config_upload_formats.split(',') if requested_file: + if config.config_check_extensions and allowed_extensions != ['']: + if not validate_mime_type(requested_file, allowed_extensions): + flash(_("File type isn't allowed to be uploaded to this server"), category="error") + return False # check for empty request if requested_file.filename != '': if not current_user.role_upload(): @@ -1160,7 +1195,7 @@ def upload_single_file(file_request, book, book_id): return False if '.' in requested_file.filename: file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() - if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD: + if file_ext not in allowed_extensions and '' not in allowed_extensions: flash(_("File extension '%(ext)s' is not allowed to be uploaded to this server", ext=file_ext), category="error") return False @@ -1177,7 +1212,8 @@ def upload_single_file(file_request, book, book_id): try: os.makedirs(filepath) except OSError: - flash(_("Failed to create path %(path)s (Permission denied).", path=filepath), category="error") + flash(_("Failed to create path %(path)s (Permission denied).", path=filepath), + category="error") return False try: requested_file.save(saved_filename) @@ -1247,7 +1283,7 @@ def handle_title_on_edit(book, book_title): def handle_author_on_edit(book, author_name, update_stored=True): change = False # handle author(s) - input_authors, renamed = prepare_authors(author_name) + input_authors = prepare_authors(author_name, config.get_book_path(), config.config_use_google_drive) # change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author') # Search for each author if author is in database, if not, author name and sorted author name is generated new @@ -1267,7 +1303,7 @@ def handle_author_on_edit(book, author_name, update_stored=True): change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author') - return input_authors, change, renamed + return input_authors, change def search_objects_remove(db_book_object, db_type, input_elements): @@ -1351,8 +1387,8 @@ def add_objects(db_book_object, db_object, db_session, db_type, add_elements): if db_no_case: # check for new case of element db_element = create_objects_for_addition(db_element, add_element, db_type) - else: - db_element = create_objects_for_addition(db_element, add_element, db_type) + #else: + # db_element = create_objects_for_addition(db_element, add_element, db_type) # add element to book db_book_object.append(db_element) diff --git a/cps/file_helper.py b/cps/file_helper.py index c714f5c2..8e2b6847 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -19,6 +19,18 @@ from tempfile import gettempdir import os import shutil +import zipfile +import mimetypes +import copy +from io import BytesIO +try: + import magic +except ImportError: + pass + +from . import logger + +log = logger.create() def get_temp_dir(): @@ -31,3 +43,29 @@ def get_temp_dir(): def del_temp_dir(): tmp_dir = os.path.join(gettempdir(), 'calibre_web') shutil.rmtree(tmp_dir) + + +def validate_mime_type(file_buffer, allowed_extensions): + mime = magic.Magic(mime=True) + allowed_mimetypes =list() + for x in allowed_extensions: + try: + allowed_mimetypes.append(mimetypes.types_map["." + x]) + except KeyError as e: + log.error("Unkown mimetype for Extension: {}".format(x)) + tmp_mime_type = mime.from_buffer(file_buffer.read()) + file_buffer.seek(0) + if any(mime_type in tmp_mime_type for mime_type in allowed_mimetypes): + return True + # Some epubs show up as zip mimetypes + elif "zip" in tmp_mime_type: + try: + with zipfile.ZipFile(BytesIO(file_buffer.read()), 'r') as epub: + file_buffer.seek(0) + if "mimetype" in epub.namelist(): + return True + except: + file_buffer.seek(0) + pass + + return False diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 93cfe1e1..926e5da3 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -581,7 +581,7 @@ def get_cover_via_gdrive(cover_path): session.add(permissionAdded) try: session.commit() - except OperationalError as ex: + except (OperationalError, IntegrityError) as ex: log.error_or_exception('Database error: {}'.format(ex)) session.rollback() return df.metadata.get('webContentLink') diff --git a/cps/helper.py b/cps/helper.py index df8fece7..abf5bb70 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -388,42 +388,27 @@ def delete_book_file(book, calibrepath, book_format=None): id=book.id, path=book.path) - -def clean_author_database(renamed_author, calibre_path="", local_book=None, gdrive=None): - valid_filename_authors = [get_valid_filename(r, chars=96) for r in renamed_author] - for r in renamed_author: - if local_book: - all_books = [local_book] +def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdrive=False): + for file_format in one_book.data: + if not gdrive: + if not os.path.exists(new_path): + os.makedirs(new_path) + shutil.move(os.path.normcase( + os.path.join(old_path, file_format.name + '.' + file_format.format.lower())), + os.path.normcase( + os.path.join(new_path, all_new_name + '.' + file_format.format.lower()))) else: - all_books = calibre_db.session.query(db.Books) \ - .filter(db.Books.authors.any(db.Authors.name == r)).all() - for book in all_books: - book_author_path = book.path.split('/')[0] - if book_author_path in valid_filename_authors or local_book: - new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == r).first() - all_new_authordir = get_valid_filename(new_author.name, chars=96) - all_titledir = book.path.split('/')[1] - all_new_path = os.path.join(calibre_path, all_new_authordir, all_titledir) - all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \ - + get_valid_filename(new_author.name, chars=42) - # change location in database to new author/title path - book.path = os.path.join(all_new_authordir, all_titledir).replace('\\', '/') - for file_format in book.data: - if not gdrive: - shutil.move(os.path.normcase(os.path.join(all_new_path, - file_format.name + '.' + file_format.format.lower())), - os.path.normcase(os.path.join(all_new_path, - all_new_name + '.' + file_format.format.lower()))) - else: - g_file = gd.getFileFromEbooksFolder(all_new_path, - file_format.name + '.' + file_format.format.lower()) - if g_file: - gd.moveGdriveFileRemote(g_file, all_new_name + '.' + file_format.format.lower()) - gd.updateDatabaseOnEdit(g_file['id'], all_new_name + '.' + file_format.format.lower()) - else: - log.error("File {} not found on gdrive" - .format(all_new_path, file_format.name + '.' + file_format.format.lower())) - file_format.name = all_new_name + g_file = gd.getFileFromEbooksFolder(old_path, + file_format.name + '.' + file_format.format.lower()) + if g_file: + gd.moveGdriveFileRemote(g_file, all_new_name + '.' + file_format.format.lower()) + gd.updateDatabaseOnEdit(g_file['id'], all_new_name + '.' + file_format.format.lower()) + else: + log.error("File {} not found on gdrive" + .format(old_path, file_format.name + '.' + file_format.format.lower())) + + # change name in Database + file_format.name = all_new_name def rename_all_authors(first_author, renamed_author, calibre_path="", localbook=None, gdrive=False): @@ -455,8 +440,32 @@ def rename_all_authors(first_author, renamed_author, calibre_path="", localbook= return new_authordir +def rename_author_path(first_author, old_author_dir, renamed_author, calibre_path="", gdrive=False): + # Create new_author_dir from parameter or from database + # Create new title_dir from database and add id + new_authordir = get_valid_filename(first_author, chars=96) + # new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == renamed_author).first() + # old_author_dir = get_valid_filename(old_author_name, chars=96) + new_author_rename_dir = get_valid_filename(renamed_author, chars=96) + if gdrive: + g_file = gd.getFileFromEbooksFolder(None, old_author_dir) + if g_file: + gd.moveGdriveFolderRemote(g_file, new_author_rename_dir) + else: + if os.path.isdir(os.path.join(calibre_path, old_author_dir)): + old_author_path = os.path.join(calibre_path, old_author_dir) + new_author_path = os.path.join(calibre_path, new_author_rename_dir) + try: + shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path)) + except OSError as ex: + log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex) + log.debug(ex, exc_info=True) + return _("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s", + src=old_author_path, dest=new_author_path, error=str(ex)) + return new_authordir + # Moves files in file storage during author/title rename, or from temp dir to file storage -def update_dir_structure_file(book_id, calibre_path, first_author, original_filepath, db_filename, renamed_author): +def update_dir_structure_file(book_id, calibre_path, original_filepath, db_filename): # get book database entry from id, if original path overwrite source with original_filepath local_book = calibre_db.get_book(book_id) if original_filepath: @@ -468,49 +477,47 @@ def update_dir_structure_file(book_id, calibre_path, first_author, original_file author_dir = local_book.path.split('/')[0] title_dir = local_book.path.split('/')[1] - # Create new_author_dir from parameter or from database - # Create new title_dir from database and add id - new_author_dir = rename_all_authors(first_author, renamed_author, calibre_path, local_book) - if first_author: - if first_author.lower() in [r.lower() for r in renamed_author]: - if os.path.isdir(os.path.join(calibre_path, new_author_dir)): - path = os.path.join(calibre_path, new_author_dir, title_dir) - new_title_dir = get_valid_filename(local_book.title, chars=96) + " (" + str(book_id) + ")" - if title_dir != new_title_dir or author_dir != new_author_dir or original_filepath: + if title_dir != new_title_dir or original_filepath: error = move_files_on_change(calibre_path, - new_author_dir, + author_dir, new_title_dir, local_book, db_filename, original_filepath, path) + new_path = os.path.join(calibre_path, author_dir, new_title_dir).replace('\\', '/') + all_new_name = get_valid_filename(local_book.title, chars=42) + ' - ' \ + + get_valid_filename(author_dir, chars=42) + # Book folder already moved, only files need to be renamed + rename_all_files_on_change(local_book, new_path, new_path, all_new_name) + if error: return error # Rename all files from old names to new names - return rename_files_on_change(first_author, renamed_author, local_book, original_filepath, path, calibre_path) + return False -def upload_new_file_gdrive(book_id, first_author, renamed_author, title, title_dir, original_filepath, filename_ext): +def upload_new_file_gdrive(book_id, first_author, title, title_dir, original_filepath, filename_ext): book = calibre_db.get_book(book_id) file_name = get_valid_filename(title, chars=42) + ' - ' + \ get_valid_filename(first_author, chars=42) + filename_ext - rename_all_authors(first_author, renamed_author, gdrive=True) gdrive_path = os.path.join(get_valid_filename(first_author, chars=96), title_dir + " (" + str(book_id) + ")") book.path = gdrive_path.replace("\\", "/") gd.uploadFileToEbooksFolder(os.path.join(gdrive_path, file_name).replace("\\", "/"), original_filepath) - return rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True) + return False # rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True) -def update_dir_structure_gdrive(book_id, first_author, renamed_author): +def update_dir_structure_gdrive(book_id): book = calibre_db.get_book(book_id) authordir = book.path.split('/')[0] titledir = book.path.split('/')[1] - new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True) + # new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True) + # new_authordir = get_valid_filename(book.title, chars=96) new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")" if titledir != new_titledir: @@ -522,21 +529,25 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author): else: return _('File %(file)s not found on Google Drive', file=book.path) # file not found - if authordir != new_authordir and authordir not in renamed_author: + '''if authordir != new_authordir: g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir) if g_file: gd.moveGdriveFolderRemote(g_file, new_authordir) book.path = new_authordir + '/' + book.path.split('/')[1] gd.updateDatabaseOnEdit(g_file['id'], book.path) else: - return _('File %(file)s not found on Google Drive', file=authordir) # file not found - + return _('File %(file)s not found on Google Drive', file=authordir) # file not found''' + if titledir != new_titledir: + all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \ + + get_valid_filename(authordir, chars=42) + rename_all_files_on_change(book, book.path, book.path, all_new_name, gdrive=True) # todo: Move filenames on gdrive # change location in database to new author/title path - book.path = os.path.join(new_authordir, new_titledir).replace('\\', '/') - return rename_files_on_change(first_author, renamed_author, book, gdrive=True) + # book.path = os.path.join(authordir, new_titledir).replace('\\', '/') + return False def move_files_on_change(calibre_path, new_authordir, new_titledir, localbook, db_filename, original_filepath, path): + new_authordir = get_valid_filename(new_authordir, chars=96) new_path = os.path.join(calibre_path, new_authordir, new_titledir) new_name = get_valid_filename(localbook.title, chars=96) + ' - ' + new_authordir try: @@ -575,15 +586,15 @@ def rename_files_on_change(first_author, calibre_path="", gdrive=False): # Rename all files from old names to new names - try: - clean_author_database(renamed_author, calibre_path, gdrive=gdrive) - if first_author and first_author not in renamed_author: - clean_author_database([first_author], calibre_path, local_book, gdrive) - if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0: - shutil.rmtree(os.path.dirname(path)) - except (OSError, FileNotFoundError) as ex: - log.error_or_exception("Error in rename file in path {}".format(ex)) - return _("Error in rename file in path: {}".format(str(ex))) + #try: + #clean_author_database(renamed_author, calibre_path, gdrive=gdrive) + #if first_author and first_author not in renamed_author: + # clean_author_database([first_author], calibre_path, local_book, gdrive) + #if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0: + # shutil.rmtree(os.path.dirname(path)) + #except (OSError, FileNotFoundError) as ex: + # log.error_or_exception("Error in rename file in path {}".format(ex)) + # return _("Error in rename file in path: {}".format(str(ex))) return False @@ -648,12 +659,6 @@ def generate_random_password(min_length): return ''.join(password) -'''def generate_random_password(min_length): - s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?" - passlen = min_length - return "".join(s[c % len(s)] for c in os.urandom(passlen))''' - - def uniq(inpt): output = [] inpt = [" ".join(inp.split()) for inp in inpt] @@ -717,17 +722,14 @@ def update_dir_structure(book_id, calibre_path, first_author=None, # change author of book to this author original_filepath=None, - db_filename=None, - renamed_author=None): - renamed_author = renamed_author or [] + db_filename=None): if config.config_use_google_drive: - return update_dir_structure_gdrive(book_id, first_author, renamed_author) + return update_dir_structure_gdrive(book_id, first_author) else: return update_dir_structure_file(book_id, calibre_path, - first_author, original_filepath, - db_filename, renamed_author) + db_filename) def delete_book(book, calibrepath, book_format): diff --git a/cps/render_template.py b/cps/render_template.py index 68b46459..89e067d0 100644 --- a/cps/render_template.py +++ b/cps/render_template.py @@ -112,7 +112,7 @@ def render_title_template(*args, **kwargs): sidebar, simple = get_sidebar_config(kwargs) try: return render_template(instance=config.config_calibre_web_title, sidebar=sidebar, simple=simple, - accept=constants.EXTENSIONS_UPLOAD, + accept=config.config_upload_formats.split(','), *args, **kwargs) except PermissionError: log.error("No permission to access {} file.".format(args[0])) diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 0d0a695f..77353241 100755 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -373,10 +373,14 @@
- +
+
+ + +
- {{ _('Read') }} + {{ _('Read') }}

@@ -264,7 +264,7 @@ - {{ _('Archived') }} + {{ _('Archive') }}

diff --git a/cps/uploader.py b/cps/uploader.py index 8f20762f..bb3a49b7 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -23,7 +23,7 @@ from flask_babel import gettext as _ from . import logger, comic, isoLanguages from .constants import BookMeta from .helper import split_authors -from .file_helper import get_temp_dir +from .file_helper import get_temp_dir, validate_mime_type log = logger.create() diff --git a/cps/web.py b/cps/web.py index 3197d77f..abe62c96 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1582,7 +1582,8 @@ def read_book(book_id, book_format): return render_title_template('readtxt.html', txtfile=book_id, title=book.title) elif book_format.lower() in ["djvu", "djv"]: log.debug("Start djvu reader for %d", book_id) - return render_title_template('readdjvu.html', djvufile=book_id, title=book.title, extension=book_format.lower()) + return render_title_template('readdjvu.html', djvufile=book_id, title=book.title, + extension=book_format.lower()) else: for fileExt in constants.EXTENSIONS_AUDIO: if book_format.lower() == fileExt: diff --git a/requirements.txt b/requirements.txt index db992a59..e7062a3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ advocate>=1.0.0,<1.1.0 Flask-Limiter>=2.3.0,<3.6.0 regex>=2022.3.2,<2024.6.25 bleach>=6.0.0,<6.2.0 +python-magic>=0.4.27,<0.5.0 diff --git a/setup.cfg b/setup.cfg index fbda4e6a..dcbbac10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,6 +60,7 @@ install_requires = Flask-Limiter>=2.3.0,<3.6.0 regex>=2022.3.2,<2024.2.25 bleach>=6.0.0,<6.2.0 + python-magic>=0.4.27,<0.5.0 [options.packages.find] diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 4cc58c7c..bb9f8214 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-06-19 18:47:42

+

Start Time: 2024-06-28 20:08:21

-

Stop Time: 2024-06-20 01:41:47

+

Stop Time: 2024-06-29 02:31:07

-

Duration: 5h 43 min

+

Duration: 5h 10 min

@@ -234,11 +234,11 @@ - + TestBackupMetadata 21 - 21 - 0 + 19 + 2 0 0 @@ -257,11 +257,38 @@ - +
TestBackupMetadata - test_backup_change_book_author
- PASS + +
+ FAIL +
+ + + + @@ -338,11 +365,32 @@ - +
TestBackupMetadata - test_backup_change_book_title
- PASS + +
+ FAIL +
+ + + + @@ -462,143 +510,57 @@ - - TestCli - 13 - 13 + + _ErrorHolder + 1 0 0 + 1 0 - Detail + Detail - + -
TestCli - test_already_started
+
setUpClass (test_cli)
- PASS - - - - - - -
TestCli - test_bind_to_single_interface
+ +
+ ERROR +
+ + + - PASS - - - - - - -
TestCli - test_change_password
- - PASS - - - - - - -
TestCli - test_cli_SSL_files
- - PASS - - - - - - -
TestCli - test_cli_different_folder
- - PASS - - - - - - -
TestCli - test_cli_different_settings_database
- - PASS - - - - - - -
TestCli - test_dryrun_update
- - PASS - - - - - - -
TestCli - test_enable_reconnect
- - PASS - - - - - - -
TestCli - test_environ_port_setting
- - PASS - - - - - - -
TestCli - test_logfile
- - PASS - - - - - - -
TestCli - test_no_database
- - PASS - - - - - - -
TestCli - test_settingsdb_not_writeable
- - PASS - - - - - - -
TestCli - test_writeonly_static_files
- - PASS - + TestCliGdrivedb 4 - 4 0 + 4 0 0 @@ -608,38 +570,131 @@ - +
TestCliGdrivedb - test_cli_gdrive_folder
- PASS + +
+ FAIL +
+ + + + - +
TestCliGdrivedb - test_cli_gdrive_location
- PASS + +
+ FAIL +
+ + + + - +
TestCliGdrivedb - test_gdrive_db_nonwrite
- PASS + +
+ FAIL +
+ + + + - +
TestCliGdrivedb - test_no_database
- PASS + +
+ FAIL +
+ + + + @@ -1127,33 +1182,11 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cb7
- -
- FAIL -
- - - - + PASS @@ -1202,11 +1235,31 @@ AssertionError: 'Test 执 to' != 'book' - +
TestEditAdditionalBooks - test_writeonly_path
- PASS + +
+ FAIL +
+ + + + @@ -1239,11 +1292,11 @@ AssertionError: 'Test 执 to' != 'book' - + TestEditBooks 38 - 36 - 0 + 35 + 1 0 2 @@ -1262,11 +1315,31 @@ AssertionError: 'Test 执 to' != 'book' - +
TestEditBooks - test_edit_author
- PASS + +
+ FAIL +
+ + + + @@ -1632,13 +1705,13 @@ AssertionError: 'Test 执 to' != 'book' TestEditAuthors - 6 - 6 + 7 + 7 0 0 0 - Detail + Detail @@ -1690,6 +1763,15 @@ AssertionError: 'Test 执 to' != 'book' + +
TestEditAuthors - test_rename_author_accent_onupload
+ + PASS + + + + +
TestEditAuthors - test_rename_capital_on_upload
@@ -1699,11 +1781,11 @@ AssertionError: 'Test 执 to' != 'book' - + TestEditAuthorsGdrive 6 - 6 0 + 6 0 0 @@ -1713,56 +1795,178 @@ AssertionError: 'Test 执 to' != 'book' - +
TestEditAuthorsGdrive - test_change_capital_co_author
- PASS + +
+ FAIL +
+ + + + - +
TestEditAuthorsGdrive - test_change_capital_one_author_one_book
- PASS + +
+ FAIL +
+ + + + - +
TestEditAuthorsGdrive - test_change_capital_one_author_two_books
- PASS + +
+ FAIL +
+ + + + - +
TestEditAuthorsGdrive - test_change_capital_rename_co_author
- PASS + +
+ FAIL +
+ + + + - +
TestEditAuthorsGdrive - test_change_capital_rename_two_co_authors
- PASS + +
+ FAIL +
+ + + + - +
TestEditAuthorsGdrive - test_rename_capital_on_upload
- PASS + +
+ FAIL +
+ + + + @@ -1989,11 +2193,11 @@ AssertionError: 0.0 not greater than or equal to 0.05 - + TestEditBooksOnGdrive 18 - 18 - 0 + 16 + 2 0 0 @@ -2012,11 +2216,33 @@ AssertionError: 0.0 not greater than or equal to 0.05 - +
TestEditBooksOnGdrive - test_edit_author
- PASS + +
+ FAIL +
+ + + + @@ -2138,11 +2364,33 @@ AssertionError: 0.0 not greater than or equal to 0.05 - +
TestEditBooksOnGdrive - test_edit_title
- PASS + +
+ FAIL +
+ + + + @@ -2938,13 +3186,13 @@ AssertionError: 0.0 not greater than or equal to 0.05 TestSecurity - 4 - 4 + 5 + 5 0 0 0 - Detail + Detail @@ -2978,6 +3226,15 @@ AssertionError: 0.0 not greater than or equal to 0.05 + +
TestSecurity - test_redis_backend
+ + PASS + + + + +
TestSecurity - test_register_limit
@@ -3940,11 +4197,11 @@ ModuleNotFoundError: No module named 'build_release' - + TestReadOnlyDatabase 1 - 1 0 + 1 0 0 @@ -3954,11 +4211,31 @@ ModuleNotFoundError: No module named 'build_release' - +
TestReadOnlyDatabase - test_readonly_path
- PASS + +
+ FAIL +
+ + + + @@ -5592,10 +5869,10 @@ ModuleNotFoundError: No module named 'build_release' Total - 492 - 478 - 3 - 1 + 482 + 451 + 19 + 2 10   @@ -5720,7 +5997,7 @@ ModuleNotFoundError: No module named 'build_release' lxml - 5.1.1 + 5.2.2 Basic @@ -5732,7 +6009,13 @@ ModuleNotFoundError: No module named 'build_release' pypdf - 4.0.2 + 4.2.0 + Basic + + + + python-magic + 0.4.27 Basic @@ -5744,7 +6027,7 @@ ModuleNotFoundError: No module named 'build_release' regex - 2023.12.25 + 2024.5.15 Basic @@ -5786,7 +6069,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestBackupMetadataGdrive @@ -5816,7 +6099,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestCliGdrivedb @@ -5846,7 +6129,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestEbookConvertCalibreGDrive @@ -5876,7 +6159,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestEbookConvertGDriveKepubify @@ -5910,12 +6193,6 @@ ModuleNotFoundError: No module named 'build_release' TestEditAdditionalBooks - - py7zr - 0.21.0 - TestEditAdditionalBooks - - rarfile 4.2 @@ -5930,7 +6207,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestEditAuthorsGdrive @@ -5966,7 +6243,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestEditBooksOnGdrive @@ -6008,7 +6285,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestEmbedMetadataGdrive @@ -6038,7 +6315,7 @@ ModuleNotFoundError: No module named 'build_release' google-api-python-client - 2.134.0 + 2.135.0 TestSetupGdrive @@ -6128,7 +6405,7 @@ ModuleNotFoundError: No module named 'build_release'