From dd30ac4fbdc9699dc4462b1a9a7efeddb53ffe3a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Mar 2021 11:32:50 +0100 Subject: [PATCH] Added thumbnails Merge remote-tracking branch 'cover/thumbnails' into development # Conflicts: # cps/admin.py # cps/templates/layout.html # cps/ub.py # cps/web.py Update join for sqlalchemy 1.4 --- .gitignore | 1 + cps.py | 5 + cps/__init__.py | 3 +- cps/admin.py | 19 +- cps/constants.py | 1 + cps/db.py | 13 +- cps/editbooks.py | 2 + cps/fs.py | 61 +++++ cps/helper.py | 89 ++++++- cps/jinjia.py | 22 ++ cps/schedule.py | 36 +++ cps/services/background_scheduler.py | 52 ++++ cps/services/worker.py | 17 +- cps/shelf.py | 2 +- cps/static/css/caliBlur.css | 82 ++++-- cps/static/js/main.js | 12 + cps/tasks/database.py | 49 ++++ cps/tasks/thumbnail.py | 366 +++++++++++++++++++++++++++ cps/templates/admin.html | 36 ++- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 13 + cps/templates/book_edit.html | 3 +- cps/templates/detail.html | 2 +- cps/templates/discover.html | 3 +- cps/templates/fragment.html | 1 + cps/templates/grid.html | 3 +- cps/templates/index.html | 5 +- cps/templates/layout.html | 1 + cps/templates/search.html | 3 +- cps/templates/shelf.html | 3 +- cps/ub.py | 34 +++ cps/web.py | 96 +++++-- requirements.txt | 1 + 33 files changed, 965 insertions(+), 73 deletions(-) create mode 100644 cps/fs.py create mode 100644 cps/schedule.py create mode 100644 cps/services/background_scheduler.py create mode 100644 cps/tasks/database.py create mode 100644 cps/tasks/thumbnail.py create mode 100644 cps/templates/book_cover.html diff --git a/.gitignore b/.gitignore index 109de4ef..903cfd36 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ vendor/ # calibre-web *.db *.log +cps/cache .idea/ *.bak diff --git a/cps.py b/cps.py index 737b0d97..19ca89b8 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.schedule import register_jobs try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,10 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + # Register scheduled jobs + register_jobs() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index 627cca0b..30029428 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -96,7 +96,7 @@ def create_app(): app.instance_path = app.instance_path.decode('utf-8') if os.environ.get('FLASK_DEBUG'): - cache_buster.init_cache_busting(app) + cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') if sys.version_info < (3, 0): @@ -121,6 +121,7 @@ def create_app(): return app + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings diff --git a/cps/admin.py b/cps/admin.py index 78cfebf1..cd548bfb 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,7 +40,7 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_ -from . import constants, logger, helper, services, isoLanguages +from . import constants, logger, helper, services, isoLanguages, fs from .cli import filepicker from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash @@ -164,6 +164,23 @@ def shutdown(): return json.dumps(showtext), 400 +@admi.route("/clear-cache") +@login_required +@admin_required +def clear_cache(): + cache_type = request.args.get('cache_type'.strip()) + showtext = {} + + if cache_type == fs.CACHE_TYPE_THUMBNAILS: + log.info('clearing cover thumbnail cache') + showtext['text'] = _(u'Cleared cover thumbnail cache') + helper.clear_cover_thumbnail_cache() + return json.dumps(showtext) + + showtext['text'] = _(u'Unknown command') + return json.dumps(showtext) + + @admi.route("/admin/view") @login_required @admin_required diff --git a/cps/constants.py b/cps/constants.py index e9c26cb1..0eb94709 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -37,6 +37,7 @@ else: STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') +CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/db.py b/cps/db.py index ac59ac2b..f43cc811 100644 --- a/cps/db.py +++ b/cps/db.py @@ -620,15 +620,18 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books) + .limit(self.config.config_random_books) \ + .all() else: randm = false() off = int(int(pagesize) * (page - 1)) - query = self.session.query(database) \ - .filter(db_filter) \ + query = self.session.query(database) + if len(join) == 3: + query = query.join(join[0], join[1]).join(join[2], isouter=True) + elif len(join) == 2: + query = query.join(join[0], join[1], isouter=True) + query = query.filter(db_filter)\ .filter(self.common_filters(allow_show_archived)) - if len(join): - query = query.join(*join, isouter=True) entries = list() pagination = list() try: diff --git a/cps/editbooks.py b/cps/editbooks.py index 28cad5c5..b7f496d0 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -614,6 +614,7 @@ def upload_cover(request, book): abort(403) ret, message = helper.save_cover(requested_file, book.path) if ret is True: + helper.clear_cover_thumbnail_cache(book.id) return True else: flash(message, category="error") @@ -710,6 +711,7 @@ def edit_book(book_id): if result is True: book.has_cover = 1 modif_date = True + helper.clear_cover_thumbnail_cache(book.id) else: flash(error, category="error") diff --git a/cps/fs.py b/cps/fs.py new file mode 100644 index 00000000..699d5991 --- /dev/null +++ b/cps/fs.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +from .constants import CACHE_DIR +from os import listdir, makedirs, remove +from os.path import isdir, isfile, join +from shutil import rmtree + +CACHE_TYPE_THUMBNAILS = 'thumbnails' + + +class FileSystem: + _instance = None + _cache_dir = CACHE_DIR + + def __new__(cls): + if cls._instance is None: + cls._instance = super(FileSystem, cls).__new__(cls) + return cls._instance + + def get_cache_dir(self, cache_type=None): + if not isdir(self._cache_dir): + makedirs(self._cache_dir) + + if cache_type and not isdir(join(self._cache_dir, cache_type)): + makedirs(join(self._cache_dir, cache_type)) + + return join(self._cache_dir, cache_type) if cache_type else self._cache_dir + + def get_cache_file_path(self, filename, cache_type=None): + return join(self.get_cache_dir(cache_type), filename) if filename else None + + def list_cache_files(self, cache_type=None): + path = self.get_cache_dir(cache_type) + return [file for file in listdir(path) if isfile(join(path, file))] + + def delete_cache_dir(self, cache_type=None): + if not cache_type and isdir(self._cache_dir): + rmtree(self._cache_dir) + if cache_type and isdir(join(self._cache_dir, cache_type)): + rmtree(join(self._cache_dir, cache_type)) + + def delete_cache_file(self, filename, cache_type=None): + if isfile(join(self.get_cache_dir(cache_type), filename)): + remove(join(self.get_cache_dir(cache_type), filename)) diff --git a/cps/helper.py b/cps/helper.py index e18ae33b..e3c79dea 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -52,12 +52,13 @@ except ImportError: from . import calibre_db from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, ub +from . import logger, config, get_locale, db, fs, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS from .tasks.mail import TaskEmail +from .tasks.thumbnail import TaskClearCoverThumbnailCache log = logger.create() @@ -514,12 +515,32 @@ def update_dir_stucture(book_id, calibrepath, first_author=None, orignal_filepat def delete_book(book, calibrepath, book_format): + clear_cover_thumbnail_cache(book.id) if config.config_use_google_drive: return delete_book_gdrive(book, book_format) else: return delete_book_file(book, calibrepath, book_format) +def get_thumbnails_for_books(books): + books_with_covers = list(filter(lambda b: b.has_cover, books)) + book_ids = list(map(lambda b: b.id, books_with_covers)) + cache = fs.FileSystem() + thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.filename.in_(thumbnail_files))\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .all() + + +def get_thumbnails_for_book_series(series): + books = list(map(lambda s: s[0], series)) + return get_thumbnails_for_books(books) + + def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -532,14 +553,54 @@ def get_book_cover(book_id): return get_book_cover_internal(book, use_generic_cover_on_failure=True) -def get_book_cover_with_uuid(book_uuid, - use_generic_cover_on_failure=True): +def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure): +def get_cached_book_cover(cache_id): + parts = cache_id.split('_') + book_uuid = parts[0] if len(parts) else None + resolution = parts[2] if len(parts) > 2 else None + book = calibre_db.get_book_by_uuid(book_uuid) if book_uuid else None + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + + +def get_cached_book_cover_thumbnail(cache_id): + parts = cache_id.split('_') + thumbnail_uuid = parts[0] if len(parts) else None + thumbnail = None + if thumbnail_uuid: + thumbnail = ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ + .first() + + if thumbnail and thumbnail.expiration > datetime.utcnow(): + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + elif thumbnail: + book = calibre_db.get_book(thumbnail.book_id) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) + + else: + return get_cover_on_failure(True) + + +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: + + # Send the book cover thumbnail if it exists in cache + if resolution: + thumbnail = get_book_cover_thumbnail(book, resolution) + if thumbnail: + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + # Send the book cover from Google Drive if configured if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -550,9 +611,11 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): else: log.error('%s/cover.jpg not found on Google Drive', book.path) return get_cover_on_failure(use_generic_cover_on_failure) - except Exception as e: - log.debug_or_exception(e) + except Exception as ex: + log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) + + # Send the book cover from the Calibre directory else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): @@ -563,6 +626,16 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): return get_cover_on_failure(use_generic_cover_on_failure) +def get_book_cover_thumbnail(book, resolution): + if book and book.has_cover: + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book.id)\ + .filter(ub.Thumbnail.resolution == resolution)\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: @@ -820,3 +893,7 @@ def get_download_link(book_id, book_format, client): return do_download_file(book, book_format, client, data1, headers) else: abort(404) + + +def clear_cover_thumbnail_cache(book_id=None): + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id)) diff --git a/cps/jinjia.py b/cps/jinjia.py index 688d1fba..b2479adc 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -128,8 +128,30 @@ def formatseriesindex_filter(series_index): return series_index return 0 + @jinjia.app_template_filter('uuidfilter') def uuidfilter(var): return uuid4() +@jinjia.app_template_filter('book_cover_cache_id') +def book_cover_cache_id(book, resolution=None): + timestamp = int(book.last_modified.timestamp() * 1000) + cache_bust = str(book.uuid) + '_' + str(timestamp) + return cache_bust if not resolution else cache_bust + '_' + str(resolution) + + +@jinjia.app_template_filter('get_book_thumbnails') +def get_book_thumbnails(book_id, thumbnails=None): + return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() + + +@jinjia.app_template_filter('get_book_thumbnail_srcset') +def get_book_thumbnail_srcset(thumbnails): + srcset = list() + for thumbnail in thumbnails: + timestamp = int(thumbnail.generated_at.timestamp() * 1000) + cache_id = str(thumbnail.uuid) + '_' + str(timestamp) + url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) + srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py new file mode 100644 index 00000000..7ee43410 --- /dev/null +++ b/cps/schedule.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from .services.background_scheduler import BackgroundScheduler +from .tasks.database import TaskReconnectDatabase +from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails + + +def register_jobs(): + scheduler = BackgroundScheduler() + + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') + + # Cleanup book cover cache every 6 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') + + # Reconnect metadata.db every 4 hours + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py new file mode 100644 index 00000000..efa57379 --- /dev/null +++ b/cps/services/background_scheduler.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import atexit + +from .. import logger +from .worker import WorkerThread +from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + + +class BackgroundScheduler: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(BackgroundScheduler, cls).__new__(cls) + + scheduler = BScheduler() + atexit.register(lambda: scheduler.shutdown()) + + cls.log = logger.create() + cls.scheduler = scheduler + cls.scheduler.start() + + return cls._instance + + def add(self, func, trigger, **trigger_args): + self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + + def add_task(self, user, task, trigger, **trigger_args): + def scheduled_task(): + worker_task = task() + self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + WorkerThread.add(user, worker_task) + + self.add(func=scheduled_task, trigger=trigger, **trigger_args) diff --git a/cps/services/worker.py b/cps/services/worker.py index 072674a0..2b6816db 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -35,7 +35,6 @@ def _get_main_thread(): raise Exception("main thread not found?!") - class ImprovedQueue(queue.Queue): def to_list(self): """ @@ -45,7 +44,8 @@ class ImprovedQueue(queue.Queue): with self.mutex: return list(self.queue) -#Class for all worker tasks in the background + +# Class for all worker tasks in the background class WorkerThread(threading.Thread): _instance = None @@ -127,6 +127,10 @@ class WorkerThread(threading.Thread): # CalibreTask.start() should wrap all exceptions in it's own error handling item.task.start(self) + # remove self_cleanup tasks from list + if item.task.self_cleanup: + self.dequeued.remove(item) + self.queue.task_done() @@ -141,6 +145,7 @@ class CalibreTask: self.end_time = None self.message = message self.id = uuid.uuid4() + self.self_cleanup = False @abc.abstractmethod def run(self, worker_thread): @@ -209,6 +214,14 @@ class CalibreTask: # todo: throw error if outside of [0,1] self._progress = x + @property + def self_cleanup(self): + return self._self_cleanup + + @self_cleanup.setter + def self_cleanup(self, is_self_cleanup): + self._self_cleanup = is_self_cleanup + def _handleError(self, error_message): self.stat = STAT_FAIL self.progress = 1 diff --git a/cps/shelf.py b/cps/shelf.py index 5c6037ac..7b00c32b 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -403,7 +403,7 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): db.Books, ub.BookShelf.shelf == shelf_id, [ub.BookShelf.order.asc()], - ub.BookShelf,ub.BookShelf.book_id == db.Books.id) + ub.BookShelf, ub.BookShelf.book_id == db.Books.id) # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web wrong_entries = calibre_db.session.query(ub.BookShelf)\ .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True)\ diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index 14e5c286..aa747c0b 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -5148,7 +5148,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head pointer-events: none } -#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { +#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #ClearCacheDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { cursor: pointer } @@ -5235,7 +5235,7 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d margin-bottom: 20px } -body.admin:not(.modal-open) .btn-default { +body.admin .btn-default { margin-bottom: 10px } @@ -5466,7 +5466,7 @@ body.admin.modal-open .navbar { z-index: 0 !important } -#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal { +#RestartDialog, #ShutdownDialog, #StatusDialog, #ClearCacheDialog, #deleteModal { top: 0; overflow: hidden; padding-top: 70px; @@ -5476,7 +5476,7 @@ body.admin.modal-open .navbar { background: rgba(0, 0, 0, .5) } -#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before { +#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #ClearCacheDialog:before, #deleteModal:before { content: "\E208"; padding-right: 10px; display: block; @@ -5498,18 +5498,18 @@ body.admin.modal-open .navbar { z-index: 99 } -#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { +#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { -webkit-transform: translate(0, 0); -ms-transform: translate(0, 0); transform: translate(0, 0) } -#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { +#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { width: 450px; margin: auto } -#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { +#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-height: calc(100% - 90px); -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); box-shadow: 0 5px 15px rgba(0, 0, 0, .5); @@ -5520,7 +5520,7 @@ body.admin.modal-open .navbar { width: 450px } -#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { +#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { padding: 15px 20px; border-radius: 3px 3px 0 0; line-height: 1.71428571; @@ -5533,7 +5533,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { padding-right: 10px; font-size: 18px; color: #999; @@ -5557,6 +5557,11 @@ body.admin.modal-open .navbar { font-family: plex-icons-new, serif } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before { + content: "\EA15"; + font-family: plex-icons-new, serif +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:before { content: "\EA6D"; font-family: plex-icons-new, serif @@ -5580,6 +5585,12 @@ body.admin.modal-open .navbar { font-size: 20px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:after { + content: "Clear Cover Thumbnail Cache"; + display: inline-block; + font-size: 20px +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:after { content: "Delete Book"; display: inline-block; @@ -5610,7 +5621,17 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body { + padding: 20px 20px 10px; + font-size: 16px; + line-height: 1.6em; + font-family: Open Sans Regular, Helvetica Neue, Helvetica, Arial, sans-serif; + color: #eee; + background: #282828; + text-align: left +} + +#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { padding: 20px 20px 0 0; font-size: 16px; line-height: 1.6em; @@ -5619,7 +5640,7 @@ body.admin.modal-open .navbar { background: #282828 } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { float: right; z-index: 9; position: relative; @@ -5655,6 +5676,18 @@ body.admin.modal-open .navbar { border-radius: 3px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > #clear_cache { + float: right; + z-index: 9; + position: relative; + margin: 25px 0 0 10px; + min-width: 80px; + padding: 10px 18px; + font-size: 16px; + line-height: 1.33; + border-radius: 3px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { float: right; z-index: 9; @@ -5675,11 +5708,15 @@ body.admin.modal-open .navbar { margin: 55px 0 0 10px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache) { + margin: 25px 0 0 10px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { margin: 0 0 0 10px } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { background-color: hsla(0, 0%, 100%, .3) } @@ -5713,6 +5750,21 @@ body.admin.modal-open .navbar { box-shadow: 0 5px 15px rgba(0, 0, 0, .5) } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body:after { + content: ''; + position: absolute; + width: 100%; + height: 72px; + background-color: #323232; + border-radius: 0 0 3px 3px; + left: 0; + margin-top: 10px; + z-index: 0; + border-top: 1px solid #222; + -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); + box-shadow: 0 5px 15px rgba(0, 0, 0, .5) +} + #deleteButton { position: fixed; top: 60px; @@ -7299,11 +7351,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. background-color: transparent !important } - #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { + #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { max-width: calc(100vw - 40px) } - #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { + #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-width: calc(100vw - 40px); left: 0 } @@ -7453,7 +7505,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. padding: 30px 15px } - #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { + #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { left: auto; right: 34px } diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 834b9b30..51d6095d 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -430,6 +430,18 @@ $(function() { } }); }); + $("#clear_cache").click(function () { + $("#spinner3").show(); + $.ajax({ + dataType: "json", + url: window.location.pathname + "/../../clear-cache", + data: {"cache_type":"thumbnails"}, + success: function(data) { + $("#spinner3").hide(); + $("#ClearCacheDialog").modal("hide"); + } + }); + }); // Init all data control handlers to default $("input[data-control]").trigger("change"); diff --git a/cps/tasks/database.py b/cps/tasks/database.py new file mode 100644 index 00000000..11f0186d --- /dev/null +++ b/cps/tasks/database.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from cps import config, logger +from cps.services.worker import CalibreTask + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + + +class TaskReconnectDatabase(CalibreTask): + def __init__(self, task_message=u'Reconnecting Calibre database'): + super(TaskReconnectDatabase, self).__init__(task_message) + self.log = logger.create() + self.listen_address = config.get_config_ipaddress() + self.listen_port = config.config_port + + def run(self, worker_thread): + address = self.listen_address if self.listen_address else 'localhost' + port = self.listen_port if self.listen_port else 8083 + + try: + urlopen('http://' + address + ':' + str(port) + '/reconnect') + self._handleSuccess() + except Exception as ex: + self._handleError(u'Unable to reconnect Calibre database: ' + str(ex)) + + @property + def name(self): + return "Reconnect Database" diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py new file mode 100644 index 00000000..70ddc06b --- /dev/null +++ b/cps/tasks/thumbnail.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from cps import config, db, fs, gdriveutils, logger, ub +from cps.services.worker import CalibreTask +from datetime import datetime, timedelta +from sqlalchemy import func + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + +THUMBNAIL_RESOLUTION_1X = 1 +THUMBNAIL_RESOLUTION_2X = 2 + + +class TaskGenerateCoverThumbnails(CalibreTask): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + super(TaskGenerateCoverThumbnails, self).__init__(task_message) + self.limit = limit + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + THUMBNAIL_RESOLUTION_1X, + THUMBNAIL_RESOLUTION_2X + ] + + def run(self, worker_thread): + if self.calibre_db.session and use_IM: + expired_thumbnails = self.get_expired_thumbnails() + thumbnail_book_ids = self.get_thumbnail_book_ids() + books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) + + count = len(books_without_thumbnails) + if count == 0: + # Do not display this task on the frontend if there are no covers to update + self.self_cleanup = True + + for i, book in enumerate(books_without_thumbnails): + for resolution in self.resolutions: + expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( + book, + resolution, + expired_thumbnails + ) + if expired_thumbnail: + self.update_book_thumbnail(book, expired_thumbnail) + else: + self.create_book_thumbnail(book, resolution) + + self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count) + self.progress = (1.0 / count) * i + + self._handleSuccess() + self.app_db_session.remove() + + def get_expired_thumbnails(self): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.expiration < datetime.utcnow())\ + .all() + + def get_thumbnail_book_ids(self): + return self.app_db_session\ + .query(ub.Thumbnail.book_id)\ + .group_by(ub.Thumbnail.book_id)\ + .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ + .distinct() + + def get_books_without_thumbnails(self, thumbnail_book_ids): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.id.notin_(thumbnail_book_ids))\ + .limit(self.limit)\ + .all() + + def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails): + for thumbnail in expired_thumbnails: + if thumbnail.book_id == book.id and thumbnail.resolution == resolution: + return thumbnail + + return None + + def update_book_thumbnail(self, book, thumbnail): + thumbnail.generated_at = datetime.utcnow() + thumbnail.expiration = datetime.utcnow() + timedelta(days=30) + + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def create_book_thumbnail(self, book, resolution): + thumbnail = ub.Thumbnail() + thumbnail.book_id = book.id + thumbnail.format = 'jpeg' + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_book_thumbnail(self, book, thumbnail): + if book and thumbnail: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + except Exception as ex: + # Bubble exception to calling function + self.log.info(u'Error generating thumbnail file: ' + str(ex)) + raise ex + finally: + stream.close() + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + + def get_thumbnail_height(self, thumbnail): + return int(225 * thumbnail.resolution) + + def get_thumbnail_width(self, height, img): + percent = (height / float(img.height)) + return int((float(img.width) * float(percent))) + + @property + def name(self): + return "ThumbnailsGenerate" + + +class TaskSyncCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Syncing cover thumbnail cache'): + super(TaskSyncCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + + def run(self, worker_thread): + cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + + # Expire thumbnails in the database if the cached file is missing + # This case will happen if a user deletes the cache dir or cached files + if self.app_db_session: + self.expire_missing_thumbnails(cached_thumbnail_files) + self.progress = 0.25 + + # Delete thumbnails in the database if the book has been removed + # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem + if self.app_db_session and self.calibre_db: + book_ids = self.get_book_ids() + self.delete_thumbnails_for_missing_books(book_ids) + self.progress = 0.50 + + # Expire thumbnails in the database if their corresponding book has been updated since they were generated + # This case will happen if the book was updated externally + if self.app_db_session and self.cache: + books = self.get_books_updated_in_the_last_day() + book_ids = list(map(lambda b: b.id, books)) + thumbnails = self.get_thumbnails_for_updated_books(book_ids) + self.expire_thumbnails_for_updated_book(books, thumbnails) + self.progress = 0.75 + + # Delete extraneous cached thumbnail files + # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally + if self.app_db_session: + db_thumbnail_files = self.get_thumbnail_filenames() + self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files) + + self._handleSuccess() + self.app_db_session.remove() + + def expire_missing_thumbnails(self, filenames): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.filename.notin_(filenames))\ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self.app_db_session.rollback() + + def get_book_ids(self): + results = self.calibre_db.session\ + .query(db.Books.id)\ + .filter(db.Books.has_cover == 1)\ + .distinct() + + return [value for value, in results] + + def delete_thumbnails_for_missing_books(self, book_ids): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.notin_(book_ids))\ + .delete(synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(str(ex)) + self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex)) + self.app_db_session.rollback() + + def get_thumbnail_filenames(self): + results = self.app_db_session\ + .query(ub.Thumbnail.filename)\ + .all() + + return [thumbnail for thumbnail, in results] + + def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files): + extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files)) + for file in extraneous_files: + self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) + + def get_books_updated_in_the_last_day(self): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ + .all() + + def get_thumbnails_for_updated_books(self, book_ids): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .all() + + def expire_thumbnails_for_updated_book(self, books, thumbnails): + thumbnail_ids = list() + for book in books: + for thumbnail in thumbnails: + if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: + thumbnail_ids.append(thumbnail.id) + + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) + self.app_db_session.rollback() + + @property + def name(self): + return "ThumbnailsSync" + + +class TaskClearCoverThumbnailCache(CalibreTask): + def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): + super(TaskClearCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.book_id = book_id + self.app_db_session = ub.get_new_session_instance() + self.cache = fs.FileSystem() + + def run(self, worker_thread): + if self.app_db_session: + if self.book_id: + thumbnails = self.get_thumbnails_for_book(self.book_id) + for thumbnail in thumbnails: + self.expire_and_delete_thumbnail(thumbnail) + else: + self.expire_and_delete_all_thumbnails() + + self._handleSuccess() + self.app_db_session.remove() + + def get_thumbnails_for_book(self, book_id): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book_id)\ + .all() + + def expire_and_delete_thumbnail(self, thumbnail): + thumbnail.expiration = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnail: ' + str(ex)) + self._handleError(u'Error expiring book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def expire_and_delete_all_thumbnails(self): + self.app_db_session\ + .query(ub.Thumbnail)\ + .update({'expiration': datetime.utcnow()}) + + try: + self.app_db_session.commit() + self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnails: ' + str(ex)) + self._handleError(u'Error expiring book thumbnails: ' + str(ex)) + self.app_db_session.rollback() + + @property + def name(self): + return "ThumbnailsClear" diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 576652d4..20d0802c 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -142,15 +142,18 @@ -
+

{{_('Administration')}}

-
{{_('Download Debug Package')}}
-
{{_('View Logs')}}
-
-
-
{{_('Reconnect Calibre Database')}}
-
{{_('Restart')}}
-
{{_('Shutdown')}}
+
{{_('Download Debug Package')}}
+
{{_('View Logs')}}
+
+
+
{{_('Reconnect Calibre Database')}}
+
{{_('Clear Cover Thumbnail Cache')}}
+
+
+
{{_('Restart')}}
+
{{_('Shutdown')}}
@@ -231,4 +234,21 @@
+ {% endblock %} diff --git a/cps/templates/author.html b/cps/templates/author.html index 4e32db80..990f60ad 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,7 +37,7 @@
- + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html new file mode 100644 index 00000000..c5281797 --- /dev/null +++ b/cps/templates/book_cover.html @@ -0,0 +1,13 @@ +{% macro book_cover_image(book, thumbnails) -%} + {%- set book_title = book.title if book.title else book.name -%} + {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} + {%- if srcset|length -%} + {{ book_title }} + {%- else -%} + {{ book_title }} + {%- endif -%} +{%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index c0fc141e..091beffc 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,9 +1,10 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ book.title }} + {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 342cce53..3e2b65ac 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index d57994b4..02eff658 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -9,7 +10,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 1421ea6a..901dd193 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 9724e31d..2eecd6c6 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -29,7 +30,7 @@
- {{ entry[0].name }} + {{ book_cover_image(entry[0], entry[0].id|get_book_thumbnails(thumbnails)) }} {{entry.count}} diff --git a/cps/templates/index.html b/cps/templates/index.html index d300fc65..579f19f7 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -9,7 +10,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} @@ -86,7 +87,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 318140fa..fef082bb 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,4 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %} +{% from 'book_cover.html' import book_cover_image %} diff --git a/cps/templates/search.html b/cps/templates/search.html index d11f3ec8..428655ea 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -44,7 +45,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 1ad79dbd..e49a008f 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -31,7 +32,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/ub.py b/cps/ub.py index ecf98679..af53e4f8 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -18,6 +18,7 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +import atexit import os import sys import datetime @@ -441,6 +442,27 @@ class RemoteAuthToken(Base): return '' % self.id +def filename(context): + file_format = context.get_current_parameters()['format'] + if file_format == 'jpeg': + return context.get_current_parameters()['uuid'] + '.jpg' + else: + return context.get_current_parameters()['uuid'] + '.' + file_format + + +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + resolution = Column(SmallInteger, default=1) + filename = Column(String, default=filename) + generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) + expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) + + # Add missing tables during migration of database def add_missing_tables(engine, session): if not engine.dialect.has_table(engine.connect(), "book_read_link"): @@ -455,6 +477,8 @@ def add_missing_tables(engine, session): KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "archived_book"): ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): Registration.__table__.create(bind=engine) with engine.connect() as conn: @@ -725,6 +749,16 @@ def init_db(app_db_path): sys.exit(3) +def get_new_session_instance(): + new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + + atexit.register(lambda: new_session.remove() if new_session else True) + + return new_session + + def dispose(): global session diff --git a/cps/web.py b/cps/web.py index ad8cd5da..5bf226ab 100644 --- a/cps/web.py +++ b/cps/web.py @@ -50,9 +50,10 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ - send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password +from .helper import check_valid_domain, render_task_status, get_cc_columns, get_book_cover, get_cached_book_cover, \ + get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, get_download_link, \ + send_mail, generate_random_password, send_registration_mail, check_send_to_kindle, check_read_formats, \ + tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back from .usermanagement import login_required_if_no_ano @@ -411,8 +412,9 @@ def render_books_list(data, sort, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Books"), page=website) + title=_(u"Books"), page=website, thumbnails=thumbnails) def render_rated_books(page, book_id, order): @@ -457,8 +459,9 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Hot Books (Most Downloaded)"), page="hot") + title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) else: abort(404) @@ -482,12 +485,14 @@ def render_downloaded_books(page, order): .filter(db.Books.id == book.id).first(): ub.delete_download(book.id) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Downloaded books by %(user)s",user=current_user.nickname), - page="download") + page="download", + thumbnails=thumbnails) else: abort(404) @@ -498,6 +503,7 @@ def render_author_books(page, author_id, order): db.Books.authors.any(db.Authors.id == author_id), [order[0], db.Series.name, db.Books.series_index], db.books_series_link, + db.Books.id==db.books_series_link.c.book, db.Series) if entries is None or not len(entries): flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), @@ -513,9 +519,10 @@ def render_author_books(page, author_id, order): author_info = services.goodreads_support.get_author_info(author_name) other_books = services.goodreads_support.get_other_books(author_info, entries) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, - other_books=other_books, page="author") + other_books=other_books, page="author", thumbnails=thumbnails) def render_publisher_books(page, book_id, order): @@ -526,9 +533,12 @@ def render_publisher_books(page, book_id, order): db.Books.publishers.any(db.Publishers.id == book_id), [db.Series.name, order[0], db.Books.series_index], db.books_series_link, + db.Books.id == db.books_series_link.c.book, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") + title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", + thumbnails=thumbnails) else: abort(404) @@ -540,8 +550,10 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Series: %(serie)s", serie=name.name), page="series") + title=_(u"Series: %(serie)s", serie=name.name), page="series", + thumbnails=thumbnails) else: abort(404) @@ -553,8 +565,10 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") + title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", + thumbnails=thumbnails) else: abort(404) @@ -566,8 +580,10 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"File format: %(format)s", format=name.format), page="formats") + title=_(u"File format: %(format)s", format=name.format), page="formats", + thumbnails=thumbnails) else: abort(404) @@ -579,9 +595,13 @@ def render_category_books(page, book_id, order): db.Books, db.Books.tags.any(db.Tags.id == book_id), [order[0], db.Series.name, db.Books.series_index], - db.books_series_link, db.Series) + db.books_series_link, + db.Books.id == db.books_series_link.c.book, + db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Category: %(name)s", name=name.name), page="category") + title=_(u"Category: %(name)s", name=name.name), page="category", + thumbnails=thumbnails) else: abort(404) @@ -599,8 +619,9 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, - title=_(u"Language: %(name)s", name=lang_name), page="language") + title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) def render_read_books(page, are_read, as_xml=False, order=None): @@ -644,8 +665,10 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" + + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_archived_books(page, order): @@ -668,8 +691,9 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_prepare_search_form(cc): @@ -702,6 +726,7 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', searchterm=term, pagination=pagination, @@ -710,7 +735,8 @@ def render_search_results(term, offset=None, order=None, limit=None): entries=entries, result_count=result_count, title=_(u"Search"), - page="search") + page="search", + thumbnails=thumbnails) # ################################### View Books list ################################################################## @@ -740,6 +766,7 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): @@ -772,6 +799,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -826,6 +854,7 @@ def publisher_list(): charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher") else: @@ -857,8 +886,10 @@ def series_list(): .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + thumbnails = get_thumbnails_for_book_series(entries) return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, - title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view") + title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", + thumbnails=thumbnails) else: abort(404) @@ -1235,13 +1266,16 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): else: offset = 0 limit_all = result_count + + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', adv_searchterm=searchterm, pagination=pagination, entries=q[offset:limit_all], result_count=result_count, - title=_(u"Advanced Search"), page="advsearch") - + title=_(u"Advanced Search"), + page="advsearch", + thumbnails=thumbnails) @web.route("/advsearch", methods=['GET']) @@ -1260,10 +1294,24 @@ def advanced_search_form(): def get_cover(book_id): return get_book_cover(book_id) + +@web.route("/cached-cover/") +@login_required_if_no_ano +def get_cached_cover(cache_id): + return get_cached_book_cover(cache_id) + + +@web.route("/cached-cover-thumbnail/") +@login_required_if_no_ano +def get_cached_cover_thumbnail(cache_id): + return get_cached_book_cover_thumbnail(cache_id) + + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano @@ -1293,7 +1341,6 @@ def serve_book(book_id, book_format, anyname): return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format) - @web.route("/download//", defaults={'anyname': 'None'}) @web.route("/download///") @login_required_if_no_ano @@ -1481,9 +1528,6 @@ def logout(): return redirect(url_for('web.login')) - - - # ################################### Users own configuration ######################################################### def change_profile_email(to_save, kobo_support, local_oauth_check, oauth_status): if "email" in to_save and to_save["email"] != current_user.email: @@ -1683,6 +1727,7 @@ def show_book(book_id): if media_format.format.lower() in constants.EXTENSIONS_AUDIO: audioentries.append(media_format.format.lower()) + thumbnails = get_thumbnails_for_books([entries]) return render_title_template('detail.html', entry=entries, audioentries=audioentries, @@ -1694,7 +1739,8 @@ def show_book(book_id): is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, - page="book") + page="book", + thumbnails=thumbnails) else: log.debug(u"Error opening eBook. File does not exist or file is not accessible") flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") diff --git a/requirements.txt b/requirements.txt index ca11eff9..5a26381e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +APScheduler>=3.6.3, <3.8.0 Babel>=1.3, <2.9 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1