From c10708ed079c9ca651246f4f7fd2a4b1ac63c2db Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 19 Sep 2022 22:39:40 +0200 Subject: [PATCH] Backup metadata 4th step --- cps/admin.py | 26 ++++++++++----- cps/db.py | 4 +-- cps/helper.py | 5 +-- cps/schedule.py | 2 +- cps/static/js/main.js | 17 ++++++++++ cps/tasks/metadata_backup.py | 65 +++++++++++++++++++++++------------- cps/templates/admin.html | 3 ++ 7 files changed, 85 insertions(+), 37 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index c3600535..d813d4f0 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -136,28 +136,38 @@ def admin_forbidden(): @admin_required def shutdown(): task = request.get_json().get('parameter', -1) - showtext = {} + show_text = {} if task in (0, 1): # valid commandos received # close all database connections calibre_db.dispose() ub.dispose() if task == 0: - showtext['text'] = _(u'Server restarted, please reload page') + show_text['text'] = _(u'Server restarted, please reload page') else: - showtext['text'] = _(u'Performing shutdown of server, please close window') + show_text['text'] = _(u'Performing shutdown of server, please close window') # stop gevent/tornado server web_server.stop(task == 0) - return json.dumps(showtext) + return json.dumps(show_text) if task == 2: log.warning("reconnecting to calibre database") calibre_db.reconnect_db(config, ub.app_DB_path) - showtext['text'] = _(u'Reconnect successful') - return json.dumps(showtext) + show_text['text'] = _(u'Reconnect successful') + return json.dumps(show_text) - showtext['text'] = _(u'Unknown command') - return json.dumps(showtext), 400 + show_text['text'] = _(u'Unknown command') + return json.dumps(show_text), 400 + +@admi.route("/metadata_backup", methods=["POST"]) +@login_required +@admin_required +def queue_metadata_backup(): + show_text = {} + log.warning("Queuing all books for metadata backup") + helper.set_all_metadata_dirty() + show_text['text'] = _(u'Books successfully queued fo Metadata Backup') + return json.dumps(show_text) # method is available without login and not protected by CSRF to make it easy reachable, is per default switched off diff --git a/cps/db.py b/cps/db.py index ed23fe5f..fbb458e6 100644 --- a/cps/db.py +++ b/cps/db.py @@ -399,7 +399,7 @@ class CustomColumns(Base): display_dict = json.loads(self.display) return display_dict - def to_json(self, value, extra): + def to_json(self, value, extra, sequence): content = dict() content['table'] = "custom_column_" + str(self.id) content['column'] = "value" @@ -417,7 +417,7 @@ class CustomColumns(Base): content['category_sort'] = "value" content['is_csp'] = False content['is_editable'] = self.editable - content['rec_index'] = self.id + 22 # toDo why ?? + content['rec_index'] = sequence + 22 # toDo why ?? content['#value#'] = value content['#extra#'] = extra content['is_multiple2'] = {} diff --git a/cps/helper.py b/cps/helper.py index c412df8e..212ce142 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -1037,6 +1037,7 @@ def update_thumbnail_cache(): def set_all_metadata_dirty(): WorkerThread.add(None, TaskBackupMetadata(export_language=get_locale(), - translated_title=_("cover"), - set_dirty=True), + translated_title=_("Cover"), + set_dirty=True, + task_message=N_("Queue all books for metadata backup")), hidden=False) diff --git a/cps/schedule.py b/cps/schedule.py index ab447570..44bb2c27 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -34,7 +34,7 @@ def get_scheduled_tasks(reconnect=True): # ToDo make configurable. Generate metadata.opf file for each changed book if True: - tasks.append([lambda: TaskBackupMetadata(), "en", 'backup metadata', False]) + tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False]) # Generate all missing book cover thumbnails if config.schedule_generate_book_covers: diff --git a/cps/static/js/main.js b/cps/static/js/main.js index cfb0b2f0..73b90a0d 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -503,6 +503,23 @@ $(function() { } }); }); + $("#metadata_backup").click(function() { + $("#DialogHeader").addClass("hidden"); + $("#DialogFinished").addClass("hidden"); + $("#DialogContent").html(""); + $("#spinner2").show(); + $.ajax({ + method: "post", + contentType: "application/json; charset=utf-8", + dataType: "json", + url: getPath() + "/metadata_backup", + success: function success(data) { + $("#spinner2").hide(); + $("#DialogContent").html(data.text); + $("#DialogFinished").removeClass("hidden"); + } + }); + }); $("#perform_update").click(function() { $("#DialogHeader").removeClass("hidden"); $("#spinner2").show(); diff --git a/cps/tasks/metadata_backup.py b/cps/tasks/metadata_backup.py index 7f2ad159..3fcb5857 100644 --- a/cps/tasks/metadata_backup.py +++ b/cps/tasks/metadata_backup.py @@ -42,7 +42,7 @@ NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE} class TaskBackupMetadata(CalibreTask): def __init__(self, export_language="en", - translated_title="cover", + translated_title="Cover", set_dirty=False, task_message=N_('Backing up Metadata')): super(TaskBackupMetadata, self).__init__(task_message) @@ -50,7 +50,7 @@ class TaskBackupMetadata(CalibreTask): self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.export_language = export_language self.translated_title = translated_title - self.set_dirty=set_dirty + self.set_dirty = set_dirty def run(self, worker_thread): if self.set_dirty: @@ -62,8 +62,9 @@ class TaskBackupMetadata(CalibreTask): try: books = self.calibre_db.session.query(db.Books).all() for book in books: - self.calibre_db.set_metadata_dirty(book) - self._handleSuccess() + self.calibre_db.set_metadata_dirty(book.id) + self.calibre_db.session.commit() + self._handleSuccess() except Exception as ex: self.log.debug('Error adding book for backup: ' + str(ex)) self._handleError('Error adding book for backup: ' + str(ex)) @@ -74,21 +75,25 @@ class TaskBackupMetadata(CalibreTask): try: metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all() custom_columns = self.calibre_db.session.query(db.CustomColumns).order_by(db.CustomColumns.label).all() + count = len(metadata_backup) + i = 0 for backup in metadata_backup: book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none() - self.calibre_db.session.query(db.Metadata_Dirtied).filter(db.Metadata_Dirtied == backup.id).delete() + self.calibre_db.session.query(db.Metadata_Dirtied).filter( + db.Metadata_Dirtied.book == backup.book).delete() self.calibre_db.session.commit() if book: self.open_metadata(book, custom_columns) - self._handleSuccess() - self.calibre_db.session.close() else: self.log.error("Book {} not found in database".format(backup.book)) - self._handleError("Book {} not found in database".format(backup.book)) - self.calibre_db.session.close() + # self._handleError("Book {} not found in database".format(backup.book)) + i += 1 + self.progress = (1.0 / count) * i + self._handleSuccess() + self.calibre_db.session.close() except Exception as ex: - self.log.debug('Error creating metadata backup: ' + str(ex)) + self.log.debug('Error creating metadata backup for book {}: '.format(book.id) + str(ex)) self._handleError('Error creating metadata backup: ' + str(ex)) self.calibre_db.session.rollback() self.calibre_db.session.close() @@ -155,36 +160,37 @@ class TaskBackupMetadata(CalibreTask): title.text = book.title for author in book.authors: creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP) - creator.text = str(author) + creator.text = str(author.name) creator.set(OPF + "file-as", book.author_sort) # ToDo Check creator.set(OPF + "role", "aut") contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP) contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]" contributor.set(OPF + "file-as", "calibre") # ToDo Check - contributor.set(OPF + "role", "bpk") + contributor.set(OPF + "role", "bkp") date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP) date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate) - + if book.comments: + for b in book.comments: + description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP) + description.text = b.text if not book.languages: language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) language.text = self.export_language else: for b in book.languages: language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) - language.text = str(b.languages) + language.text = str(b.lang_code) for b in book.tags: subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP) - subject.text = str(b.tags) - if book.comments: - description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP) - description.text = escape(str(book.comments)) + subject.text = str(b.name) etree.SubElement(metadata, "meta", name="calibre:author_link_map", - content="{" + escape(",".join(['"' + str(a) + '":""' for a in book.authors])) + "}", - nsmap=NSMAP) - etree.SubElement(metadata, "meta", name="calibre:series", - content=str(book.series), + content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}", nsmap=NSMAP) + for b in book.series: + etree.SubElement(metadata, "meta", name="calibre:series", + content=str(str(b.name)), + nsmap=NSMAP) etree.SubElement(metadata, "meta", name="calibre:series_index", content=str(book.series_index), nsmap=NSMAP) @@ -195,6 +201,7 @@ class TaskBackupMetadata(CalibreTask): etree.SubElement(metadata, "meta", name="calibre:title_sort", content=book.sort, nsmap=NSMAP) + sequence = 0 for cc in custom_columns: value = None extra = None @@ -203,8 +210,9 @@ class TaskBackupMetadata(CalibreTask): value = cc_entry[0].get("value") extra = cc_entry[0].get("extra") etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label), - content=escape(cc.to_json(value, extra)), + content=cc.to_json(value, extra, sequence), nsmap=NSMAP) + sequence += 1 # generate guide element and all sub elements of it # Title is translated from default export language @@ -213,15 +221,24 @@ class TaskBackupMetadata(CalibreTask): # prepare finalize everything and output doc = etree.ElementTree(package) + # doc = etree.tostring(package, xml_declaration=True, encoding='utf-8', pretty_print=True) # .replace(b""", b""") try: with open(book_metadata_filepath, 'wb') as f: + # f.write(doc) doc.write(f, xml_declaration=True, encoding='utf-8', pretty_print=True) except Exception: # ToDo: Folder not writeable error pass @property def name(self): - return "Backing up Metadata" + return "Metadata backup" + + # needed for logging + def __str__(self): + if self.set_dirty: + return "Queue all books for metadata backup" + else: + return "Perform metadata backup" @property def is_cancellable(self): diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 553dcbb6..4d6eff95 100755 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -207,6 +207,9 @@
{{_('Restart')}}
{{_('Shutdown')}}
+
+
{{_('Queue all books for metadata backup')}}
+