From 53578671033d57c62835fb863b80d8414a66a771 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Tue, 5 Nov 2019 23:18:52 -0500 Subject: [PATCH 01/15] Add initial support for Kobo device Sync endpoint. - Supports /v1/library/sync call to get list of books - Supports /v1/library/metadata call to get metadata for a given book + Assumes books are stored on Backblaze for metadata call - Changes to helper.py so that we can return no cover instead of a blank image. --- .gitignore | 1 + cps.py | 3 + cps/admin.py | 2 + cps/config_sql.py | 1 + cps/db.py | 30 +- cps/helper.py | 26 +- cps/kobo.py | 582 +++++++++++++++++++++++++++++++++ cps/templates/config_edit.html | 4 + requirements.txt | 2 + 9 files changed, 638 insertions(+), 13 deletions(-) create mode 100644 cps/kobo.py diff --git a/.gitignore b/.gitignore index 0ce14757..5cf14e51 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ gdrive_credentials vendor client_secrets.json +b2_secrets.json diff --git a/cps.py b/cps.py index ca7d7230..412604d2 100755 --- a/cps.py +++ b/cps.py @@ -41,6 +41,8 @@ from cps.shelf import shelf from cps.admin import admi from cps.gdrive import gdrive from cps.editbooks import editbook +from cps.kobo import kobo + try: from cps.oauth_bb import oauth oauth_available = True @@ -58,6 +60,7 @@ def main(): app.register_blueprint(admi) app.register_blueprint(gdrive) app.register_blueprint(editbook) + app.register_blueprint(kobo) if oauth_available: app.register_blueprint(oauth) success = web_server.start() diff --git a/cps/admin.py b/cps/admin.py index ccb07d84..0e30109c 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -294,6 +294,8 @@ def _configuration_update_helper(): reboot_required |= _config_string("config_certfile") if config.config_certfile and not os.path.isfile(config.config_certfile): return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError) + + _config_string("config_server_url") _config_checkbox_int("config_uploading") _config_checkbox_int("config_anonbrowse") diff --git a/cps/config_sql.py b/cps/config_sql.py index 809e97d8..8ea8b978 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -49,6 +49,7 @@ class _Settings(_Base): config_port = Column(Integer, default=constants.DEFAULT_PORT) config_certfile = Column(String) config_keyfile = Column(String) + config_server_url = Column(String, default='') config_calibre_web_title = Column(String, default=u'Calibre-Web') config_books_per_page = Column(Integer, default=60) diff --git a/cps/db.py b/cps/db.py index b9853896..dc280ad8 100755 --- a/cps/db.py +++ b/cps/db.py @@ -25,13 +25,13 @@ import ast from sqlalchemy import create_engine from sqlalchemy import Table, Column, ForeignKey -from sqlalchemy import String, Integer, Boolean +from sqlalchemy import String, Integer, Boolean, TIMESTAMP from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base session = None -cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series'] +cc_exceptions = ['comments', 'float', 'composite', 'series'] cc_classes = {} @@ -251,10 +251,10 @@ class Books(Base): title = Column(String) sort = Column(String) author_sort = Column(String) - timestamp = Column(String) + timestamp = Column(TIMESTAMP) pubdate = Column(String) series_index = Column(String) - last_modified = Column(String) + last_modified = Column(TIMESTAMP) path = Column(String) has_cover = Column(Integer) uuid = Column(String) @@ -353,7 +353,7 @@ def setup_db(config): # conn.connection.create_function('upper', 1, ucase) if not cc_classes: - cc = conn.execute("SELECT id, datatype FROM custom_columns") + cc = conn.execute("SELECT id, datatype, normalized FROM custom_columns") cc_ids = [] books_custom_column_links = {} @@ -366,7 +366,7 @@ def setup_db(config): ForeignKey('custom_column_' + str(row.id) + '.id'), primary_key=True) ) - cc_ids.append([row.id, row.datatype]) + cc_ids.append([row.id, row.datatype, row.normalized]) if row.datatype == 'bool': ccdict = {'__tablename__': 'custom_column_' + str(row.id), 'id': Column(Integer, primary_key=True), @@ -377,6 +377,11 @@ def setup_db(config): 'id': Column(Integer, primary_key=True), 'book': Column(Integer, ForeignKey('books.id')), 'value': Column(Integer)} + elif not row.normalized: + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(String)} else: ccdict = {'__tablename__': 'custom_column_' + str(row.id), 'id': Column(Integer, primary_key=True), @@ -384,7 +389,8 @@ def setup_db(config): cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict) for cc_id in cc_ids: - if (cc_id[1] == 'bool') or (cc_id[1] == 'int'): + normalized = cc_id[2] + if (not normalized): setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], primaryjoin=( Books.id == cc_classes[cc_id[0]].book), @@ -393,6 +399,16 @@ def setup_db(config): setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], secondary=books_custom_column_links[cc_id[0]], backref='books')) + #for cc_id in cc_ids: + # if (cc_id[1] == 'bool') or (cc_id[1] == 'int'): + # setattr(Books, 'custom_column_' + str(cc_id[2]), relationship(cc_classes[cc_id[0]], + # primaryjoin=( + # Books.id == cc_classes[cc_id[0]].book), + # backref='books')) + # else: + # setattr(Books, 'custom_column_' + str(cc_id[2]), relationship(cc_classes[cc_id[0]], + # secondary=books_custom_column_links[cc_id[0]], + # backref='books')) global session diff --git a/cps/helper.py b/cps/helper.py index e909086e..a68aad7b 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -428,32 +428,46 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_cover_on_failure(use_generic_cover): + if use_generic_cover: + return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + else: + return None + def get_book_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - if book.has_cover: + return get_book_cover_internal(book, False) +def get_book_cover_with_uuid(book_uuid, + use_generic_cover_on_failure=True): + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + return get_book_cover_internal(book, use_generic_cover_on_failure) + +def get_book_cover_internal(book, + use_generic_cover_on_failure): + if book.has_cover: if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) path=gd.get_cover_via_gdrive(book.path) if path: return redirect(path) else: log.error('%s/cover.jpg not found on Google Drive', book.path) - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) except Exception as e: log.exception(e) # traceback.print_exc() - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): return send_from_directory(cover_file_path, "cover.jpg") else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) # saves book cover from url diff --git a/cps/kobo.py b/cps/kobo.py new file mode 100644 index 00000000..7abfe4d0 --- /dev/null +++ b/cps/kobo.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from flask import Blueprint, request, flash, redirect, url_for +from . import logger, ub, searched_ids, db, helper +from . import config + +from flask import make_response +from flask import jsonify +from flask import json +from flask import send_file +from time import gmtime, strftime +import uuid +from uuid import uuid4, uuid3 +from collections import defaultdict +from b2sdk.account_info.in_memory import InMemoryAccountInfo +from b2sdk.api import B2Api +import os +import subprocess +from datetime import datetime, tzinfo, timedelta +from .constants import CONFIG_DIR as _CONFIG_DIR +import copy +import jsonschema +from sqlalchemy import func + +B2_SECRETS = os.path.join(_CONFIG_DIR, "b2_secrets.json") + +kobo = Blueprint("kobo", __name__) +log = logger.create() + +import base64 + + +def b64encode(data): + return base64.b64encode(data) + + +def b64encode_json(json_data): + return b64encode(json.dumps(json_data)) + + +# Python3 has a timestamp() method we could be calling, however it's not avaiable in python2. +def to_epoch_timestamp(datetime_object): + return (datetime_object - datetime(1970, 1, 1)).total_seconds() + + +class SyncToken: + """ The SyncToken is used to persist state accross requests. + When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service. + As an example use-case, the SyncToken is used to detect books that have been added to the library since the last time the device synced to the server. + + Attributes: + books_last_created: Datetime representing the newest book that the device knows about. + books_last_modified: Datetime representing the last modified book that the device knows about. + """ + + SYNC_TOKEN_HEADER = "x-kobo-synctoken" + VERSION = "1-0-0" + MIN_VERSION = "1-0-0" + + token_schema = { + "type": "object", + "properties": {"version": {"type": "string"}, "data": {"type": "object"},}, + } + # This Schema doesn't contain enough information to detect and propagate book deletions from Calibre to the device. + # A potential solution might be to keep a list of all known book uuids in the token, and look for any missing from the db. + data_schema_v1 = { + "type": "object", + "properties": { + "raw_kobo_store_token": {"type": "string"}, + "books_last_modified": {"type": "string"}, + "books_last_created": {"type": "string"}, + }, + } + + def __init__( + self, + raw_kobo_store_token="", + books_last_created=datetime.min, + books_last_modified=datetime.min, + ): + self.raw_kobo_store_token = raw_kobo_store_token + self.books_last_created = books_last_created + self.books_last_modified = books_last_modified + + @staticmethod + def from_headers(headers): + sync_token_header = headers.get(SyncToken.SYNC_TOKEN_HEADER, "") + if sync_token_header == "": + return SyncToken() + + # On the first sync from a Kobo device, we may receive the SyncToken + # from the official Kobo store. Without digging too deep into it, that + # token is of the form [b64encoded blob].[b64encoded blob 2] + if "." in sync_token_header: + return SyncToken(raw_kobo_store_token=sync_token_header) + + sync_token_json = json.loads( + base64.b64decode(sync_token_header + "=" * (-len(sync_token_header) % 4)) + ) + try: + jsonschema.validate(sync_token_json, SyncToken.token_schema) + if sync_token_json["version"] < SyncToken.MIN_VERSION: + raise ValueError + + data_json = sync_token_json["data"] + jsonschema.validate(sync_token_json, SyncToken.data_schema_v1) + except (jsonschema.exceptions.ValidationError, ValueError) as e: + log.error("Sync token contents do not follow the expected json schema.") + return SyncToken() + + raw_kobo_store_token = data_json["raw_kobo_store_token"] + try: + books_last_modified = datetime.utcfromtimestamp( + data_json["books_last_modified"] + ) + books_last_created = datetime.utcfromtimestamp( + data_json["books_last_created"] + ) + except TypeError: + log.error("SyncToken timestamps don't parse to a datetime.") + return SyncToken(raw_kobo_store_token=raw_kobo_store_token) + + return SyncToken( + raw_kobo_store_token=raw_kobo_store_token, + books_last_created=books_last_created, + books_last_modified=books_last_modified, + ) + + def to_headers(self, headers): + headers[SyncToken.SYNC_TOKEN_HEADER] = self.build_sync_token() + + def build_sync_token(self): + token = { + "version": SyncToken.VERSION, + "data": { + "raw_kobo_store_token": self.raw_kobo_store_token, + "books_last_modified": to_epoch_timestamp(self.books_last_modified), + "books_last_created": to_epoch_timestamp(self.books_last_created), + }, + } + return b64encode_json(token) + + +@kobo.route("/v1/library/sync") +def HandleSyncRequest(): + sync_token = SyncToken.from_headers(request.headers) + log.info("Kobo library sync request received.") + + # TODO: Limit the number of books return per sync call, and rely on the sync-continuatation header + # instead so that the device triggers another sync. + + new_books_last_modified = sync_token.books_last_modified + new_books_last_created = sync_token.books_last_created + entitlements = [] + + # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast. + # It looks like it's treating the db.Books.last_modified field as a string and may fail + # the comparison because of the +00:00 suffix. + changed_entries = ( + db.session.query(db.Books) + .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified) + .all() + ) + for book in changed_entries: + entitlement = CreateEntitlement(book) + if book.timestamp > sync_token.books_last_created: + entitlements.append({"NewEntitlement": entitlement}) + else: + entitlements.append({"ChangedEntitlement": entitlement}) + + new_books_last_modified = max( + book.last_modified, sync_token.books_last_modified + ) + new_books_last_created = max(book.timestamp, sync_token.books_last_modified) + + sync_token.books_last_created = new_books_last_created + sync_token.books_last_modified = new_books_last_modified + + # Missing feature: Detect server-side book deletions. + + # Missing feature: Join the response with results from the official Kobo store so that users can still buy and access books from the device store (particularly while on-the-road). + + response = make_response(jsonify(entitlements)) + + sync_token.to_headers(response.headers) + response.headers["x-kobo-sync-mode"] = "delta" + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +@kobo.route("/v1/library//metadata") +def get_metadata__v1(book_uuid): + log.info("Kobo library metadata request received for book %s" % book_uuid) + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + if not book: + log.info(u"Book %s not found in database", book_uuid) + return make_response("Book not found in database.", 404) + + download_url = get_download_url_for_book(book) + if not download_url: + return make_response("Could not get a download url for book.", 500) + + metadata = create_metadata(book) + metadata["DownloadUrls"] = [ + { + "DrmType": "SignedNoDrm", + "Format": "KEPUB", + "Platform": "Android", + # TODO: Set the file size. + # "Size": file_info["contentLength"], + "Url": download_url, + } + ] + return jsonify([metadata]) + + +def get_download_url_for_book(book): + # TODO: Research what formats Kobo will support over the sync protocol. + # For now let's just assume all books are converted to KEPUB. + data = ( + db.session.query(db.Data) + .filter(db.Data.book == book.id) + .filter(db.Data.format == "KEPUB") + .first() + ) + + if not data: + log.info(u"Book %s does have a kepub format", book_uuid) + return None + + file_name = data.name + ".kepub" + file_path = os.path.join(book.path, file_name) + + if not os.path.isfile(B2_SECRETS): + log.error(u"b2 secret file not found") + return None + with open(B2_SECRETS, "r") as filedata: + secrets = json.load(filedata) + + info = InMemoryAccountInfo() + b2_api = B2Api(info) + b2_api.authorize_account( + "production", secrets["application_key_id"], secrets["application_key"] + ) + bucket = b2_api.get_bucket_by_name(secrets["bucket_name"]) + if not bucket: + log.error(u"b2 bucket not found") + return None + + download_url = b2_api.get_download_url_for_file_name( + secrets["bucket_name"], file_path + ) + download_authorization = bucket.get_download_authorization( + file_path, valid_duration_in_seconds=600 + ) + return download_url + "?Authorization=" + download_authorization + + +def CreateBookEntitlement(book): + book_uuid = book.uuid + return { + "Accessibility": "Full", + "ActivePeriod": {"From": current_time(),}, + "Created": book.timestamp, + "CrossRevisionId": book_uuid, + "Id": book_uuid, + "IsHiddenFromArchive": False, + "IsLocked": False, + # Setting this to true removes from the device. + "IsRemoved": False, + "LastModified": book.last_modified, + "OriginCategory": "Imported", + "RevisionId": book_uuid, + "Status": "Active", + } + + +def CreateEntitlement(book): + return { + "BookEntitlement": CreateBookEntitlement(book), + "BookMetadata": create_metadata(book), + "ReadingState": reading_state(book), + } + + +def current_time(): + return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) + + +def get_description(book): + if not book.comments: + return None + return book.comments[0].text + + +# TODO handle multiple authors +def get_author(book): + if not book.authors: + return None + return book.authors[0].name + + +def get_publisher(book): + if not book.publishers: + return None + return book.publishers[0].name + + +def get_series(book): + if not book.series: + return None + return book.series[0].name + + +def create_metadata(book): + book_uuid = book.uuid + metadata = { + "Categories": ["00000000-0000-0000-0000-000000000001",], + "Contributors": get_author(book), + "CoverImageId": book_uuid, + "CrossRevisionId": book_uuid, + "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0}, + "CurrentLoveDisplayPrice": {"TotalAmount": 0}, + "Description": get_description(book), + "DownloadUrls": [ + # Looks like we need to pass at least one url in the + # v1/library/sync call. The new entitlement is ignored + # otherwise. + # May want to experiment more with this. + { + "DrmType": "None", + "Format": "KEPUB", + "Platform": "Android", + "Size": 1024775, + "Url": "https://google.com", + }, + ], + "EntitlementId": book_uuid, + "ExternalIds": [], + "Genre": "00000000-0000-0000-0000-000000000001", + "IsEligibleForKoboLove": False, + "IsInternetArchive": False, + "IsPreOrder": False, + "IsSocialEnabled": True, + "Language": "en", + "PhoneticPronunciations": {}, + "PublicationDate": "2019-02-03T00:25:03.0000000Z", # current_time(), + "Publisher": {"Imprint": "", "Name": get_publisher(book),}, + "RevisionId": book_uuid, + "Title": book.title, + "WorkId": book_uuid, + } + + if get_series(book): + metadata["Series"] = { + "Name": get_series(book), + "Number": book.series_index, + "NumberFloat": float(book.series_index), + # Get a deterministic id based on the series name. + "Id": uuid3(uuid.NAMESPACE_DNS, get_series(book).encode("utf-8")), + } + + return metadata + + +def get_single_cc_value(book, custom_column_name): + custom_column_values = get_custom_column_values(book, custom_column_name) + if custom_column_values: + return custom_column_values[0].value + return None + + +def get_custom_column_values(book, custom_column_name): + custom_column = ( + db.session.query(db.Custom_Columns) + .filter(db.Custom_Columns.label == custom_column_name) + .one() + ) + cc_string = "custom_column_" + str(custom_column.id) + + return getattr(book, cc_string) + + +def reading_state(book): + # TODO: Make the state custom columns configurable. + # Possibly use calibre-web User db instead of the Calibre metadata.db? + reading_state = { + "StatusInfo": { + "LastModified": get_single_cc_value(book, "lastreadtimestamp"), + "Status": get_single_cc_value(book, "reading_status"), + } + # TODO: CurrentBookmark, Location + } + return reading_state + + +# def get_shelves(book): +# shelves = get_custom_column_values(book, "myshelves") +# return shelves + + +@kobo.route( + "//////image.jpg" +) +def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monochrome): + book_cover = helper.get_book_cover_with_uuid( + book_uuid, use_generic_cover_on_failure=False + ) + if not book_cover: + return make_response() + return book_cover + + +@kobo.route("/v1/user/profile") +@kobo.route("/v1/user/loyalty/benefits") +@kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"]) +@kobo.route("/v1/user/wishlist") +@kobo.route("/v1/user/") +@kobo.route("/v1/user/recommendations") +@kobo.route("/v1/products/") +@kobo.route("/v1/products//nextread") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/library/", methods=["DELETE", "GET"]) # TODO: implement +def HandleDummyRequest(dummy=None): + return make_response(jsonify({})) + + +@kobo.route("/v1/auth/device", methods=["POST"]) +def HandleAuthRequest(): + # Missing feature: Authentication :) + response = make_response( + jsonify( + { + "AccessToken": "abcde", + "RefreshToken": "abcde", + "TokenType": "Bearer", + "TrackingId": "abcde", + "UserKey": "abcdefgeh", + } + ) + ) + return response + + +@kobo.route("/v1/initialization") +def HandleInitRequest(): + resources = NATIVE_KOBO_RESOURCES(calibre_web_url=config.config_server_url) + response = make_response(jsonify({"Resources": resources})) + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +def NATIVE_KOBO_RESOURCES(calibre_web_url): + return { + "account_page": "https://secure.kobobooks.com/profile", + "account_page_rakuten": "https://my.rakuten.co.jp/", + "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}", + "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate", + "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion", + "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations", + "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete", + "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"}, + "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}", + "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}", + "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}", + "book_landing_page": "https://store.kobobooks.com/ebooks", + "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions", + "categories": "https://storeapi.kobo.com/v1/categories", + "categories_page": "https://store.kobobooks.com/ebooks/categories", + "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}", + "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured", + "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products", + "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow", + "configuration_data": "https://storeapi.kobo.com/v1/configuration", + "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access", + "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO", + "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal", + "deals": "https://storeapi.kobo.com/v1/deals", + "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}", + "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete", + "device_auth": "https://storeapi.kobo.com/v1/auth/device", + "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh", + "dictionary_host": "https://kbdownload1-a.akamaihd.net", + "discovery_host": "https://discovery.kobobooks.com", + "eula_page": "https://www.kobo.com/termsofuse?style=onestore", + "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", + "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", + "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", + "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", + "featured_lists": "https://storeapi.kobo.com/v1/products/featured", + "free_books_page": { + "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", + "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", + "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", + "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis", + }, + "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", + "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests", + "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader", + "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem", + "help_page": "http://www.kobo.com/help", + "image_host": calibre_web_url, + "image_url_quality_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg", + "image_url_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/false/image.jpg", + "kobo_audiobooks_enabled": "False", + "kobo_audiobooks_orange_deal_enabled": "False", + "kobo_audiobooks_subscriptions_enabled": "False", + "kobo_nativeborrow_enabled": "True", + "kobo_onestorelibrary_enabled": "False", + "kobo_redeem_enabled": "True", + "kobo_shelfie_enabled": "False", + "kobo_subscriptions_enabled": "False", + "kobo_superpoints_enabled": "False", + "kobo_wishlist_enabled": "True", + "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}", + "library_items": "https://storeapi.kobo.com/v1/user/library", + "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata", + "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices", + "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}", + "library_sync": "https://storeapi.kobo.com/v1/library/sync", + "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints", + "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", + "magazine_landing_page": "https://store.kobobooks.com/emagazines", + "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", + "oauth_host": "https://oauth.kobo.com", + "overdrive_account": "https://auth.overdrive.com/account", + "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library", + "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com", + "overdrive_thunder_host": "https://thunder.api.overdrive.com", + "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html", + "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event", + "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore", + "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread", + "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices", + "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", + "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", + "products": "https://storeapi.kobo.com/v1/products", + "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", + "purchase_buy": "https://www.kobo.com/checkout/createpurchase/", + "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}", + "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", + "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase", + "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}", + "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state", + "redeem_interstitial_page": "https://store.kobobooks.com", + "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/", + "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related", + "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}", + "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}", + "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}", + "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie", + "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/", + "social_authorization_host": "https://social.kobobooks.com:8443", + "social_host": "https://social.kobobooks.com", + "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/", + "store_home": "www.kobo.com/{region}/{language}", + "store_host": "store.kobobooks.com", + "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA", + "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}", + "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top", + "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items", + "tags": "https://storeapi.kobo.com/v1/library/tags", + "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile", + "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview", + "use_one_store": "False", + "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits", + "user_platform": "https://storeapi.kobo.com/v1/user/platform", + "user_profile": "https://storeapi.kobo.com/v1/user/profile", + "user_ratings": "https://storeapi.kobo.com/v1/user/ratings", + "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations", + "user_reviews": "https://storeapi.kobo.com/v1/user/reviews", + "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist", + "userguide_host": "https://kbdownload1-a.akamaihd.net", + "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist", + } diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index dd406d39..b73ecd21 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -104,6 +104,10 @@ +
+ + +
diff --git a/requirements.txt b/requirements.txt index daf2538d..d0c541a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ SQLAlchemy>=1.1.0 tornado>=4.1 Wand>=0.4.4 unidecode>=0.04.19 +b2sdk>=1.0.2,<2.0.0 +jsonschema>=3.2.0 From 55b54de6a013b61e74ac09705799cdc6554bdfd0 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Thu, 5 Dec 2019 19:06:39 -0500 Subject: [PATCH 02/15] Add simple get_download_url implementation to replace the backblaze-backed implementation --- cps/kobo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cps/kobo.py b/cps/kobo.py index 7abfe4d0..c41b6a4c 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -214,8 +214,10 @@ def get_metadata__v1(book_uuid): ] return jsonify([metadata]) - def get_download_url_for_book(book): + return "{url_base}/download/{book_id}/kepub".format(url_base=config.config_server_url, book_id=book.id) + +def get_download_url_for_book_b2(book): # TODO: Research what formats Kobo will support over the sync protocol. # For now let's just assume all books are converted to KEPUB. data = ( From 9ede01f130269694efad9fa813626e642db60b8a Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Sat, 7 Dec 2019 19:21:08 -0500 Subject: [PATCH 03/15] * Add a UserKeyToken to the User table for Kobo authorization. * Add proper authorization checks on the new Kobo endpoints. Important Note: As a side-effect, all CalibreWeb API calls can be authorized using this token (i.e without a username&password). --- cps/admin.py | 13 ++++++- cps/kobo.py | 13 ++++--- cps/kobo_auth.py | 67 ++++++++++++++++++++++++++++++++++++ cps/templates/user_edit.html | 4 +++ cps/ub.py | 13 +++++-- cps/web.py | 4 ++- 6 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 cps/kobo_auth.py diff --git a/cps/admin.py b/cps/admin.py index 0e30109c..0292eee3 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -565,7 +565,6 @@ def edit_user(user_id): else: if "password" in to_save and to_save["password"]: content.password = generate_password_hash(to_save["password"]) - anonymous = content.is_anonymous content.role = constants.selected_roles(to_save) if anonymous: @@ -593,6 +592,18 @@ def edit_user(user_id): content.default_language = to_save["default_language"] if "locale" in to_save and to_save["locale"]: content.locale = to_save["locale"] + + if "kobo_user_key" in to_save and to_save["kobo_user_key"]: + kobo_user_key_hash = generate_password_hash(to_save["kobo_user_key"]) + if kobo_user_key_hash != content.kobo_user_key_hash: + existing_kobo_user_key = ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash == kobo_user_key_hash).first() + if not existing_kobo_user_key: + content.kobo_user_key_hash = kobo_user_key_hash + else: + flash(_(u"Found an existing account for this Kobo UserKey."), category="error") + return render_title_template("user_edit.html", translations=translations, languages=languages, + new_user=0, content=content, downloads=downloads, registered_oauth=oauth_check, + title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") if to_save["email"] and to_save["email"] != content.email: existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ .first() diff --git a/cps/kobo.py b/cps/kobo.py index c41b6a4c..d78e6f19 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- from flask import Blueprint, request, flash, redirect, url_for -from . import logger, ub, searched_ids, db, helper -from . import config - +from . import config, logger, kobo_auth, ub, db, helper +from .web import download_required from flask import make_response from flask import jsonify from flask import json @@ -22,15 +21,17 @@ from .constants import CONFIG_DIR as _CONFIG_DIR import copy import jsonschema from sqlalchemy import func +from flask_login import login_required B2_SECRETS = os.path.join(_CONFIG_DIR, "b2_secrets.json") kobo = Blueprint("kobo", __name__) +kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) + log = logger.create() import base64 - def b64encode(data): return base64.b64encode(data) @@ -143,6 +144,8 @@ class SyncToken: @kobo.route("/v1/library/sync") +@login_required +@download_required def HandleSyncRequest(): sync_token = SyncToken.from_headers(request.headers) log.info("Kobo library sync request received.") @@ -190,6 +193,8 @@ def HandleSyncRequest(): @kobo.route("/v1/library//metadata") +@login_required +@download_required def get_metadata__v1(book_uuid): log.info("Kobo library metadata request received for book %s" % book_uuid) book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py new file mode 100644 index 00000000..c1f45f08 --- /dev/null +++ b/cps/kobo_auth.py @@ -0,0 +1,67 @@ +"""This module is used to control authentication/authorization of Kobo sync requests. +This module also includes research notes into the auth protocol used by Kobo devices. + +Log-in: +When first booting a Kobo device the user must sign into a Kobo (or affiliate) account. +Upon successful sign-in, the user is redirected to + https://auth.kobobooks.com/CrossDomainSignIn?id= +which serves the following response: + . +And triggers the insertion of a userKey into the device's User table. + +IMPORTANT SECURITY CAUTION: +Together, the device's DeviceId and UserKey act as an *irrevocable* authentication +token to most (if not all) Kobo APIs. In fact, in most cases only the UserKey is +required to authorize the API call. + +Changing Kobo password *does not* invalidate user keys! This is apparently a known +issue for a few years now https://www.mobileread.com/forums/showpost.php?p=3476851&postcount=13 +(although this poster hypothesised that Kobo could blacklist a DeviceId, many endpoints +will still grant access given the userkey.) + +Api authorization: +* For most of the endpoints we care about (sync, metadata, tags, etc), the userKey is +passed in the x-kobo-userkey header, and is sufficient to authorize the API call. +* Some endpoints (e.g: AnnotationService) instead make use of Bearer tokens. To get a +BearerToken, the device makes a POST request to the v1/auth/device endpoint with the +secret UserKey and the device's DeviceId. + +Our implementation: +For now, we rely on the official Kobo store's UserKey for authentication. Because of the +irrevocable power granted by the key, we only ever store and compare a hash of the key. +To obtain their UserKey, a user can query the user table from the +.kobo/KoboReader.sqlite database found on their device. +This isn't exactly user friendly however. + +Some possible alternatives that require more research: + * Instead of having users query the device database to find out their UserKey, we could + provide a list of recent Kobo sync attempts in the calibre-web UI for users to + authenticate sync attempts (e.g: 'this was me' button). + * We may be able to craft a sign-in flow with a redirect back to the CalibreWeb + server containing the KoboStore's UserKey. + * Can we create our own UserKey instead of relying on the real store's userkey? + (Maybe using something like location.href=kobo://UserAuthenticated?userId=...?) +""" + +from functools import wraps +from flask import request, make_response +from werkzeug.security import check_password_hash +from . import logger, ub, lm + +USER_KEY_HEADER = "x-kobo-userkey" +log = logger.create() + +def disable_failed_auth_redirect_for_blueprint(bp): + lm.blueprint_login_views[bp.name] = None + +@lm.request_loader +def load_user_from_kobo_request(request): + user_key = request.headers.get(USER_KEY_HEADER) + if user_key: + for user in ( + ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash != "").all() + ): + if check_password_hash(str(user.kobo_user_key_hash), user_key): + return user + log.info("Received Kobo request without a recognizable UserKey.") + return None \ No newline at end of file diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index e22a9415..1e7fa9b9 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -28,6 +28,10 @@
+
+ + +
-
- - -
-
- - -
From 2118d920f544035d0dd99899d218a7cf471264d8 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:04:12 -0500 Subject: [PATCH 13/15] Formatter. --- cps/kobo.py | 15 ++++++++++++--- cps/kobo_auth.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index db16fe9a..2acecb6c 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -33,7 +33,7 @@ from sqlalchemy import func from . import config, logger, kobo_auth, db, helper from .web import download_required -kobo = Blueprint("kobo", __name__, url_prefix='/kobo/') +kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) kobo_auth.register_url_value_preprocessor(kobo) @@ -218,7 +218,12 @@ def HandleMetadataRequest(book_uuid): def get_download_url_for_book(book, book_format): - return url_for("web.download_link", book_id=book.id, book_format=book_format.lower(), _external = True) + return url_for( + "web.download_link", + book_id=book.id, + book_format=book_format.lower(), + _external=True, + ) def create_book_entitlement(book): @@ -350,10 +355,12 @@ def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monoc return make_response() return book_cover + @kobo.route("") def TopLevelEndpoint(): return make_response(jsonify({})) + @kobo.route("/v1/user/profile") @kobo.route("/v1/user/loyalty/benefits") @kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"]) @@ -388,7 +395,9 @@ def HandleAuthRequest(): @kobo.route("/v1/initialization") def HandleInitRequest(): - resources = NATIVE_KOBO_RESOURCES(calibre_web_url=url_for("web.index", _external=True).strip("/")) + resources = NATIVE_KOBO_RESOURCES( + calibre_web_url=url_for("web.index", _external=True).strip("/") + ) response = make_response(jsonify({"Resources": resources})) response.headers["x-kobo-apitoken"] = "e30=" return response diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index fef92599..0b9eba6e 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -70,10 +70,11 @@ from .web import render_title_template log = logger.create() + def register_url_value_preprocessor(kobo): @kobo.url_value_preprocessor def pop_auth_token(endpoint, values): - g.auth_token = values.pop('auth_token') + g.auth_token = values.pop("auth_token") def disable_failed_auth_redirect_for_blueprint(bp): @@ -82,31 +83,44 @@ def disable_failed_auth_redirect_for_blueprint(bp): @lm.request_loader def load_user_from_kobo_request(request): - if 'auth_token' in g: - auth_token = g.get('auth_token') - user = ub.session.query(ub.User).join(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == auth_token).first() + if "auth_token" in g: + auth_token = g.get("auth_token") + user = ( + ub.session.query(ub.User) + .join(ub.RemoteAuthToken) + .filter(ub.RemoteAuthToken.auth_token == auth_token) + .first() + ) if user is not None: login_user(user) return user log.info("Received Kobo request without a recognizable auth token.") return None -kobo_auth = Blueprint("kobo_auth", __name__, url_prefix='/kobo_auth') -@kobo_auth.route('/generate_auth_token') +kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth") + + +@kobo_auth.route("/generate_auth_token") @login_required def generate_auth_token(): # Invalidate any prevously generated Kobo Auth token for this user. - ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == current_user.id).delete() + ub.session.query(ub.RemoteAuthToken).filter( + ub.RemoteAuthToken.user_id == current_user.id + ).delete() auth_token = ub.RemoteAuthToken() auth_token.user_id = current_user.id auth_token.expiration = datetime.max - auth_token.auth_token = (hexlify(urandom(16))).decode('utf-8') + auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8") ub.session.add(auth_token) ub.session.commit() - - return render_title_template('generate_kobo_auth_url.html', title=_(u"Kobo Set-up"), - kobo_auth_url=url_for("kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True)) \ No newline at end of file + return render_title_template( + "generate_kobo_auth_url.html", + title=_(u"Kobo Set-up"), + kobo_auth_url=url_for( + "kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True + ), + ) From f84274f1c534a5d589fd844239c1a61a6d2f0fc7 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:08:15 -0500 Subject: [PATCH 14/15] git add missing generate_kobo_auth_url.html --- cps/templates/generate_kobo_auth_url.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cps/templates/generate_kobo_auth_url.html diff --git a/cps/templates/generate_kobo_auth_url.html b/cps/templates/generate_kobo_auth_url.html new file mode 100644 index 00000000..28b098cf --- /dev/null +++ b/cps/templates/generate_kobo_auth_url.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% block body %} +
+

{{_('Generate Kobo Auth URL')}}

+

+ {{_('Open the .kobo/Kobo eReader.conf file in a text editor and add (or edit):')}}. +

+

+ {{_('api_endpoint=')}}{{kobo_auth_url}} +

+

+ {{_('Please note that every visit to this current page invalidates any previously generated Authentication url for this user.')}}. +

+
+{% endblock %} \ No newline at end of file From d6a974682406edd366cd972b0823002526ec06dc Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:28:53 -0500 Subject: [PATCH 15/15] Add a filter to the Sync request endpoint to ignore books that don't have any formats supported by the device. --- cps/kobo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 2acecb6c..c44915c4 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -33,6 +33,9 @@ from sqlalchemy import func from . import config, logger, kobo_auth, db, helper from .web import download_required +#TODO: Test more formats :) . +KOBO_SUPPORTED_FORMATS = {"KEPUB"} + kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) kobo_auth.register_url_value_preprocessor(kobo) @@ -169,7 +172,9 @@ def HandleSyncRequest(): # the comparison because of the +00:00 suffix. changed_entries = ( db.session.query(db.Books) + .join(db.Data) .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified) + .filter(db.Data.format.in_(KOBO_SUPPORTED_FORMATS)) .all() ) for book in changed_entries: @@ -275,11 +280,10 @@ def get_series(book): def get_metadata(book): - ALLOWED_FORMATS = {"KEPUB"} download_urls = [] for book_data in book.data: - if book_data.format in ALLOWED_FORMATS: + if book_data.format in KOBO_SUPPORTED_FORMATS: download_urls.append( { "Format": book_data.format,