### Fixed

- No longer broken JSON files including `\n`, during the bucket synchronisation. (#623)
- Custom headers and JWT tokens are now correctly sent to the server during configuration checking. (#624)

### Improved

- Bucket synchronisation has been enhanced for better performance and reliability.
    - Now less duplicated chunks are sent to the server.
    - Fetching conflicted files from the server is now more reliable.
    - Dependent libraries have been updated to the latest version.

package.json

@@ -68,10 +68,10 @@
         "typescript": "^5.7.3"
     },
     "dependencies": {
-        "@aws-sdk/client-s3": "^3.645.0",
-        "@smithy/fetch-http-handler": "^3.2.4",
-        "@smithy/protocol-http": "^4.1.0",
-        "@smithy/querystring-builder": "^3.0.3",
+        "@aws-sdk/client-s3": "^3.787.0",
+        "@smithy/fetch-http-handler": "^5.0.2",
+        "@smithy/protocol-http": "^5.1.0",
+        "@smithy/querystring-builder": "^4.0.2",
         "diff-match-patch": "^1.0.5",
         "esbuild-plugin-inline-worker": "^0.1.1",
         "fflate": "^0.8.2",
@@ -80,7 +80,7 @@
         "octagonal-wheels": "^0.1.25",
         "qrcode-generator": "^1.4.4",
         "svelte-check": "^4.1.4",
-        "trystero": "^0.20.1",
+        "trystero": "^0.21.1",
         "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
     }
 }

src/common/utils.ts

@@ -15,6 +15,7 @@ import {
     LOG_LEVEL_NOTICE,
     LOG_LEVEL_VERBOSE,
     type AnyEntry,
+    type CouchDBCredentials,
     type DocumentID,
     type EntryHasPath,
     type FilePath,
@@ -31,6 +32,7 @@ import type { KeyValueDatabase } from "./KeyValueDB.ts";
 import { scheduleTask } from "octagonal-wheels/concurrency/task";
 import { EVENT_PLUGIN_UNLOADED, eventHub } from "./events.ts";
 import { promiseWithResolver, type PromiseWithResolvers } from "octagonal-wheels/promises";
+import { AuthorizationHeaderGenerator } from "../lib/src/replication/httplib.ts";
 
 export { scheduleTask, cancelTask, cancelAllTasks } from "../lib/src/concurrency/task.ts";
 
@@ -230,17 +232,17 @@ export const _requestToCouchDBFetch = async (
 
 export const _requestToCouchDB = async (
     baseUri: string,
-    username: string,
-    password: string,
+    credentials: CouchDBCredentials,
     origin: string,
     path?: string,
     body?: any,
-    method?: string
+    method?: string,
+    customHeaders?: Record<string, string>
 ) => {
-    const utf8str = String.fromCharCode.apply(null, [...writeString(`${username}:${password}`)]);
-    const encoded = window.btoa(utf8str);
-    const authHeader = "Basic " + encoded;
-    const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin };
+    // Create each time to avoid caching.
+    const authHeaderGen = new AuthorizationHeaderGenerator();
+    const authHeader = await authHeaderGen.getAuthorizationHeader(credentials);
+    const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin, ...customHeaders };
     const uri = `${baseUri}/${path}`;
     const requestParam: RequestUrlParam = {
         url: uri,
@@ -251,6 +253,9 @@ export const _requestToCouchDB = async (
     };
     return await requestUrl(requestParam);
 };
+/**
+ * @deprecated Use requestToCouchDBWithCredentials instead.
+ */
 export const requestToCouchDB = async (
     baseUri: string,
     username: string,
@@ -258,12 +263,34 @@ export const requestToCouchDB = async (
     origin: string = "",
     key?: string,
     body?: string,
-    method?: string
+    method?: string,
+    customHeaders?: Record<string, string>
 ) => {
     const uri = `_node/_local/_config${key ? "/" + key : ""}`;
-    return await _requestToCouchDB(baseUri, username, password, origin, uri, body, method);
+    return await _requestToCouchDB(
+        baseUri,
+        { username, password, type: "basic" },
+        origin,
+        uri,
+        body,
+        method,
+        customHeaders
+    );
 };
 
+export function requestToCouchDBWithCredentials(
+    baseUri: string,
+    credentials: CouchDBCredentials,
+    origin: string = "",
+    key?: string,
+    body?: string,
+    method?: string,
+    customHeaders?: Record<string, string>
+) {
+    const uri = `_node/_local/_config${key ? "/" + key : ""}`;
+    return _requestToCouchDB(baseUri, credentials, origin, uri, body, method, customHeaders);
+}
+
 export const BASE_IS_NEW = Symbol("base");
 export const TARGET_IS_NEW = Symbol("target");
 export const EVEN = Symbol("even");

src/modules/essentialObsidian/ModuleObsidianAPI.ts

@@ -1,16 +1,7 @@
 import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
 import { LOG_LEVEL_DEBUG, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
 import { Notice, requestUrl, type RequestUrlParam, type RequestUrlResponse } from "../../deps.ts";
-import {
-    type CouchDBCredentials,
-    type EntryDoc,
-    type FilePathWithPrefix,
-    type JWTCredentials,
-    type JWTHeader,
-    type JWTParams,
-    type JWTPayload,
-    type PreparedJWT,
-} from "../../lib/src/common/types.ts";
+import { type CouchDBCredentials, type EntryDoc, type FilePathWithPrefix } from "../../lib/src/common/types.ts";
 import { getPathFromTFile } from "../../common/utils.ts";
 import {
     disableEncryption,
@@ -22,9 +13,7 @@ import {
 import { setNoticeClass } from "../../lib/src/mock_and_interop/wrapper.ts";
 import { ObsHttpHandler } from "./APILib/ObsHttpHandler.ts";
 import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser.ts";
-import { reactive, reactiveSource } from "octagonal-wheels/dataobject/reactive.js";
-import { arrayBufferToBase64Single, writeString } from "../../lib/src/string_and_binary/convert.ts";
-import { Refiner } from "octagonal-wheels/dataobject/Refiner";
+import { AuthorizationHeaderGenerator } from "../../lib/src/replication/httplib.ts";
 
 setNoticeClass(Notice);
 
@@ -44,10 +33,8 @@ async function fetchByAPI(request: RequestUrlParam): Promise<RequestUrlResponse>
 
 export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidianModule {
     _customHandler!: ObsHttpHandler;
-    authHeaderSource = reactiveSource<string>("");
-    authHeader = reactive(() =>
-        this.authHeaderSource.value == "" ? "" : "Basic " + window.btoa(this.authHeaderSource.value)
-    );
+
+    _authHeader = new AuthorizationHeaderGenerator();
 
     last_successful_post = false;
     $$customFetchHandler(): ObsHttpHandler {
@@ -110,135 +97,6 @@ export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidi
         }
     }
 
-    _importKey(auth: JWTCredentials) {
-        if (auth.jwtAlgorithm == "HS256" || auth.jwtAlgorithm == "HS512") {
-            const key = (auth.jwtKey || "").trim();
-            if (key == "") {
-                throw new Error("JWT key is empty");
-            }
-            const binaryDerString = window.atob(key);
-            const binaryDer = new Uint8Array(binaryDerString.length);
-            for (let i = 0; i < binaryDerString.length; i++) {
-                binaryDer[i] = binaryDerString.charCodeAt(i);
-            }
-            const hashName = auth.jwtAlgorithm == "HS256" ? "SHA-256" : "SHA-512";
-            return crypto.subtle.importKey("raw", binaryDer, { name: "HMAC", hash: { name: hashName } }, true, [
-                "sign",
-            ]);
-        } else if (auth.jwtAlgorithm == "ES256" || auth.jwtAlgorithm == "ES512") {
-            const pem = auth.jwtKey
-                .replace(/-----BEGIN [^-]+-----/, "")
-                .replace(/-----END [^-]+-----/, "")
-                .replace(/\s+/g, "");
-            // const pem = key.replace(/\s/g, "");
-            const binaryDerString = window.atob(pem);
-            const binaryDer = new Uint8Array(binaryDerString.length);
-            for (let i = 0; i < binaryDerString.length; i++) {
-                binaryDer[i] = binaryDerString.charCodeAt(i);
-            }
-            // const binaryDer = base64ToArrayBuffer(pem);
-            const namedCurve = auth.jwtAlgorithm == "ES256" ? "P-256" : "P-521";
-            const param = { name: "ECDSA", namedCurve };
-            return crypto.subtle.importKey("pkcs8", binaryDer, param, true, ["sign"]);
-        } else {
-            throw new Error("Supplied JWT algorithm is not supported.");
-        }
-    }
-
-    _currentCryptoKey = new Refiner<JWTCredentials, CryptoKey>({
-        evaluation: async (auth, previous) => {
-            return await this._importKey(auth);
-        },
-    });
-
-    _jwt = new Refiner<JWTParams, PreparedJWT>({
-        evaluation: async (params, previous) => {
-            const encodedHeader = btoa(JSON.stringify(params.header));
-            const encodedPayload = btoa(JSON.stringify(params.payload));
-            const buff = `${encodedHeader}.${encodedPayload}`.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
-
-            const key = await this._currentCryptoKey.update(params.credentials).value;
-            let token = "";
-            if (params.header.alg == "ES256" || params.header.alg == "ES512") {
-                const jwt = await crypto.subtle.sign(
-                    { name: "ECDSA", hash: { name: "SHA-256" } },
-                    key,
-                    writeString(buff)
-                );
-                token = (await arrayBufferToBase64Single(jwt))
-                    .replace(/\+/g, "-")
-                    .replace(/\//g, "_")
-                    .replace(/=/g, "");
-            } else if (params.header.alg == "HS256" || params.header.alg == "HS512") {
-                const jwt = await crypto.subtle.sign(
-                    { name: "HMAC", hash: { name: params.header.alg } },
-                    key,
-                    writeString(buff)
-                );
-                token = (await arrayBufferToBase64Single(jwt))
-                    .replace(/\+/g, "-")
-                    .replace(/\//g, "_")
-                    .replace(/=/g, "");
-            } else {
-                throw new Error("JWT algorithm is not supported.");
-            }
-            return {
-                ...params,
-                token: `${buff}.${token}`,
-            } as PreparedJWT;
-        },
-    });
-
-    _jwtParams = new Refiner<JWTCredentials, JWTParams>({
-        evaluation(source, previous) {
-            const kid = source.jwtKid || undefined;
-            const sub = (source.jwtSub || "").trim();
-            if (sub == "") {
-                throw new Error("JWT sub is empty");
-            }
-            const algorithm = source.jwtAlgorithm || "";
-            if (!algorithm) {
-                throw new Error("JWT algorithm is not configured.");
-            }
-            if (algorithm != "HS256" && algorithm != "HS512" && algorithm != "ES256" && algorithm != "ES512") {
-                throw new Error("JWT algorithm is not supported.");
-            }
-            const header: JWTHeader = {
-                alg: source.jwtAlgorithm || "HS256",
-                typ: "JWT",
-                kid,
-            };
-            const iat = ~~(new Date().getTime() / 1000);
-            const exp = iat + (source.jwtExpDuration || 5) * 60; // 5 minutes
-            const payload = {
-                exp,
-                iat,
-                sub: source.jwtSub || "",
-                "_couchdb.roles": ["_admin"],
-            } satisfies JWTPayload;
-            return {
-                header,
-                payload,
-                credentials: source,
-            };
-        },
-        shouldUpdate(isDifferent, source, previous) {
-            if (isDifferent) {
-                return true;
-            }
-            if (!previous) {
-                return true;
-            }
-            // if expired.
-            const d = ~~(new Date().getTime() / 1000);
-            if (previous.payload.exp < d) {
-                // console.warn(`jwt expired ${previous.payload.exp} < ${d}`);
-                return true;
-            }
-            return false;
-        },
-    });
-
     async $$connectRemoteCouchDB(
         uri: string,
         auth: CouchDBCredentials,
@@ -253,25 +111,14 @@ export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidi
         if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
         if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
         if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
-        let authHeader = "";
-        if ("username" in auth) {
-            const userNameAndPassword = auth.username && auth.password ? `${auth.username}:${auth.password}` : "";
-            if (this.authHeaderSource.value != userNameAndPassword) {
-                this.authHeaderSource.value = userNameAndPassword;
-            }
-            authHeader = this.authHeader.value;
-        } else if ("jwtAlgorithm" in auth) {
-            const params = await this._jwtParams.update(auth).value;
-            const jwt = await this._jwt.update(params).value;
-            const token = jwt.token;
-            authHeader = `Bearer ${token}`;
-        }
+        // let authHeader = await this._authHeader.getAuthorizationHeader(auth);
 
         const conf: PouchDB.HttpAdapter.HttpAdapterConfiguration = {
             adapter: "http",
             auth: "username" in auth ? auth : undefined,
             skip_setup: !performSetup,
             fetch: async (url: string | Request, opts?: RequestInit) => {
+                const authHeader = await this._authHeader.getAuthorizationHeader(auth);
                 let size = "";
                 const localURL = url.toString().substring(uri.length);
                 const method = opts?.method ?? "GET";
@@ -288,27 +135,27 @@ export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidi
                     size = ` (${opts_length})`;
                 }
                 try {
+                    const headers = new Headers(opts?.headers);
+                    if (customHeaders) {
+                        for (const [key, value] of Object.entries(customHeaders)) {
+                            if (key && value) {
+                                headers.append(key, value);
+                            }
+                        }
+                    }
+                    if (!("username" in auth)) {
+                        headers.append("authorization", authHeader);
+                    }
+
                     if (!disableRequestURI && typeof url == "string" && typeof (opts?.body ?? "") == "string") {
-                        return await this.fetchByAPI(url, localURL, method, authHeader, opts);
+                        // Deprecated configuration, only for backward compatibility.
+                        return await this.fetchByAPI(url, localURL, method, authHeader, { ...opts, headers });
                     }
                     // --> native Fetch API.
 
                     try {
-                        if (customHeaders) {
-                            for (const [key, value] of Object.entries(customHeaders)) {
-                                if (key && value) {
-                                    (opts!.headers as Headers).append(key, value);
-                                }
-                            }
-                            // // Issue #407
-                            // (opts!.headers as Headers).append("ngrok-skip-browser-warning", "123");
-                        }
-                        // debugger;
-                        if (!("username" in auth)) {
-                            (opts!.headers as Headers).append("authorization", authHeader);
-                        }
                         this.plugin.requestCount.value = this.plugin.requestCount.value + 1;
-                        const response: Response = await fetch(url, opts);
+                        const response: Response = await fetch(url, { ...opts, headers });
                         if (method == "POST" || method == "PUT") {
                             this.last_successful_post = response.ok;
                         } else {
@@ -344,7 +191,10 @@ export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidi
                             this._log(
                                 "Failed to fetch by native fetch API. Trying to fetch by API to get more information."
                             );
-                            const resp2 = await this.fetchByAPI(url.toString(), localURL, method, authHeader, opts);
+                            const resp2 = await this.fetchByAPI(url.toString(), localURL, method, authHeader, {
+                                ...opts,
+                                headers,
+                            });
                             if (resp2.status / 100 == 2) {
                                 this._log(
                                     "The request was successful by API. But the native fetch API failed! Please check CORS settings on the remote database!. While this condition, you cannot enable LiveSync",

src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts

@@ -32,6 +32,7 @@ import {
     delay,
     isDocContentSame,
     isObjectDifferent,
+    parseHeaderValues,
     readAsBlob,
     sizeToHumanReadable,
 } from "../../../lib/src/common/utils.ts";
@@ -45,7 +46,7 @@ import {
 } from "../../../lib/src/pouchdb/utils_couchdb.ts";
 import { testCrypt } from "../../../lib/src/encryption/e2ee_v2.ts";
 import ObsidianLiveSyncPlugin from "../../../main.ts";
-import { getPath, requestToCouchDB, scheduleTask } from "../../../common/utils.ts";
+import { getPath, requestToCouchDBWithCredentials, scheduleTask } from "../../../common/utils.ts";
 import { request } from "obsidian";
 import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "../../../lib/src/string_and_binary/path.ts";
 import MultipleRegExpControl from "./MultipleRegExpControl.svelte";
@@ -83,6 +84,7 @@ import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
 import { LocalDatabaseMaintenance } from "../../../features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts";
 import { mount } from "svelte";
 import { getWebCrypto } from "../../../lib/src/mods.ts";
+import { generateCredentialObject } from "../../../lib/src/replication/httplib.ts";
 
 export type OnUpdateResult = {
     visibility?: boolean;
@@ -1184,11 +1186,16 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     return;
                 }
                 // Tip: Add log for cloudant as Logger($msg("obsidianLiveSyncSettingTab.logServerConfigurationCheck"));
-                const r = await requestToCouchDB(
+                const customHeaders = parseHeaderValues(this.editingSettings.couchDB_CustomHeaders);
+                const credential = generateCredentialObject(this.editingSettings);
+                const r = await requestToCouchDBWithCredentials(
                     this.editingSettings.couchDB_URI,
-                    this.editingSettings.couchDB_USER,
-                    this.editingSettings.couchDB_PASSWORD,
-                    window.origin
+                    credential,
+                    window.origin,
+                    undefined,
+                    undefined,
+                    undefined,
+                    customHeaders
                 );
                 const responseConfig = r.json;
 
@@ -1201,13 +1208,14 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     x.querySelector("button")?.addEventListener("click", () => {
                         fireAndForget(async () => {
                             Logger($msg("obsidianLiveSyncSettingTab.logCouchDbConfigSet", { title, key, value }));
-                            const res = await requestToCouchDB(
+                            const res = await requestToCouchDBWithCredentials(
                                 this.editingSettings.couchDB_URI,
-                                this.editingSettings.couchDB_USER,
-                                this.editingSettings.couchDB_PASSWORD,
+                                credential,
                                 undefined,
                                 key,
-                                value
+                                value,
+                                undefined,
+                                customHeaders
                             );
                             if (res.status == 200) {
                                 Logger(
@@ -1342,11 +1350,14 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 // Request header check
                 const origins = ["app://obsidian.md", "capacitor://localhost", "http://localhost"];
                 for (const org of origins) {
-                    const rr = await requestToCouchDB(
+                    const rr = await requestToCouchDBWithCredentials(
                         this.editingSettings.couchDB_URI,
-                        this.editingSettings.couchDB_USER,
-                        this.editingSettings.couchDB_PASSWORD,
-                        org
+                        credential,
+                        org,
+                        undefined,
+                        undefined,
+                        undefined,
+                        customHeaders
                     );
                     const responseHeaders = Object.fromEntries(
                         Object.entries(rr.headers).map((e) => {
@@ -2406,11 +2417,16 @@ The pane also can be launched by \`P2P Replicator\` command from the Command Pal
                             const REDACTED = "𝑅𝐸𝐷𝐴𝐶𝑇𝐸𝐷";
                             if (this.editingSettings.remoteType == REMOTE_COUCHDB) {
                                 try {
-                                    const r = await requestToCouchDB(
+                                    const credential = generateCredentialObject(this.editingSettings);
+                                    const customHeaders = parseHeaderValues(this.editingSettings.couchDB_CustomHeaders);
+                                    const r = await requestToCouchDBWithCredentials(
                                         this.editingSettings.couchDB_URI,
-                                        this.editingSettings.couchDB_USER,
-                                        this.editingSettings.couchDB_PASSWORD,
-                                        window.origin
+                                        credential,
+                                        window.origin,
+                                        undefined,
+                                        undefined,
+                                        undefined,
+                                        customHeaders
                                     );
 
                                     Logger(JSON.stringify(r.json, null, 2));