From d91c4f50b4e71adac3a810bf4938c1beb6235f6f Mon Sep 17 00:00:00 2001
From: vorotamoroz <vrtmrz@gmail.com>
Date: Fri, 29 Sep 2023 18:55:46 +0900
Subject: [PATCH] - Improved:   - A New binary file handling implemented   - A
 new encrypted format has been implemented   - Now the chunk sizes will be
 adjusted for efficient sync - Fixed:   - levels of exception in some logs
 have been fixed - Tidied:   - Some Lint warnings have been suppressed.

---
 src/CmdConfigSync.ts              |  6 ++---
 src/CmdHiddenFileSync.ts          | 10 ++++-----
 src/CmdSetupLiveSync.ts           |  4 ++--
 src/DocumentHistoryModal.ts       |  8 +++----
 src/GlobalHistory.svelte          |  4 ++--
 src/JsonResolvePane.svelte        |  4 ++--
 src/ObsidianLiveSyncSettingTab.ts | 13 +++++++++--
 src/dialogs.ts                    |  1 -
 src/lib                           |  2 +-
 src/main.ts                       | 37 ++++++++++++++++++++++---------
 10 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/src/CmdConfigSync.ts b/src/CmdConfigSync.ts
index 7e2af1f..09e4189 100644
--- a/src/CmdConfigSync.ts
+++ b/src/CmdConfigSync.ts
@@ -7,7 +7,7 @@ import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
 import { delay, getDocData } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { WrappedNotice } from "./lib/src/wrapper";
-import { base64ToArrayBuffer, arrayBufferToBase64, readString, crc32CKHash } from "./lib/src/strbin";
+import { readString, crc32CKHash, decodeBinary, encodeBinary } from "./lib/src/strbin";
 import { serialized } from "./lib/src/lock";
 import { LiveSyncCommands } from "./LiveSyncCommands";
 import { stripAllPrefixes } from "./lib/src/path";
@@ -328,7 +328,7 @@ export class ConfigSync extends LiveSyncCommands {
                     const path = `${baseDir}/${f.filename}`;
                     await this.ensureDirectoryEx(path);
                     if (!content) {
-                        const dt = base64ToArrayBuffer(f.data);
+                        const dt = decodeBinary(f.data);
                         await this.app.vault.adapter.writeBinary(path, dt);
                     } else {
                         await this.app.vault.adapter.write(path, content);
@@ -460,7 +460,7 @@ export class ConfigSync extends LiveSyncCommands {
         const contentBin = await this.app.vault.adapter.readBinary(path);
         let content: string[];
         try {
-            content = await arrayBufferToBase64(contentBin);
+            content = await encodeBinary(contentBin, this.settings.useV1);
             if (path.toLowerCase().endsWith("/manifest.json")) {
                 const v = readString(new Uint8Array(contentBin));
                 try {
diff --git a/src/CmdHiddenFileSync.ts b/src/CmdHiddenFileSync.ts
index 87c14d7..6fcfaef 100644
--- a/src/CmdHiddenFileSync.ts
+++ b/src/CmdHiddenFileSync.ts
@@ -6,7 +6,7 @@ import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils";
 import { WrappedNotice } from "./lib/src/wrapper";
-import { base64ToArrayBuffer, arrayBufferToBase64 } from "./lib/src/strbin";
+import { decodeBinary, encodeBinary } from "./lib/src/strbin";
 import { serialized } from "./lib/src/lock";
 import { JsonResolveModal } from "./JsonResolveModal";
 import { LiveSyncCommands } from "./LiveSyncCommands";
@@ -411,7 +411,7 @@ export class HiddenFileSync extends LiveSyncCommands {
         const contentBin = await this.app.vault.adapter.readBinary(file.path);
         let content: string[];
         try {
-            content = await arrayBufferToBase64(contentBin);
+            content = await encodeBinary(contentBin, this.settings.useV1);
         } catch (ex) {
             Logger(`The file ${file.path} could not be encoded`);
             Logger(ex, LOG_LEVEL_VERBOSE);
@@ -547,7 +547,7 @@ export class HiddenFileSync extends LiveSyncCommands {
                 }
                 if (!isExists) {
                     await this.ensureDirectoryEx(filename);
-                    await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
+                    await this.app.vault.adapter.writeBinary(filename, decodeBinary(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
                     try {
                         //@ts-ignore internalAPI
                         await this.app.vault.adapter.reconcileInternalFile(filename);
@@ -559,12 +559,12 @@ export class HiddenFileSync extends LiveSyncCommands {
                     return true;
                 } else {
                     const contentBin = await this.app.vault.adapter.readBinary(filename);
-                    const content = await arrayBufferToBase64(contentBin);
+                    const content = await encodeBinary(contentBin, this.settings.useV1);
                     if (isDocContentSame(content, fileOnDB.data) && !force) {
                         // Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
                         return true;
                     }
-                    await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
+                    await this.app.vault.adapter.writeBinary(filename, decodeBinary(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
                     try {
                         //@ts-ignore internalAPI
                         await this.app.vault.adapter.reconcileInternalFile(filename);
diff --git a/src/CmdSetupLiveSync.ts b/src/CmdSetupLiveSync.ts
index 857b83d..91680be 100644
--- a/src/CmdSetupLiveSync.ts
+++ b/src/CmdSetupLiveSync.ts
@@ -60,7 +60,7 @@ export class SetupLiveSync extends LiveSyncCommands {
                 delete setting[k];
             }
         }
-        const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
+        const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, false));
         const uri = `${configURIBase}${encryptedSetting}`;
         await navigator.clipboard.writeText(uri);
         Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
@@ -70,7 +70,7 @@ export class SetupLiveSync extends LiveSyncCommands {
         if (encryptingPassphrase === false)
             return;
         const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
-        const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
+        const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, false));
         const uri = `${configURIBase}${encryptedSetting}`;
         await navigator.clipboard.writeText(uri);
         Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
diff --git a/src/DocumentHistoryModal.ts b/src/DocumentHistoryModal.ts
index 7721caf..350b0e5 100644
--- a/src/DocumentHistoryModal.ts
+++ b/src/DocumentHistoryModal.ts
@@ -1,6 +1,6 @@
 import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
 import { getPathFromTFile, isValidPath } from "./utils";
-import { base64ToArrayBuffer, base64ToString, escapeStringToHTML } from "./lib/src/strbin";
+import { decodeBinary, escapeStringToHTML, readString } from "./lib/src/strbin";
 import ObsidianLiveSyncPlugin from "./main";
 import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types";
 import { Logger } from "./lib/src/logger";
@@ -89,7 +89,7 @@ export class DocumentHistoryModal extends Modal {
             this.currentDoc = w;
             this.info.innerHTML = `Modified:${new Date(w.mtime).toLocaleString()}`;
             let result = "";
-            const w1data = w.datatype == "plain" ? getDocData(w.data) : base64ToString(w.data);
+            const w1data = w.datatype == "plain" ? getDocData(w.data) : readString(new Uint8Array(decodeBinary(w.data)));
             this.currentDeleted = !!w.deleted;
             this.currentText = w1data;
             if (this.showDiff) {
@@ -99,7 +99,7 @@ export class DocumentHistoryModal extends Modal {
                     const w2 = await db.getDBEntry(this.file, { rev: oldRev }, false, false, true);
                     if (w2 != false) {
                         const dmp = new diff_match_patch();
-                        const w2data = w2.datatype == "plain" ? getDocData(w2.data) : base64ToString(w2.data);
+                        const w2data = w2.datatype == "plain" ? getDocData(w2.data) : readString(new Uint8Array(decodeBinary(w.data)));
                         const diff = dmp.diff_main(w2data, w1data);
                         dmp.diff_cleanupSemantic(diff);
                         for (const v of diff) {
@@ -204,7 +204,7 @@ export class DocumentHistoryModal extends Modal {
                     await focusFile(pathToWrite);
                     this.close();
                 } else if (this.currentDoc?.datatype == "newnote") {
-                    await this.app.vault.adapter.writeBinary(pathToWrite, base64ToArrayBuffer(this.currentDoc.data));
+                    await this.app.vault.adapter.writeBinary(pathToWrite, decodeBinary(this.currentDoc.data));
                     await focusFile(pathToWrite);
                     this.close();
                 } else {
diff --git a/src/GlobalHistory.svelte b/src/GlobalHistory.svelte
index 33a51e8..9463383 100644
--- a/src/GlobalHistory.svelte
+++ b/src/GlobalHistory.svelte
@@ -7,7 +7,7 @@
     import { DocumentHistoryModal } from "./DocumentHistoryModal";
     import { isPlainText, stripAllPrefixes } from "./lib/src/path";
     import { TFile } from "./deps";
-    import { arrayBufferToBase64 } from "./lib/src/strbin";
+    import { encodeBinary } from "./lib/src/strbin";
     export let plugin: ObsidianLiveSyncPlugin;
 
     let showDiffInfo = false;
@@ -116,7 +116,7 @@
                                         result = isDocContentSame(data, doc.data);
                                     } else {
                                         const data = await plugin.app.vault.readBinary(abs);
-                                        const dataEEncoded = await arrayBufferToBase64(data);
+                                        const dataEEncoded = await encodeBinary(data, plugin.settings.useV1);
                                         result = isDocContentSame(dataEEncoded, doc.data);
                                     }
                                     if (result) {
diff --git a/src/JsonResolvePane.svelte b/src/JsonResolvePane.svelte
index e814966..f3269cc 100644
--- a/src/JsonResolvePane.svelte
+++ b/src/JsonResolvePane.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
     import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "./deps";
     import type { FilePath, LoadedEntry } from "./lib/src/types";
-    import { base64ToString } from "./lib/src/strbin";
+    import { decodeBinary, readString } from "./lib/src/strbin";
     import { getDocData } from "./lib/src/utils";
     import { mergeObject } from "./utils";
 
@@ -26,7 +26,7 @@
     let mode: SelectModes = defaultSelect as SelectModes;
 
     function docToString(doc: LoadedEntry) {
-        return doc.datatype == "plain" ? getDocData(doc.data) : base64ToString(doc.data);
+        return doc.datatype == "plain" ? getDocData(doc.data) : readString(new Uint8Array(decodeBinary(doc.data)));
     }
     function revStringToRevNumber(rev: string) {
         return rev.split("-")[0];
diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts
index 65c2afc..fd7b5d3 100644
--- a/src/ObsidianLiveSyncSettingTab.ts
+++ b/src/ObsidianLiveSyncSettingTab.ts
@@ -1270,7 +1270,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
 
         new Setting(containerSyncSettingEl)
             .setName("Enhance chunk size")
-            .setDesc("Enhance chunk size for binary files (0.1MBytes). This cannot be increased when using IBM Cloudant.")
+            .setDesc("Enhance chunk size for binary files (Ratio). This cannot be increased when using IBM Cloudant.")
             .setClass("wizardHidden")
             .addText((text) => {
                 text.setPlaceholder("")
@@ -1278,7 +1278,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     .onChange(async (value) => {
                         let v = Number(value);
                         if (isNaN(v) || v < 1) {
-                            v = 1;
+                            v = 0;
                         }
                         this.plugin.settings.customChunkSize = v;
                         await this.plugin.saveSettings();
@@ -1816,6 +1816,15 @@ ${stringifyYaml(pluginConfig)}`;
                     await this.plugin.saveSettings();
                 })
             );
+        new Setting(containerHatchEl)
+            .setName("Use binary and encryption version 1")
+            .setDesc("Use the previous data format for other products which shares the remote database.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.useV1).onChange(async (value) => {
+                    this.plugin.settings.useV1 = value;
+                    await this.plugin.saveSettings();
+                })
+            );
         addScreenElement("50", containerHatchEl);
 
 
diff --git a/src/dialogs.ts b/src/dialogs.ts
index 201a680..01ca03b 100644
--- a/src/dialogs.ts
+++ b/src/dialogs.ts
@@ -182,7 +182,6 @@ export class MessageBox extends Modal {
                 this.timer = undefined;
             }
         })
-        contentEl.createEl("h1", { text: this.title });
         const div = contentEl.createDiv();
         MarkdownRenderer.render(this.plugin.app, this.contentMd, div, "/", this.plugin);
         const buttonSetting = new Setting(contentEl);
diff --git a/src/lib b/src/lib
index 6548bd3..4a955ad 160000
--- a/src/lib
+++ b/src/lib
@@ -1 +1 @@
-Subproject commit 6548bd3ed75dc087d74acdac5a473f95aaccf6a9
+Subproject commit 4a955add9c274ea5fb3965e441e0202819f87249
diff --git a/src/main.ts b/src/main.ts
index 3af7829..8c36c49 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -2,7 +2,7 @@ const isDebug = false;
 
 import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
 import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
-import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE } from "./lib/src/types";
+import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, } from "./lib/src/types";
 import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
 import { arrayToChunkedArray, getDocData, isDocContentSame } from "./lib/src/utils";
 import { Logger, setGlobalLogFunction } from "./lib/src/logger";
@@ -16,7 +16,7 @@ import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissin
 import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store";
 import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores";
 import { setNoticeClass } from "./lib/src/wrapper";
-import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayBufferToBase64, writeString } from "./lib/src/strbin";
+import { versionNumberString2Number, writeString, decodeBinary, encodeBinary, readString } from "./lib/src/strbin";
 import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path";
 import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock";
 import { Semaphore } from "./lib/src/semaphore";
@@ -64,8 +64,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     replicator!: LiveSyncDBReplicator;
 
     statusBar?: HTMLElement;
-    suspended: boolean = false;
-    deviceAndVaultName: string = "";
+    suspended = false;
+    deviceAndVaultName = "";
     isMobile = false;
     isReady = false;
     packageVersion = "";
@@ -204,7 +204,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
 
         const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
         if (passphrase !== "false" && typeof passphrase === "string") {
-            enableEncryption(db, passphrase, useDynamicIterationCount);
+            enableEncryption(db, passphrase, useDynamicIterationCount, false, this.settings.useV1);
         }
         if (skipInfo) {
             return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } };
@@ -404,6 +404,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         Logger(`Checking expired file history done`);
     }
     async onLayoutReady() {
+        if (this.settings.useV1 === undefined) {
+            this.settings.useV1 = await this.askEnableV2();
+            await this.saveSettingData();
+        }
         this.registerFileWatchEvents();
         if (!this.localDatabase.isReady) {
             Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE);
@@ -500,6 +504,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         }
         Logger(`Additional safety scan done`, LOG_LEVEL_VERBOSE);
     }
+    async askEnableV2() {
+        const message = `Since v0.20.0, Self-hosted LiveSync uses a new format for binary files and encrypted things. In the new format, files are split at meaningful delimitations, increasing the effectiveness of deduplication.
+However, the new format lacks compatibility with LiveSync before v0.20.0 and related projects. Basically enabling V2 is recommended. but If you are using some related products, stay in a while, please!
+Note: We can always able to read V1 format. It will be progressively converted. And, we can change this toggle later.`
+        const CHOICE_V2 = "Enable v2";
+        const CHOICE_V1 = "Keep v1";
+
+        const ret = await confirmWithMessage(this, "binary and encryption", message, [CHOICE_V2, CHOICE_V1], CHOICE_V1, 40);
+        return ret == CHOICE_V1;
+    }
 
     async onload() {
         logStore.subscribe(e => this.addLog(e.message, e.level, e.key));
@@ -521,6 +535,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         if (lastVersion > this.settings.lastReadUpdates) {
             Logger("Self-hosted LiveSync has undergone a major upgrade. Please open the setting dialog, and check the information pane.", LOG_LEVEL_NOTICE);
         }
+
         //@ts-ignore
         if (this.app.isMobile) {
             this.isMobile = true;
@@ -797,7 +812,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
 
     async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) {
         if (this.usedPassphrase != "") {
-            return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false);
+            return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false, true);
         }
 
         const passphrase = await this.getPassphrase(settings);
@@ -805,7 +820,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT);
             return "";
         }
-        const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false);
+        const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false, true);
         if (dec) {
             this.usedPassphrase = passphrase;
             return dec;
@@ -1270,7 +1285,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL_NOTICE);
             return;
         }
-        const writeData = doc.datatype == "newnote" ? base64ToArrayBuffer(doc.data) : getDocData(doc.data);
+        const writeData = doc.datatype == "newnote" ? decodeBinary(doc.data) : getDocData(doc.data);
         await this.ensureDirectoryEx(path);
         try {
             let outFile;
@@ -1953,7 +1968,7 @@ Or if you are sure know what had been happened, we can unlock the database from
             if (doc === false) return false;
             let data = getDocData(doc.data)
             if (doc.datatype == "newnote") {
-                data = base64ToString(data);
+                data = readString(new Uint8Array(decodeBinary(doc.data)));
             } else if (doc.datatype == "plain") {
                 // NO OP.
             }
@@ -2473,7 +2488,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                 const contentBin = await this.app.vault.readBinary(file);
                 Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
                 try {
-                    content = await arrayBufferToBase64(contentBin);
+                    content = await encodeBinary(contentBin, this.settings.useV1);
                 } catch (ex) {
                     Logger(`The file ${file.path} could not be encoded`);
                     Logger(ex, LOG_LEVEL_VERBOSE);
@@ -2488,7 +2503,7 @@ Or if you are sure know what had been happened, we can unlock the database from
             if (cache instanceof ArrayBuffer) {
                 Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
                 try {
-                    content = await arrayBufferToBase64(cache);
+                    content = await encodeBinary(cache, this.settings.useV1);
                 } catch (ex) {
                     Logger(`The file ${file.path} could not be encoded`);
                     Logger(ex, LOG_LEVEL_VERBOSE);