Fixed:

- Internal documents are now ignored.
- Merge dialogue now respond immediately to button pressing.
- Periodic processing now works fine
- The checking interval of detecting conflicted has got shorter
- Replication is now cancelled while cleaning up
- The database locking by the cleaning up is now carefully unlocked
- Missing chunks message is correctly reported

New feature:
- Suspend database reflecting has been implemented
- Now fetch suspends the reflecting database and storage changes temporarily to improve the performance.
- We can choose the action when the remote database has been cleaned
- Merge dialogue now show `↲` before the new line.

Improved:
- Now progress is reported while the cleaning up and fetch process
- Cancelled replication is now detected

src/CmdConfigSync.ts

@@ -4,7 +4,7 @@ import { Notice, type PluginManifest, parseYaml } from "./deps";
 import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry } from "./lib/src/types";
 import { LOG_LEVEL } from "./lib/src/types";
 import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
-import { Parallels, delay, getDocData } from "./lib/src/utils";
+import { delay, getDocData } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { WrappedNotice } from "./lib/src/wrapper";
 import { base64ToArrayBuffer, arrayBufferToBase64, readString, crc32CKHash } from "./lib/src/strbin";

src/CmdSetupLiveSync.ts

@@ -8,6 +8,7 @@ import { LiveSyncCommands } from "./LiveSyncCommands";
 import { delay } from "./lib/src/utils";
 import { confirmWithMessage } from "./dialogs";
 import { Platform } from "./deps";
+import { fetchAllUsedChunks } from "./lib/src/utils_couchdb";
 
 export class SetupLiveSync extends LiveSyncCommands {
     onunload() { }
@@ -284,6 +285,26 @@ Of course, we are able to disable these features.`
         this.plugin.settings.syncAfterMerge = false;
         //this.suspendExtraSync();
     }
+    async suspendReflectingDatabase() {
+        if (this.plugin.settings.doNotSuspendOnFetching) return;
+        Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL.NOTICE);
+        this.plugin.settings.suspendParseReplicationResult = true;
+        this.plugin.settings.suspendFileWatching = true;
+        await this.plugin.saveSettings();
+    }
+    async resumeReflectingDatabase() {
+        if (this.plugin.settings.doNotSuspendOnFetching) return;
+        Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL.NOTICE);
+        this.plugin.settings.suspendParseReplicationResult = false;
+        this.plugin.settings.suspendFileWatching = false;
+        await this.plugin.saveSettings();
+        if (this.plugin.settings.readChunksOnline) {
+            await this.plugin.syncAllFiles(true);
+            await this.plugin.loadQueuedFiles();
+            // Start processing
+            this.plugin.procQueuedFiles();
+        }
+    }
     async askUseNewAdapter() {
         if (!this.plugin.settings.useIndexedDBAdapter) {
             const message = `Now this plugin has been configured to use the old database adapter for keeping compatibility. Do you want to deactivate it?`;
@@ -297,9 +318,22 @@ Of course, we are able to disable these features.`
             }
         }
     }
+    async fetchRemoteChunks() {
+        if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline) {
+            Logger(`Fetching chunks`, LOG_LEVEL.NOTICE);
+            const remoteDB = await this.plugin.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true);
+            if (typeof remoteDB == "string") {
+                Logger(remoteDB, LOG_LEVEL.NOTICE);
+            } else {
+                await fetchAllUsedChunks(this.localDatabase.localDatabase, remoteDB.db);
+            }
+            Logger(`Fetching chunks done`, LOG_LEVEL.NOTICE);
+        }
+    }
     async fetchLocal() {
         this.suspendExtraSync();
         this.askUseNewAdapter();
+        await this.suspendReflectingDatabase();
         await this.plugin.realizeSettingSyncMode();
         await this.plugin.resetLocalDatabase();
         await delay(1000);
@@ -310,6 +344,8 @@ Of course, we are able to disable these features.`
         await this.plugin.replicateAllFromServer(true);
         await delay(1000);
         await this.plugin.replicateAllFromServer(true);
+        await this.fetchRemoteChunks();
+        await this.resumeReflectingDatabase();
         await this.askHiddenFileConfiguration({ enableFetch: true });
     }
     async rebuildRemote() {

src/ConflictResolveModal.ts

@@ -30,11 +30,11 @@ export class ConflictResolveModal extends Modal {
             const x1 = v[0];
             const x2 = v[1];
             if (x1 == DIFF_DELETE) {
-                diff += "<span class='deleted'>" + escapeStringToHTML(x2) + "</span>";
+                diff += "<span class='deleted'>" + escapeStringToHTML(x2).replace(/\n/g, "<span class='ls-mark-cr'></span>\n") + "</span>";
             } else if (x1 == DIFF_EQUAL) {
-                diff += "<span class='normal'>" + escapeStringToHTML(x2) + "</span>";
+                diff += "<span class='normal'>" + escapeStringToHTML(x2).replace(/\n/g, "<span class='ls-mark-cr'></span>\n") + "</span>";
             } else if (x1 == DIFF_INSERT) {
-                diff += "<span class='added'>" + escapeStringToHTML(x2) + "</span>";
+                diff += "<span class='added'>" + escapeStringToHTML(x2).replace(/\n/g, "<span class='ls-mark-cr'></span>\n") + "</span>";
             }
         }
 
@@ -48,23 +48,26 @@ export class ConflictResolveModal extends Modal {
         `;
         contentEl.createEl("button", { text: "Keep A" }, (e) => {
             e.addEventListener("click", async () => {
-                await this.callback(this.result.right.rev);
+                const callback = this.callback;
                 this.callback = null;
                 this.close();
+                await callback(this.result.right.rev);
             });
         });
         contentEl.createEl("button", { text: "Keep B" }, (e) => {
             e.addEventListener("click", async () => {
-                await this.callback(this.result.left.rev);
+                const callback = this.callback;
                 this.callback = null;
                 this.close();
+                await callback(this.result.left.rev);
             });
         });
         contentEl.createEl("button", { text: "Concat both" }, (e) => {
             e.addEventListener("click", async () => {
-                await this.callback("");
+                const callback = this.callback;
                 this.callback = null;
                 this.close();
+                await callback("");
             });
         });
         contentEl.createEl("button", { text: "Not now" }, (e) => {

src/ObsidianLiveSyncSettingTab.ts

@@ -7,7 +7,7 @@ import { Logger } from "./lib/src/logger";
 import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
 import { testCrypt } from "./lib/src/e2ee_v2";
 import ObsidianLiveSyncPlugin from "./main";
-import { performRebuildDB, requestToCouchDB } from "./utils";
+import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils";
 
 
 export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -1610,6 +1610,27 @@ ${stringifyYaml(pluginConfig)}`;
                 toggle.setValue(this.plugin.settings.suspendFileWatching).onChange(async (value) => {
                     this.plugin.settings.suspendFileWatching = value;
                     await this.plugin.saveSettings();
+                    scheduleTask("configReload", 250, async () => {
+                        if (await askYesNo(this.app, "Do you want to restart and reload Obsidian now?") == "yes") {
+                            // @ts-ignore
+                            this.app.commands.executeCommandById("app:reload")
+                        }
+                    })
+                })
+            );
+        new Setting(containerHatchEl)
+            .setName("Suspend database reflecting")
+            .setDesc("Stop reflecting database changes to storage files.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.suspendParseReplicationResult).onChange(async (value) => {
+                    this.plugin.settings.suspendParseReplicationResult = value;
+                    await this.plugin.saveSettings();
+                    scheduleTask("configReload", 250, async () => {
+                        if (await askYesNo(this.app, "Do you want to restart and reload Obsidian now?") == "yes") {
+                            // @ts-ignore
+                            this.app.commands.executeCommandById("app:reload")
+                        }
+                    })
                 })
             );
         new Setting(containerHatchEl)
@@ -1731,6 +1752,16 @@ ${stringifyYaml(pluginConfig)}`;
             )
             .setClass("wizardHidden");
 
+
+        new Setting(containerHatchEl)
+            .setName("Fetch database with previous behaviour")
+            .setDesc("")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.doNotSuspendOnFetching).onChange(async (value) => {
+                    this.plugin.settings.doNotSuspendOnFetching = value;
+                    await this.plugin.saveSettings();
+                })
+            );
         addScreenElement("50", containerHatchEl);
 
 

src/dialogs.ts

@@ -183,7 +183,7 @@ export class MessageBox extends Modal {
         })
         contentEl.createEl("h1", { text: this.title });
         const div = contentEl.createDiv();
-        MarkdownRenderer.renderMarkdown(this.contentMd, div, "/", null);
+        MarkdownRenderer.renderMarkdown(this.contentMd, div, "/", this.plugin);
         const buttonSetting = new Setting(contentEl);
         for (const button of this.buttons) {
             buttonSetting.addButton((btn) => {

src/main.ts

@@ -18,7 +18,7 @@ import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/s
 import { setNoticeClass } from "./lib/src/wrapper";
 import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayBufferToBase64 } from "./lib/src/strbin";
 import { addPrefix, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path";
-import { runWithLock } from "./lib/src/lock";
+import { isLockAcquired, runWithLock } from "./lib/src/lock";
 import { Semaphore } from "./lib/src/semaphore";
 import { StorageEventManager, StorageEventManagerObsidian } from "./StorageEventManager";
 import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/LiveSyncLocalDB";
@@ -240,6 +240,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     createPouchDBInstance<T>(name?: string, options?: PouchDB.Configuration.DatabaseConfiguration): PouchDB.Database<T> {
         if (this.settings.useIndexedDBAdapter) {
             options.adapter = "indexeddb";
+            //@ts-ignore :missing def
             options.purged_infos_limit = 1;
             return new PouchDB(name + "-indexeddb", options);
         }
@@ -421,11 +422,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                     Logger(`${FLAGMD_REDFLAG3} has been detected! Self-hosted LiveSync will discard the local database and fetch everything from the remote once again.`, LOG_LEVEL.NOTICE);
                     await this.addOnSetup.fetchLocal();
                     await this.deleteRedFlag3();
-                    if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
-                        this.settings.suspendFileWatching = false;
-                        await this.saveSettings();
-                        // @ts-ignore
-                        this.app.commands.executeCommandById("app:reload")
+                    if (this.settings.suspendFileWatching) {
+                        if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
+                            this.settings.suspendFileWatching = false;
+                            await this.saveSettings();
+                            // @ts-ignore
+                            this.app.commands.executeCommandById("app:reload")
+                        }
                     }
                 } else {
                     this.settings.writeLogToTheFile = true;
@@ -438,6 +441,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 if (this.settings.suspendFileWatching) {
                     Logger("'Suspend file watching' turned on. Are you sure this is what you intended? Every modification on the vault will be ignored.", LOG_LEVEL.NOTICE);
                 }
+                if (this.settings.suspendParseReplicationResult) {
+                    Logger("'Suspend database reflecting' turned on. Are you sure this is what you intended? Every replicated change will be postponed until disabling this option.", LOG_LEVEL.NOTICE);
+                }
                 const isInitialized = await this.initializeDatabase(false, false);
                 if (!isInitialized) {
                     //TODO:stop all sync.
@@ -1317,12 +1323,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         localStorage.setItem(lsKey, saveData);
     }
     async loadQueuedFiles() {
-        const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName();
-        const ids = JSON.parse(localStorage.getItem(lsKey) || "[]") as string[];
-        const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: ids, include_docs: true });
-        for (const doc of ret.rows) {
-            if (doc.doc && !this.queuedFiles.some((e) => e.entry._id == doc.doc._id)) {
-                await this.parseIncomingDoc(doc.doc as PouchDB.Core.ExistingDocument<EntryBody & PouchDB.Core.AllDocsMeta>);
+        if (!this.settings.suspendParseReplicationResult) {
+            const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName();
+            const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
+            const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: ids, include_docs: true });
+            for (const doc of ret.rows) {
+                if (doc.doc && !this.queuedFiles.some((e) => e.entry._id == doc.doc._id)) {
+                    await this.parseIncomingDoc(doc.doc as PouchDB.Core.ExistingDocument<EntryBody & PouchDB.Core.AllDocsMeta>);
+                }
             }
         }
     }
@@ -1384,6 +1392,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         // It is better for your own safety, not to handle the following files
         const ignoreFiles = [
             "_design/replicate",
+            "_design/chunks",
             FLAGMD_REDFLAG,
             FLAGMD_REDFLAG2,
             FLAGMD_REDFLAG3
@@ -1432,21 +1441,41 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         L1:
         for (const change of docsSorted) {
             if (isChunk(change._id)) {
-                await this.parseIncomingChunk(change);
+                if (!this.settings.suspendParseReplicationResult) {
+                    await this.parseIncomingChunk(change);
+                }
                 continue;
             }
-            for (const proc of this.addOns) {
-                if (await proc.parseReplicationResultItem(change)) {
-                    continue L1;
+            if (!this.settings.suspendParseReplicationResult) {
+                for (const proc of this.addOns) {
+                    if (await proc.parseReplicationResultItem(change)) {
+                        continue L1;
+                    }
                 }
             }
             if (change._id == SYNCINFO_ID) {
                 continue;
             }
-            if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo") {
-                await this.parseIncomingDoc(change);
+            if (change._id.startsWith("_design")) {
                 continue;
             }
+
+            if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo") {
+                if (this.settings.suspendParseReplicationResult) {
+                    const newQueue = {
+                        entry: change,
+                        missingChildren: [] as string[],
+                        timeout: 0,
+                    };
+                    Logger(`Processing scheduled: ${change.path}`, LOG_LEVEL.INFO);
+                    this.queuedFiles.push(newQueue);
+                    this.saveQueuedFiles();
+                    continue;
+                } else {
+                    await this.parseIncomingDoc(change);
+                    continue;
+                }
+            }
             if (change.type == "versioninfo") {
                 if (change.version > VER) {
                     this.replicator.closeReplication();
@@ -1589,8 +1618,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin
 
     async replicate(showMessage?: boolean) {
         if (!this.isReady) return;
+        if (isLockAcquired("cleanup")) {
+            Logger("Database cleaning up is in process. replication has been cancelled", LOG_LEVEL.NOTICE);
+            return;
+        }
         if (this.settings.versionUpFlash != "") {
-            Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE);
+            Logger("Open settings and check message, please. replication has been cancelled.", LOG_LEVEL.NOTICE);
             return;
         }
         await this.applyBatchChange();
@@ -1600,19 +1633,41 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         if (!ret) {
             if (this.replicator.remoteLockedAndDeviceNotAccepted) {
                 if (this.replicator.remoteCleaned) {
-                    Logger(`The remote database has been cleaned up. The local database of this device also should be done.`, showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
-                    const remoteDB = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
-                    if (typeof remoteDB == "string") {
-                        Logger(remoteDB, LOG_LEVEL.NOTICE);
-                        return false;
-                    }
-                    // TODO Check actually sent.
+                    Logger(`The remote database has been cleaned.`, showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
                     await runWithLock("cleanup", true, async () => {
-                        await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
-                        await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
-                        await this.getReplicator().markRemoteResolved(this.settings);
+                        const count = await purgeUnreferencedChunks(this.localDatabase.localDatabase, true);
+                        const message = `The remote database has been cleaned up.
+To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
+However, If there are many chunks to be deleted, maybe fetching again is faster.
+We will lose the history of this device if we fetch the remote database again.
+Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`
+                        const CHOICE_FETCH = "Fetch again";
+                        const CHOICE_CLEAN = "Cleanup";
+                        const CHOICE_DISMISS = "Dismiss";
+                        const ret = await confirmWithMessage(this, "Cleaned", message, [CHOICE_FETCH, CHOICE_CLEAN, CHOICE_DISMISS], CHOICE_DISMISS, 30);
+                        if (ret == CHOICE_FETCH) {
+                            await performRebuildDB(this, "localOnly");
+                        }
+                        if (ret == CHOICE_CLEAN) {
+                            const remoteDB = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
+                            if (typeof remoteDB == "string") {
+                                Logger(remoteDB, LOG_LEVEL.NOTICE);
+                                return false;
+                            }
+
+                            await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
+                            // Perform the synchronisation once.
+                            if (await this.replicator.openReplication(this.settings, false, showMessage, true)) {
+                                await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
+                                await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
+                                await this.getReplicator().markRemoteResolved(this.settings);
+                                Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO)
+                            } else {
+                                Logger("Replication has been cancelled. Please try it again.", showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO)
+                            }
+
+                        }
                     });
-                    Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO)
                 } else {
                     const message = `
 The remote database has been rebuilt.
@@ -2164,7 +2219,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                         setTimeout(() => {
                             //resolved, check again.
                             this.showIfConflicted(filename);
-                        }, 500);
+                        }, 50);
                     } else if (toDelete == null) {
                         Logger("Leave it still conflicted");
                     } else {
@@ -2177,7 +2232,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                         setTimeout(() => {
                             //resolved, check again.
                             this.showIfConflicted(filename);
-                        }, 500);
+                        }, 50);
                     }
 
                     return res(true);
@@ -2221,7 +2276,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                 Logger("conflict:Automatically merged, but we have to check it again");
                 setTimeout(() => {
                     this.showIfConflicted(filename);
-                }, 500);
+                }, 50);
                 return;
             }
             //there conflicts, and have to resolve ;

src/utils.ts

@@ -430,11 +430,12 @@ export class PeriodicProcessor {
     enable(interval: number) {
         this.disable();
         if (interval == 0) return;
-        this._timer = window.setInterval(() => this._process().then(() => { }), interval);
+        this._timer = window.setInterval(() => this.process().then(() => { }), interval);
         this._plugin.registerInterval(this._timer);
     }
     disable() {
-        if (this._timer) clearInterval(this._timer);
+        if (this._timer !== undefined) window.clearInterval(this._timer);
+        this._timer = undefined;
     }
 }
 

styles.css

@@ -260,3 +260,14 @@ div.sls-setting-menu-btn {
 .password-input > .setting-item-control >input {
   -webkit-text-security: disc;
 }
+
+span.ls-mark-cr::after {
+    user-select: none;
+    content: "↲";
+    color: var(--text-muted);
+    font-size: 0.8em;
+}
+
+.deleted span.ls-mark-cr::after {
+    color: var(--text-on-accent);
+}