- Fixed: Skip patterns now handle capital letters.

- Improved
  - New configuration to avoid exceeding throttle capacity.
  - The conflicted `data.json` is no longer merged automatically.

src/JsonResolveModal.ts

@@ -0,0 +1,54 @@
+import { App, Modal } from "obsidian";
+import { LoadedEntry } from "./lib/src/types";
+import JsonResolvePane from "./JsonResolvePane.svelte";
+
+export class JsonResolveModal extends Modal {
+    // result: Array<[number, string]>;
+    filename: string;
+    callback: (keepRev: string, mergedStr?: string) => Promise<void>;
+    docs: LoadedEntry[];
+    component: JsonResolvePane;
+
+    constructor(app: App, filename: string, docs: LoadedEntry[], callback: (keepRev: string, mergedStr?: string) => Promise<void>) {
+        super(app);
+        this.callback = callback;
+        this.filename = filename;
+        this.docs = docs;
+    }
+    async UICallback(keepRev: string, mergedStr?: string) {
+        this.close();
+        await this.callback(keepRev, mergedStr);
+        this.callback = null;
+    }
+
+    onOpen() {
+        const { contentEl } = this;
+
+        contentEl.empty();
+
+        if (this.component == null) {
+            this.component = new JsonResolvePane({
+                target: contentEl,
+                props: {
+                    docs: this.docs,
+                    callback: (keepRev, mergedStr) => this.UICallback(keepRev, mergedStr),
+                },
+            });
+        }
+        return;
+    }
+
+
+    onClose() {
+        const { contentEl } = this;
+        contentEl.empty();
+        // contentEl.empty();
+        if (this.callback != null) {
+            this.callback(null);
+        }
+        if (this.component != null) {
+            this.component.$destroy();
+            this.component = null;
+        }
+    }
+}

src/JsonResolvePane.svelte

@@ -0,0 +1,162 @@
+<script lang="ts">
+    import { Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
+    import type { LoadedEntry } from "./lib/src/types";
+    import { base64ToString } from "./lib/src/strbin";
+    import { getDocData } from "./lib/src/utils";
+    import { mergeObject } from "./utils";
+
+    export let docs: LoadedEntry[] = [];
+    export let callback: (keepRev: string, mergedStr?: string) => Promise<void> = async (_, __) => {
+        Promise.resolve();
+    };
+
+    let docA: LoadedEntry = undefined;
+    let docB: LoadedEntry = undefined;
+    let docAContent = "";
+    let docBContent = "";
+    let objA: any = {};
+    let objB: any = {};
+    let objAB: any = {};
+    let objBA: any = {};
+    let diffs: Diff[];
+    const modes = [
+        ["", "Not now"],
+        ["A", "A"],
+        ["B", "B"],
+        ["AB", "A + B"],
+        ["BA", "B + A"],
+    ] as ["" | "A" | "B" | "AB" | "BA", string][];
+    let mode: "" | "A" | "B" | "AB" | "BA" = "";
+
+    function docToString(doc: LoadedEntry) {
+        return doc.datatype == "plain" ? getDocData(doc.data) : base64ToString(doc.data);
+    }
+    function revStringToRevNumber(rev: string) {
+        return rev.split("-")[0];
+    }
+
+    function getDiff(left: string, right: string) {
+        const dmp = new diff_match_patch();
+        const mapLeft = dmp.diff_linesToChars_(left, right);
+        const diffLeftSrc = dmp.diff_main(mapLeft.chars1, mapLeft.chars2, false);
+        dmp.diff_charsToLines_(diffLeftSrc, mapLeft.lineArray);
+        return diffLeftSrc;
+    }
+    function getJsonDiff(a: object, b: object) {
+        return getDiff(JSON.stringify(a, null, 2), JSON.stringify(b, null, 2));
+    }
+    function apply() {
+        if (mode == "A") return callback(docA._rev, null);
+        if (mode == "B") return callback(docB._rev, null);
+        if (mode == "BA") return callback(null, JSON.stringify(objBA, null, 2));
+        if (mode == "AB") return callback(null, JSON.stringify(objAB, null, 2));
+        callback(null, null);
+    }
+    $: {
+        if (docs && docs.length >= 1) {
+            if (docs[0].mtime < docs[1].mtime) {
+                docA = docs[0];
+                docB = docs[1];
+            } else {
+                docA = docs[1];
+                docB = docs[0];
+            }
+            docAContent = docToString(docA);
+            docBContent = docToString(docB);
+
+            try {
+                objA = false;
+                objB = false;
+                objA = JSON.parse(docAContent);
+                objB = JSON.parse(docBContent);
+                objAB = mergeObject(objA, objB);
+                objBA = mergeObject(objB, objA);
+                if (JSON.stringify(objAB) == JSON.stringify(objBA)) {
+                    objBA = false;
+                }
+            } catch (ex) {
+                objBA = false;
+                objAB = false;
+            }
+        }
+    }
+    $: mergedObjs = {
+        "": false,
+        A: objA,
+        B: objB,
+        AB: objAB,
+        BA: objBA,
+    };
+
+    $: selectedObj = mode in mergedObjs ? mergedObjs[mode] : {};
+    $: {
+        diffs = getJsonDiff(objA, selectedObj);
+        console.dir(selectedObj);
+    }
+</script>
+
+<h1>File Conflicted</h1>
+{#if !docA || !docB}
+    <div class="message">Just for a minute, please!</div>
+    <div class="buttons">
+        <button on:click={apply}>Dismiss</button>
+    </div>
+{:else}
+    <div class="options">
+        {#each modes as m}
+            {#if m[0] == "" || mergedObjs[m[0]] != false}
+                <label class={`sls-setting-label ${m[0] == mode ? "selected" : ""}`}
+                    ><input type="radio" name="disp" bind:group={mode} value={m[0]} class="sls-setting-tab" />
+                    <div class="sls-setting-menu-btn">{m[1]}</div></label
+                >
+            {/if}
+        {/each}
+    </div>
+
+    {#if selectedObj != false}
+        <div class="op-scrollable json-source">
+            {#each diffs as diff}
+                <span class={diff[0] == DIFF_DELETE ? "deleted" : diff[0] == DIFF_INSERT ? "added" : "normal"}>{diff[1]}</span>
+            {/each}
+        </div>
+    {:else}
+        NO PREVIEW
+    {/if}
+    <div>
+        A Rev:{revStringToRevNumber(docA._rev)} ,{new Date(docA.mtime).toLocaleString()}
+        {docAContent.length} letters
+    </div>
+
+    <div>
+        B Rev:{revStringToRevNumber(docB._rev)} ,{new Date(docB.mtime).toLocaleString()}
+        {docBContent.length} letters
+    </div>
+
+    <div class="buttons">
+        <button on:click={apply}>Apply</button>
+    </div>
+{/if}
+
+<style>
+    .deleted {
+        text-decoration: line-through;
+    }
+    * {
+        box-sizing: border-box;
+    }
+
+    .scroller {
+        display: flex;
+        flex-direction: column;
+        overflow-y: scroll;
+        max-height: 60vh;
+        user-select: text;
+    }
+    .json-source {
+        white-space: pre;
+        height: auto;
+        overflow: auto;
+        min-height: var(--font-ui-medium);
+        flex-grow: 1;
+    }
+</style>

src/lib

@@ -1 +1 @@
-Subproject commit 9e993fd984b0aabbe63b0cce17d056f40b2e650b
+Subproject commit a765f8eac5e4a46d4a93b6e48038a7e4f3b7a88e

src/main.ts

@@ -2,14 +2,14 @@ import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbst
 import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2 } from "./lib/src/types";
 import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo, queueItem, FileInfo } from "./types";
-import { getDocData, isDocContentSame } from "./lib/src/utils";
+import { delay, getDocData, isDocContentSame } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { LocalPouchDB } from "./LocalPouchDB";
 import { LogDisplayModal } from "./LogDisplayModal";
 import { ConflictResolveModal } from "./ConflictResolveModal";
 import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
-import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
+import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, flattenObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
 import { decrypt, encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
 
 const isDebug = false;
@@ -23,6 +23,7 @@ import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayB
 import { isPlainText, isValidPath, shouldBeIgnored } from "./lib/src/path";
 import { runWithLock } from "./lib/src/lock";
 import { Semaphore } from "./lib/src/semaphore";
+import { JsonResolveModal } from "./JsonResolveModal";
 
 setNoticeClass(Notice);
 
@@ -1148,9 +1149,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (!this.settings.syncInternalFiles) return;
         if (!this.settings.watchInternalFileChanges) return;
         if (!path.startsWith(this.app.vault.configDir)) return;
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         if (ignorePatterns.some(e => path.match(e))) return;
         this.appendWatchEvent(
             [{
@@ -2325,6 +2326,26 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
             const diffLeft = generatePatchObj(baseObj, leftObj);
             const diffRight = generatePatchObj(baseObj, rightObj);
+
+            // If each value of the same key has been modified, the automatic merge should be prevented.
+            //TODO Does it have to be a configurable item?
+            const diffSetLeft = new Map(flattenObject(diffLeft));
+            const diffSetRight = new Map(flattenObject(diffRight));
+            for (const [key, value] of diffSetLeft) {
+                if (diffSetRight.has(key)) {
+                    if (diffSetRight.get(key) == value) {
+                        // No matter, if changed to the same value.
+                        diffSetRight.delete(key);
+                    }
+                }
+            }
+            for (const [key, value] of diffSetRight) {
+                if (diffSetLeft.has(key) && diffSetLeft.get(key) != value) {
+                    // Some changes are conflicted
+                    return false;
+                }
+            }
+
             const patches = [
                 { mtime: leftLeaf.mtime, patch: diffLeft },
                 { mtime: rightLeaf.mtime, patch: diffRight }
@@ -2505,7 +2526,60 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }).open();
         });
     }
-
+    showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
+        return new Promise((res) => {
+            Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE);
+            const docs = [docA, docB];
+            const modal = new JsonResolveModal(this.app, id2path(docA._id), [docA, docB], async (keep, result) => {
+                // modal.close();
+                try {
+                    const filename = id2filenameInternalMetadata(docA._id);
+                    let needFlush = false;
+                    if (!result && !keep) {
+                        Logger(`Skipped merging: ${filename}`);
+                    }
+                    //Delete old revisions
+                    if (result || keep) {
+                        for (const doc of docs) {
+                            if (doc._rev != keep) {
+                                if (await this.localDatabase.deleteDBEntry(doc._id, { rev: doc._rev })) {
+                                    Logger(`Conflicted revision has been deleted: ${filename}`);
+                                    needFlush = true;
+                                }
+                            }
+                        }
+                    }
+                    if (!keep && result) {
+                        const isExists = await this.app.vault.adapter.exists(filename);
+                        if (!isExists) {
+                            await this.ensureDirectoryEx(filename);
+                        }
+                        await this.app.vault.adapter.write(filename, result);
+                        const stat = await this.app.vault.adapter.stat(filename);
+                        await this.storeInternalFileToDatabase({ path: filename, ...stat }, true);
+                        try {
+                            //@ts-ignore internalAPI
+                            await app.vault.adapter.reconcileInternalFile(filename);
+                        } catch (ex) {
+                            Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
+                            Logger(ex, LOG_LEVEL.VERBOSE);
+                        }
+                        Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`);
+                    }
+                    if (needFlush) {
+                        await this.extractInternalFileFromDatabase(filename, false);
+                        Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`);
+                    }
+                    res(true);
+                } catch (ex) {
+                    Logger("Could not merge conflicted json");
+                    Logger(ex, LOG_LEVEL.VERBOSE)
+                    res(false);
+                }
+            })
+            modal.open();
+        });
+    }
     conflictedCheckFiles: string[] = [];
 
     // queueing the conflicted file check
@@ -2976,9 +3050,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async scanInternalFiles(): Promise<InternalFileInfo[]> {
-        const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         const root = this.app.vault.getRoot();
         const findRoot = root.path;
         const filenames = (await this.getFiles(findRoot, [], null, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
@@ -3116,8 +3190,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
         return await runWithLock("file-" + id, false, async () => {
             try {
-                const fileOnDB = await this.localDatabase.getDBEntry(id, null, false, false) as false | LoadedEntry;
+                // Check conflicted status 
+                //TODO option
+                const fileOnDB = await this.localDatabase.getDBEntry(id, { conflicts: true }, false, false) as false | LoadedEntry;
                 if (fileOnDB === false) throw new Error(`File not found on database.:${id}`);
+                // Prevent overrite for Prevent overwriting while some conflicted revision exists.
+                if (fileOnDB?._conflicts?.length) {
+                    Logger(`Hidden file ${id} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL.INFO);
+                    return;
+                }
                 const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false;
                 if (deleted) {
                     if (!isExists) {
@@ -3175,9 +3256,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     filterTargetFiles(files: InternalFileInfo[], targetFiles: string[] | false = false) {
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         return files.filter(file => !ignorePatterns.some(e => file.path.match(e))).filter(file => !targetFiles || (targetFiles && targetFiles.indexOf(file.path) !== -1))
     }
 
@@ -3200,7 +3281,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async resolveConflictOnInternalFile(id: string): Promise<boolean> {
-        try {// Retrieve data
+        try {
+            // Retrieve data
             const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
             // If there is no conflict, return with false.
             if (!("_conflicts" in doc)) return false;
@@ -3233,6 +3315,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 } else {
                     Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
                 }
+
+                const docAMerge = await this.localDatabase.getDBEntry(id, { rev: revA });
+                const docBMerge = await this.localDatabase.getDBEntry(id, { rev: revB });
+                if (docAMerge != false && docBMerge != false) {
+                    if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
+                        await delay(200);
+                        // Again for other conflicted revisions.
+                        return this.resolveConflictOnInternalFile(id);
+                    }
+                    return false;
+                }
             }
             const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
             // determine which revision should been deleted.
@@ -3258,9 +3351,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         await this.resolveConflictOnInternalFiles();
         const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
         Logger("Scanning hidden files.", logLevel, "sync_internal");
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         if (!files) files = await this.scanInternalFiles();
         const filesOnDB = ((await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
 
@@ -3299,7 +3392,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }
         }
         const p = [] as Promise<void>[];
-        const semaphore = Semaphore(15);
+        const semaphore = Semaphore(10);
         // Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
         let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
         caches = await this.localDatabase.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches-internal") || {};

src/utils.ts

@@ -198,3 +198,65 @@ export function applyPatch(from: Record<string | number | symbol, any>, patch: R
     }
     return ret;
 }
+
+export function mergeObject(
+    objA: Record<string | number | symbol, any>,
+    objB: Record<string | number | symbol, any>
+) {
+    const newEntries = Object.entries(objB);
+    const ret: any = { ...objA };
+    if (
+        typeof objA !== typeof objB ||
+        Array.isArray(objA) !== Array.isArray(objB)
+    ) {
+        return objB;
+    }
+
+    for (const [key, v] of newEntries) {
+        if (key in ret) {
+            const value = ret[key];
+            if (
+                typeof v !== typeof value ||
+                Array.isArray(v) !== Array.isArray(value)
+            ) {
+                //if type is not match, replace completely.
+                ret[key] = v;
+            } else {
+                if (
+                    typeof v == "object" &&
+                    typeof value == "object" &&
+                    !Array.isArray(v) &&
+                    !Array.isArray(value)
+                ) {
+                    ret[key] = mergeObject(v, value);
+                } else if (
+                    typeof v == "object" &&
+                    typeof value == "object" &&
+                    Array.isArray(v) &&
+                    Array.isArray(value)
+                ) {
+                    ret[key] = [...new Set([...v, ...value])];
+                } else {
+                    ret[key] = v;
+                }
+            }
+        } else {
+            ret[key] = v;
+        }
+    }
+    return Object.entries(ret)
+        .sort()
+        .reduce((p, [key, value]) => ({ ...p, [key]: value }), {});
+}
+
+export function flattenObject(obj: Record<string | number | symbol, any>, path: string[] = []): [string, any][] {
+    if (typeof (obj) != "object") return [[path.join("."), obj]];
+    if (Array.isArray(obj)) return [[path.join("."), JSON.stringify(obj)]];
+    const e = Object.entries(obj);
+    const ret = []
+    for (const [key, value] of e) {
+        const p = flattenObject(value, [...path, key]);
+        ret.push(...p);
+    }
+    return ret;
+}