From e61bebd3ee3b1cc03f68ece5b02d52524b63e8f8 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 3 Feb 2023 18:30:40 +0900 Subject: [PATCH] - Fixed: Skip patterns now handle capital letters. - Improved - New configuration to avoid exceeding throttle capacity. - The conflicted `data.json` is no longer merged automatically. --- src/JsonResolveModal.ts | 54 +++++++++++++ src/JsonResolvePane.svelte | 162 +++++++++++++++++++++++++++++++++++++ src/lib | 2 +- src/main.ts | 147 ++++++++++++++++++++++++++------- src/utils.ts | 62 ++++++++++++++ 5 files changed, 399 insertions(+), 28 deletions(-) create mode 100644 src/JsonResolveModal.ts create mode 100644 src/JsonResolvePane.svelte diff --git a/src/JsonResolveModal.ts b/src/JsonResolveModal.ts new file mode 100644 index 0000000..be1f0d5 --- /dev/null +++ b/src/JsonResolveModal.ts @@ -0,0 +1,54 @@ +import { App, Modal } from "obsidian"; +import { LoadedEntry } from "./lib/src/types"; +import JsonResolvePane from "./JsonResolvePane.svelte"; + +export class JsonResolveModal extends Modal { + // result: Array<[number, string]>; + filename: string; + callback: (keepRev: string, mergedStr?: string) => Promise; + docs: LoadedEntry[]; + component: JsonResolvePane; + + constructor(app: App, filename: string, docs: LoadedEntry[], callback: (keepRev: string, mergedStr?: string) => Promise) { + super(app); + this.callback = callback; + this.filename = filename; + this.docs = docs; + } + async UICallback(keepRev: string, mergedStr?: string) { + this.close(); + await this.callback(keepRev, mergedStr); + this.callback = null; + } + + onOpen() { + const { contentEl } = this; + + contentEl.empty(); + + if (this.component == null) { + this.component = new JsonResolvePane({ + target: contentEl, + props: { + docs: this.docs, + callback: (keepRev, mergedStr) => this.UICallback(keepRev, mergedStr), + }, + }); + } + return; + } + + + onClose() { + const { contentEl } = this; + contentEl.empty(); + // contentEl.empty(); + if (this.callback != null) { + this.callback(null); + } + if (this.component != null) { + this.component.$destroy(); + this.component = null; + } + } +} diff --git a/src/JsonResolvePane.svelte b/src/JsonResolvePane.svelte new file mode 100644 index 0000000..1f5a14a --- /dev/null +++ b/src/JsonResolvePane.svelte @@ -0,0 +1,162 @@ + + +

File Conflicted

+{#if !docA || !docB} +
Just for a minute, please!
+
+ +
+{:else} +
+ {#each modes as m} + {#if m[0] == "" || mergedObjs[m[0]] != false} + + {/if} + {/each} +
+ + {#if selectedObj != false} +
+ {#each diffs as diff} + {diff[1]} + {/each} +
+ {:else} + NO PREVIEW + {/if} +
+ A Rev:{revStringToRevNumber(docA._rev)} ,{new Date(docA.mtime).toLocaleString()} + {docAContent.length} letters +
+ +
+ B Rev:{revStringToRevNumber(docB._rev)} ,{new Date(docB.mtime).toLocaleString()} + {docBContent.length} letters +
+ +
+ +
+{/if} + + diff --git a/src/lib b/src/lib index 9e993fd..a765f8e 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 9e993fd984b0aabbe63b0cce17d056f40b2e650b +Subproject commit a765f8eac5e4a46d4a93b6e48038a7e4f3b7a88e diff --git a/src/main.ts b/src/main.ts index e85b241..5552421 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,14 +2,14 @@ import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbst import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch"; import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2 } from "./lib/src/types"; import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo, queueItem, FileInfo } from "./types"; -import { getDocData, isDocContentSame } from "./lib/src/utils"; +import { delay, getDocData, isDocContentSame } from "./lib/src/utils"; import { Logger } from "./lib/src/logger"; import { LocalPouchDB } from "./LocalPouchDB"; import { LogDisplayModal } from "./LogDisplayModal"; import { ConflictResolveModal } from "./ConflictResolveModal"; import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab"; import { DocumentHistoryModal } from "./DocumentHistoryModal"; -import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils"; +import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, flattenObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils"; import { decrypt, encrypt, tryDecrypt } from "./lib/src/e2ee_v2"; const isDebug = false; @@ -23,6 +23,7 @@ import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayB import { isPlainText, isValidPath, shouldBeIgnored } from "./lib/src/path"; import { runWithLock } from "./lib/src/lock"; import { Semaphore } from "./lib/src/semaphore"; +import { JsonResolveModal } from "./JsonResolveModal"; setNoticeClass(Notice); @@ -1148,9 +1149,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin { if (!this.settings.syncInternalFiles) return; if (!this.settings.watchInternalFileChanges) return; if (!path.startsWith(this.app.vault.configDir)) return; - const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase() + const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns .replace(/\n| /g, "") - .split(",").filter(e => e).map(e => new RegExp(e)); + .split(",").filter(e => e).map(e => new RegExp(e, "i")); if (ignorePatterns.some(e => path.match(e))) return; this.appendWatchEvent( [{ @@ -2325,6 +2326,26 @@ export default class ObsidianLiveSyncPlugin extends Plugin { const diffLeft = generatePatchObj(baseObj, leftObj); const diffRight = generatePatchObj(baseObj, rightObj); + + // If each value of the same key has been modified, the automatic merge should be prevented. + //TODO Does it have to be a configurable item? + const diffSetLeft = new Map(flattenObject(diffLeft)); + const diffSetRight = new Map(flattenObject(diffRight)); + for (const [key, value] of diffSetLeft) { + if (diffSetRight.has(key)) { + if (diffSetRight.get(key) == value) { + // No matter, if changed to the same value. + diffSetRight.delete(key); + } + } + } + for (const [key, value] of diffSetRight) { + if (diffSetLeft.has(key) && diffSetLeft.get(key) != value) { + // Some changes are conflicted + return false; + } + } + const patches = [ { mtime: leftLeaf.mtime, patch: diffLeft }, { mtime: rightLeaf.mtime, patch: diffRight } @@ -2505,7 +2526,60 @@ export default class ObsidianLiveSyncPlugin extends Plugin { }).open(); }); } - + showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise { + return new Promise((res) => { + Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE); + const docs = [docA, docB]; + const modal = new JsonResolveModal(this.app, id2path(docA._id), [docA, docB], async (keep, result) => { + // modal.close(); + try { + const filename = id2filenameInternalMetadata(docA._id); + let needFlush = false; + if (!result && !keep) { + Logger(`Skipped merging: ${filename}`); + } + //Delete old revisions + if (result || keep) { + for (const doc of docs) { + if (doc._rev != keep) { + if (await this.localDatabase.deleteDBEntry(doc._id, { rev: doc._rev })) { + Logger(`Conflicted revision has been deleted: ${filename}`); + needFlush = true; + } + } + } + } + if (!keep && result) { + const isExists = await this.app.vault.adapter.exists(filename); + if (!isExists) { + await this.ensureDirectoryEx(filename); + } + await this.app.vault.adapter.write(filename, result); + const stat = await this.app.vault.adapter.stat(filename); + await this.storeInternalFileToDatabase({ path: filename, ...stat }, true); + try { + //@ts-ignore internalAPI + await app.vault.adapter.reconcileInternalFile(filename); + } catch (ex) { + Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE); + Logger(ex, LOG_LEVEL.VERBOSE); + } + Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`); + } + if (needFlush) { + await this.extractInternalFileFromDatabase(filename, false); + Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`); + } + res(true); + } catch (ex) { + Logger("Could not merge conflicted json"); + Logger(ex, LOG_LEVEL.VERBOSE) + res(false); + } + }) + modal.open(); + }); + } conflictedCheckFiles: string[] = []; // queueing the conflicted file check @@ -2976,9 +3050,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } async scanInternalFiles(): Promise { - const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase() + const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns .replace(/\n| /g, "") - .split(",").filter(e => e).map(e => new RegExp(e)); + .split(",").filter(e => e).map(e => new RegExp(e, "i")); const root = this.app.vault.getRoot(); const findRoot = root.path; const filenames = (await this.getFiles(findRoot, [], null, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash")); @@ -3116,8 +3190,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin { return await runWithLock("file-" + id, false, async () => { try { - const fileOnDB = await this.localDatabase.getDBEntry(id, null, false, false) as false | LoadedEntry; + // Check conflicted status + //TODO option + const fileOnDB = await this.localDatabase.getDBEntry(id, { conflicts: true }, false, false) as false | LoadedEntry; if (fileOnDB === false) throw new Error(`File not found on database.:${id}`); + // Prevent overrite for Prevent overwriting while some conflicted revision exists. + if (fileOnDB?._conflicts?.length) { + Logger(`Hidden file ${id} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL.INFO); + return; + } const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false; if (deleted) { if (!isExists) { @@ -3125,6 +3206,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } else { Logger(`STORAGE e).map(e => new RegExp(e)); + .split(",").filter(e => e).map(e => new RegExp(e, "i")); return files.filter(file => !ignorePatterns.some(e => file.path.match(e))).filter(file => !targetFiles || (targetFiles && targetFiles.indexOf(file.path) !== -1)) } @@ -3200,7 +3281,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } async resolveConflictOnInternalFile(id: string): Promise { - try {// Retrieve data + try { + // Retrieve data const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true }); // If there is no conflict, return with false. if (!("_conflicts" in doc)) return false; @@ -3233,6 +3315,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } else { Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE); } + + const docAMerge = await this.localDatabase.getDBEntry(id, { rev: revA }); + const docBMerge = await this.localDatabase.getDBEntry(id, { rev: revB }); + if (docAMerge != false && docBMerge != false) { + if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) { + await delay(200); + // Again for other conflicted revisions. + return this.resolveConflictOnInternalFile(id); + } + return false; + } } const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB }); // determine which revision should been deleted. @@ -3258,9 +3351,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin { await this.resolveConflictOnInternalFiles(); const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO; Logger("Scanning hidden files.", logLevel, "sync_internal"); - const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase() + const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns .replace(/\n| /g, "") - .split(",").filter(e => e).map(e => new RegExp(e)); + .split(",").filter(e => e).map(e => new RegExp(e, "i")); if (!files) files = await this.scanInternalFiles(); const filesOnDB = ((await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted); @@ -3299,7 +3392,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { } } const p = [] as Promise[]; - const semaphore = Semaphore(15); + const semaphore = Semaphore(10); // Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content) let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {}; caches = await this.localDatabase.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches-internal") || {}; diff --git a/src/utils.ts b/src/utils.ts index 3a60b35..41a30d9 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -198,3 +198,65 @@ export function applyPatch(from: Record, patch: R } return ret; } + +export function mergeObject( + objA: Record, + objB: Record +) { + const newEntries = Object.entries(objB); + const ret: any = { ...objA }; + if ( + typeof objA !== typeof objB || + Array.isArray(objA) !== Array.isArray(objB) + ) { + return objB; + } + + for (const [key, v] of newEntries) { + if (key in ret) { + const value = ret[key]; + if ( + typeof v !== typeof value || + Array.isArray(v) !== Array.isArray(value) + ) { + //if type is not match, replace completely. + ret[key] = v; + } else { + if ( + typeof v == "object" && + typeof value == "object" && + !Array.isArray(v) && + !Array.isArray(value) + ) { + ret[key] = mergeObject(v, value); + } else if ( + typeof v == "object" && + typeof value == "object" && + Array.isArray(v) && + Array.isArray(value) + ) { + ret[key] = [...new Set([...v, ...value])]; + } else { + ret[key] = v; + } + } + } else { + ret[key] = v; + } + } + return Object.entries(ret) + .sort() + .reduce((p, [key, value]) => ({ ...p, [key]: value }), {}); +} + +export function flattenObject(obj: Record, path: string[] = []): [string, any][] { + if (typeof (obj) != "object") return [[path.join("."), obj]]; + if (Array.isArray(obj)) return [[path.join("."), JSON.stringify(obj)]]; + const e = Object.entries(obj); + const ret = [] + for (const [key, value] of e) { + const p = flattenObject(value, [...path, key]); + ret.push(...p); + } + return ret; +} \ No newline at end of file