self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-01-05 12:50:41 +02:00
Code Issues Releases Activity

chunk splitting method changed

Browse Source 
less change sensitivity
This commit is contained in:
vrtmrz 2021-10-17 22:10:03 +09:00
parent 57187a0926
commit 518ae46cf9
4 changed files with 245 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

282
main.ts
View File

@ -1,4 +1,4 @@
import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder } from "obsidian";
import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder, ItemView } from "obsidian";
import { PouchDB } from "./pouchdb-browser-webpack/dist/pouchdb-browser";
import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
import xxhash from "xxhash-wasm";
@ -6,8 +6,9 @@ import xxhash from "xxhash-wasm";
// docs should be encoded as base64, so 1 char -> 1 bytes
// and cloudant limitation is 1MB , we use 900kb;
// const MAX_DOC_SIZE = 921600;
const MAX_DOC_SIZE = 200; // for .md file
const MAX_DOC_SIZE = 1000; // for .md file, but if delimiters exists. use that before.
const MAX_DOC_SIZE_BIN = 102400; // 100kb
const VER = 10
interface ObsidianLiveSyncSettings {
couchDB_URI: string;
@ -19,6 +20,9 @@ interface ObsidianLiveSyncSettings {
savingDelay: number;
lessInformationInLog: boolean;
gcDelay: number;
versionUpFlash: string;
minimumChunkSize: number;
longLineThreshold: number
}
const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@ -31,6 +35,9 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
savingDelay: 200,
lessInformationInLog: false,
gcDelay: 30,
versionUpFlash: "",
minimumChunkSize: 20,
longLineThreshold: 250,
};
interface Entry {
@ -54,8 +61,20 @@ interface NewEntry {
NewNote: true;
type: "newnote";
}
interface PlainEntry {
_id: string;
children: string[];
_rev?: string;
ctime: number;
mtime: number;
size: number;
_deleted?: boolean;
NewNote: true;
type: "plain";
}
type LoadedEntry = Entry & {
children: string[];
datatype: "plain" | "newnote"
};
interface EntryLeaf {
@ -63,9 +82,10 @@ interface EntryLeaf {
data: string;
_deleted?: boolean;
type: "leaf";
_rev?: string;
}
type EntryDoc = Entry | NewEntry | LoadedEntry | EntryLeaf;
type EntryDoc = Entry | NewEntry | PlainEntry | LoadedEntry | EntryLeaf;
type diff_result_leaf = {
rev: string;
data: string;
@ -166,6 +186,7 @@ class LocalPouchDB {
addLog: (message: any, isNotify?: boolean) => Promise<void>;
localDatabase: PouchDB.Database<EntryDoc>;
recentModifiedDocs: string[] = [];
h32: (input: string, seed?: number) => string;
h64: (input: string, seedHigh?: number, seedLow?: number) => string;
constructor(app: App, plugin: ObsidianLiveSyncPlugin, dbname: string) {
@ -189,6 +210,17 @@ class LocalPouchDB {
}
return "disabled";
}
updateRecentModifiedDocs(id: string, rev: string) {
let idrev = id + rev;
this.recentModifiedDocs.push(idrev);
if (this.recentModifiedDocs.length > 10) {
this.recentModifiedDocs = this.recentModifiedDocs.slice(-30);
}
}
isSelfModified(id: string, rev: string): boolean {
let idrev = id + rev;
return this.recentModifiedDocs.indexOf(idrev) !== -1;
}
async initializeDatabase() {
if (this.localDatabase != null) this.localDatabase.close();
this.localDatabase = null;
@ -231,21 +263,29 @@ class LocalPouchDB {
_deleted: obj._deleted,
_rev: obj._rev,
children: [],
datatype: "newnote",
};
return doc;
// simple note
}
if (obj.type == "newnote") {
if (obj.type == "newnote" || obj.type == "plain") {
// search childrens
try {
let childrens = [];
for (var v of obj.children) {
// childPromise.push(this.localDatabase.get(v));
let elem = await this.localDatabase.get(v);
if (elem.type && elem.type == "leaf") {
childrens.push(elem.data);
} else {
throw new Error("linked document is not leaf");
try {
let elem = await this.localDatabase.get(v);
if (elem.type && elem.type == "leaf") {
childrens.push(elem.data);
} else {
throw new Error("linked document is not leaf");
}
} catch (ex) {
if (ex.status && ex.status == 404) {
this.addLog(`Missing document content!, could not read ${v} of ${obj._id} from database.`, true);
return false;
}
throw ex;
}
}
let data = childrens.join("");
@ -258,12 +298,13 @@ class LocalPouchDB {
_deleted: obj._deleted,
_rev: obj._rev,
children: obj.children,
datatype: obj.type
};
return doc;
} catch (ex) {
if (ex.status && ex.status == 404) {
this.addLog(`Missing document content!, could not read ${obj._id} from database.`, true);
// this.addLog(ex);
return false;
}
this.addLog(`Something went wrong on reading ${obj._id} from database.`, true);
@ -295,13 +336,15 @@ class LocalPouchDB {
if (!obj.type || (obj.type && obj.type == "notes")) {
obj._deleted = true;
let r = await this.localDatabase.put(obj);
this.updateRecentModifiedDocs(r.id, r.rev);
return true;
// simple note
}
if (obj.type == "newnote") {
if (obj.type == "newnote" || obj.type == "plain") {
obj._deleted = true;
await this.localDatabase.put(obj);
let r = await this.localDatabase.put(obj);
this.addLog(`entry removed:${obj._id}`);
this.updateRecentModifiedDocs(r.id, r.rev);
return true;
}
} catch (ex) {
@ -318,15 +361,61 @@ class LocalPouchDB {
let processed = 0;
let made = 0;
let skiped = 0;
let pieceSize = MAX_DOC_SIZE;
if (!note._id.endsWith(".md")) {
pieceSize = MAX_DOC_SIZE_BIN;
let pieceSize = MAX_DOC_SIZE_BIN;
let plainSplit = false;
if (note._id.endsWith(".md")) {
pieceSize = MAX_DOC_SIZE;
plainSplit = true;
}
do {
// To keep low bandwith and database size,
// Dedup pieces on database.
let piece = leftData.substring(0, pieceSize);
leftData = leftData.substring(pieceSize);
// from 0.1.10, for best performance. we use markdown delimiters
// 1. \n[^\n]{longLineThreshold}[^\n]*\n -> long sentence shuld break.
// 2. \n\n shold break
// 3. \r\n\r\n should break
// 4. \n# should break.
let cPieceSize = pieceSize
let minimumChunkSize = this.plugin.settings.minimumChunkSize;
if (minimumChunkSize < 10) minimumChunkSize = 10;
let longLineThreshold = this.plugin.settings.longLineThreshold;
if (longLineThreshold < 100) longLineThreshold = 100;
if (plainSplit) {
cPieceSize = 0;
// lookup for next splittion .
// we're standing on "\n"
// debugger
do {
let n1 = leftData.indexOf("\n", cPieceSize + 1);
let n2 = leftData.indexOf("\n\n", cPieceSize + 1);
let n3 = leftData.indexOf("\r\n\r\n", cPieceSize + 1);
let n4 = leftData.indexOf("\n#", cPieceSize + 1);
if (n1 == -1 && n2 == -1 && n3 == -1 && n4 == -1) {
cPieceSize = MAX_DOC_SIZE;
break;
}
if (n1 > longLineThreshold) {
// long sentence is an established piece
cPieceSize = n1 + 1;
} else {
// cPieceSize = Math.min.apply([n2, n3, n4].filter((e) => e > 1));
// ^ heavy.
if (n2 > 0 && cPieceSize < n2) cPieceSize = n2 + 1;
if (n3 > 0 && cPieceSize < n3) cPieceSize = n3 + 3;
if (n4 > 0 && cPieceSize < n4) cPieceSize = n4 + 0;
cPieceSize++;
}
} while (cPieceSize < minimumChunkSize)
// console.log("and we use:" + cPieceSize)
}
let piece = leftData.substring(0, cPieceSize);
// if (plainSplit) {
// this.addLog(`piece_len:${cPieceSize}`);
// this.addLog("piece:" + piece);
// }
leftData = leftData.substring(cPieceSize);
processed++;
// Get has of piece.
let hashedPiece = this.h32(piece);
@ -342,7 +431,7 @@ class LocalPouchDB {
// console.log(nleafid);
let pieceData = await this.localDatabase.get<EntryLeaf>(nleafid);
if (pieceData.type == "leaf" && pieceData.data == piece) {
this.addLog("hash:data exists.");
// this.addLog("hash:data exists.");
leafid = nleafid;
needMake = false;
tryNextHash = false;
@ -351,14 +440,14 @@ class LocalPouchDB {
hashQ++;
tryNextHash = true;
} else {
this.addLog("hash:no collision, it's not leaf. what's going on..");
// this.addLog("hash:no collision, it's not leaf. what's going on..");
leafid = nleafid;
tryNextHash = false;
}
} catch (ex) {
if (ex.status && ex.status == 404) {
//not found, we can use it.
this.addLog(`hash:not found.`);
// this.addLog(`hash:not found.`);
leafid = nleafid;
needMake = true;
} else {
@ -375,6 +464,7 @@ class LocalPouchDB {
type: "leaf",
};
let result = await this.localDatabase.put(d);
this.updateRecentModifiedDocs(result.id, result.rev);
if (result.ok) {
this.addLog(`ok:saven`);
made++;
@ -387,21 +477,21 @@ class LocalPouchDB {
savenNotes.push(leafid);
} while (leftData != "");
this.addLog(`note content saven, pieces:${processed} new:${made}, skip:${skiped}`);
let newDoc: NewEntry = {
let newDoc: PlainEntry | NewEntry = {
NewNote: true,
children: savenNotes,
_id: note._id,
ctime: note.ctime,
mtime: note.mtime,
size: note.size,
type: "newnote",
type: plainSplit ? "plain" : "newnote",
};
let deldocs: string[] = [];
// Here for upsert logic,
try {
let old = await this.localDatabase.get(newDoc._id);
if (!old.type || old.type == "notes" || old.type == "newnote") {
if (!old.type || old.type == "notes" || old.type == "newnote" || old.type == "plain") {
// simple use rev for new doc
newDoc._rev = old._rev;
}
@ -412,13 +502,18 @@ class LocalPouchDB {
throw ex;
}
}
await this.localDatabase.put(newDoc);
let r = await this.localDatabase.put(newDoc);
this.updateRecentModifiedDocs(r.id, r.rev);
this.addLog(`note saven:${newDoc._id}`);
}
syncHandler: PouchDB.Replication.Sync<{}> = null;
async openReplication(setting: ObsidianLiveSyncSettings, keepAlive: boolean, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<{}>[]) => Promise<void>) {
if (setting.versionUpFlash != "") {
new Notice("Open settings and check message, please.");
return;
}
let uri = setting.couchDB_URI;
let auth: Credential = {
username: setting.couchDB_USER,
@ -438,6 +533,7 @@ class LocalPouchDB {
//replicate once
let replicate = this.localDatabase.replicate.from(db);
// console.log("replication start.")
replicate
.on("change", async (e) => {
try {
@ -452,8 +548,14 @@ class LocalPouchDB {
replicate.removeAllListeners();
replicate.cancel();
// this.syncHandler = null;
if (this.syncHandler != null) {
this.syncHandler.removeAllListeners();
}
this.syncHandler = this.localDatabase.sync(db, syncOption);
this.syncHandler
.on("active", () => {
this.addLog("Replication activated");
})
.on("change", async (e) => {
try {
callback(e.change.docs);
@ -463,13 +565,8 @@ class LocalPouchDB {
this.addLog(ex);
}
})
.on("active", () => {
this.addLog("Replication activated");
})
.on("complete", (e) => {
this.addLog("Replication completed", showResult);
// this.addLog(e);
console.dir(this.syncHandler);
this.syncHandler = null;
})
.on("denied", (e) => {
@ -553,7 +650,7 @@ class LocalPouchDB {
//there are some result
for (let v of result.rows) {
let doc = v.doc;
if (doc.type == "newnote") {
if (doc.type == "newnote" || doc.type == "plain") {
// used pieces memo.
usedPieces = Array.from(new Set([...usedPieces, ...doc.children]));
}
@ -599,9 +696,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async onload() {
this.addLog = this.addLog.bind(this);
this.addLog("loading plugin");
await this.openDatabase();
const lsname = "obsidian-live-sync-ver" + this.app.vault.getName();
const last_version = localStorage.getItem(lsname);
await this.loadSettings();
if (!last_version || Number(last_version) < VER) {
this.settings.liveSync = false;
this.settings.syncOnSave = false;
this.settings.syncOnStart = false;
this.settings.versionUpFlash = "I changed specifications incompatiblly, so when you enable sync again, be sure to made version up all nother devides.";
this.saveSettings();
}
localStorage.setItem(lsname, `${VER}`);
await this.openDatabase();
addIcon(
"replicate",
`<g transform="matrix(1.15 0 0 1.15 -8.31 -9.52)" fill="currentColor" fill-rule="evenodd">
@ -811,12 +918,21 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async doc2storage_create(docEntry: Entry, force?: boolean) {
let doc = await this.localDatabase.getDatabaseDoc(docEntry._id, { _rev: docEntry._rev });
if (doc === false) return;
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
await this.ensureDirectory(doc._id);
let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
this.addLog("live : write to local (newfile:b) " + doc._id);
await this.app.vault.trigger("create", newfile);
}
} else if (doc.datatype == "plain") {
await this.ensureDirectory(doc._id);
let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
this.addLog("live : write to local (newfile) " + doc._id);
let newfile = await this.app.vault.create(doc._id, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
this.addLog("live : write to local (newfile:p) " + doc._id);
await this.app.vault.trigger("create", newfile);
} else {
this.addLog("live : New data imcoming, but we cound't parse that.1" + doc.datatype, true);
}
}
@ -846,11 +962,22 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (file.stat.mtime < docEntry.mtime || force) {
let doc = await this.localDatabase.getDatabaseDoc(docEntry._id);
if (doc === false) return;
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
// debugger;
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
this.addLog("livesync : newer local files so write to local:" + file.path);
await this.app.vault.trigger("modify", file);
}
} if (doc.datatype == "plain") {
await this.ensureDirectory(doc._id);
await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
this.addLog("livesync : newer local files so write to local:" + file.path);
await this.app.vault.trigger("modify", file);
} else {
this.addLog("live : New data imcoming, but we cound't parse that.2:" + doc.datatype + "-", true);
}
} else if (file.stat.mtime > docEntry.mtime) {
// newer local file.
@ -881,8 +1008,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
//---> Sync
async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<Entry>>): Promise<void> {
for (var change of docs) {
if (this.localDatabase.isSelfModified(change._id, change._rev)) {
return;
}
this.addLog("replication change arrived");
await this.pouchdbChanged(change);
this.gcHook();
}
}
async realizeSettingSyncMode() {
@ -897,6 +1028,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.statusBar.setText("Sync:" + statusStr);
}
async replicate(showMessage?: boolean) {
if (this.settings.versionUpFlash != "") {
new Notice("Open settings and check message, please.");
return;
}
this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
}
//<-- Sync
@ -983,11 +1118,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
try {
let doc = await this.localDatabase.getDatabaseDoc(path, { rev: rev });
if (doc === false) return false;
let data = doc.data;
if (doc.datatype == "newnote") {
data = base64ToString(doc.data);
} else if (doc.datatype == "plain") {
data = doc.data;
}
return {
ctime: doc.ctime,
mtime: doc.mtime,
rev: rev,
data: base64ToString(doc.data),
data: data,
};
} catch (ex) {
if (ex.status && ex.status == 404) {
@ -1109,8 +1250,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async updateIntoDB(file: TFile) {
let contentBin = await this.app.vault.readBinary(file);
let content = arrayBufferToBase64(contentBin);
let content = "";
let datatype: "plain" | "newnote" = "newnote";
if (file.extension != "md") {
let contentBin = await this.app.vault.readBinary(file);
content = arrayBufferToBase64(contentBin);
datatype = "newnote";
} else {
content = await this.app.vault.read(file);
datatype = "plain";
}
let fullpath = file.path;
let d: LoadedEntry = {
_id: fullpath,
@ -1119,6 +1268,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
mtime: file.stat.mtime,
size: file.stat.size,
children: [],
datatype: datatype
};
//From here
let old = await this.localDatabase.getDatabaseDoc(fullpath);
@ -1133,7 +1283,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
let ret = await this.localDatabase.putDBEntry(d);
this.addLog("put database:" + fullpath);
this.addLog("put database:" + fullpath + "(" + datatype + ")");
if (this.settings.syncOnSave) {
await this.replicate();
}
@ -1371,6 +1521,18 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
if (this.plugin.settings.versionUpFlash != "") {
let c = containerEl.createEl("div", { text: this.plugin.settings.versionUpFlash });
c.createEl("button", { text: "I got it and updated." }, (e) => {
e.addEventListener("click", async () => {
this.plugin.settings.versionUpFlash = "";
this.plugin.saveSettings();
c.remove();
});
});
c.addClass("op-warn")
}
// containerEl.createDiv(this.plugin.settings.versionUpFlash);
new Setting(containerEl)
.setName("LiveSync")
.setDesc("Sync realtime")
@ -1399,6 +1561,38 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("Minimum chunk size")
.setDesc("(letters), minimum chunk size.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.minimumChunkSize + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
return 10;
}
this.plugin.settings.minimumChunkSize = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerEl)
.setName("LongLine Threshold")
.setDesc("(letters), If the line is longer than this, make the line to chunk")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.longLineThreshold + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
return 10;
}
this.plugin.settings.longLineThreshold = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerEl)
.setName("Local Database Operations")
.addButton((button) =>

2
manifest.json
View File

@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Obsidian Live sync",
"version": "0.0.9",
"version": "0.1.0",
"minAppVersion": "0.9.12",
"description": "obsidian Live synchronization plugin.",
"author": "vorotamoroz",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.0.9",
"version": "0.10.0",
"description": "obsidian Live synchronization plugin.",
"main": "main.js",
"scripts": {

5
styles.css
View File

@ -18,3 +18,8 @@
.op-pre {
white-space: pre-wrap;
}
.op-warn {
border:1px solid salmon;
padding:2px;
border-radius: 4px;
}