1
0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2024-12-12 09:04:06 +02:00

fix some,

- Add Utility functions.
- Database reset opeartion and corrupt preventions.
- Fixing file deleting.
- Tidy up setting dialog.
- Add notice about the file that having platform dependant name.
- Add webclip on readme
This commit is contained in:
vorotamoroz 2021-10-26 18:08:01 +09:00
parent a01079d4b1
commit ccbf1b2ffe
5 changed files with 369 additions and 65 deletions

View File

@ -16,6 +16,7 @@ Limitations: Folder deletion handling is not completed.
- Live Sync
- Self-Hosted data synchronization with conflict detection and resolving in Obsidian.
- Off-line sync is also available.
- Receive WebClip from [obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)
## IMPORTANT NOTICE
@ -33,11 +34,14 @@ If you want to synchronize to both backend, sync one by one, please.
5. Setup LiveSync or SyncOnSave or SyncOnStart as you like.
## Test Server
Setting up an instance of Cloudant or local CouchDB is a little complicated, so I made the [Tasting server of obsidian-livesync](https://olstaste.vrtmrz.net/) up. Try free!
Setting up an instance of Cloudant or local CouchDB is a little complicated, so I made the [Tasting server of obsidian-livesync](https://olstaste.vrtmrz.net/) up. Try free!
Note: Please read "Limitations" carefully. Do not send your private vault.
## WebClipper is also available.
See [obsidian-livesync-webclip](https://github.com/vrtmrz/obsidian-livesync-webclip). (Docs are work in progress.)
Available from on Chrome Web Store:[obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)
Repo is here: [obsidian-livesync-webclip](https://github.com/vrtmrz/obsidian-livesync-webclip). (Docs are work in progress.)
## When your database looks corrupted

418
main.ts
View File

@ -21,6 +21,8 @@ const LOG_LEVEL = {
type LOG_LEVEL = typeof LOG_LEVEL[keyof typeof LOG_LEVEL];
const VERSIONINFO_DOCID = "obsydian_livesync_version";
const MILSTONE_DOCID = "_local/obsydian_livesync_milestone";
const NODEINFO_DOCID = "_local/obsydian_livesync_nodeinfo";
interface ObsidianLiveSyncSettings {
couchDB_URI: string;
@ -110,8 +112,27 @@ interface EntryVersionInfo {
version: number;
_deleted?: boolean;
}
interface EntryMilestoneInfo {
_id: typeof MILSTONE_DOCID;
_rev?: string;
type: "milestoneinfo";
_deleted?: boolean;
created: number;
accepted_nodes: string[];
locked: boolean;
}
interface EntryNodeInfo {
_id: typeof NODEINFO_DOCID;
_rev?: string;
_deleted?: boolean;
type: "nodeinfo";
nodeid: string;
}
type EntryBody = Entry | NewEntry | PlainEntry;
type EntryDoc = EntryBody | LoadedEntry | EntryLeaf | EntryVersionInfo;
type EntryDoc = EntryBody | LoadedEntry | EntryLeaf | EntryVersionInfo | EntryMilestoneInfo | EntryNodeInfo;
type diff_result_leaf = {
rev: string;
@ -195,6 +216,13 @@ const escapeStringToHTML = (str: string) => {
return escape[match];
});
};
function resolveWithIgnoreKnownError<T>(p: Promise<T>, def: T): Promise<T> {
return new Promise((res, rej) => {
p.then(res).catch((ex) => (ex.status && ex.status == 404 ? res(def) : rej(ex)));
});
}
const isValidRemoteCouchDBURI = (uri: string): boolean => {
if (uri.startsWith("https://")) return true;
if (uri.startsWith("http://")) return true;
@ -251,22 +279,21 @@ const bumpRemoteVersion = async (db: PouchDB.Database, barrier: number = VER): P
version: barrier,
type: "versioninfo",
};
try {
let versionInfo = (await db.get(VERSIONINFO_DOCID)) as EntryVersionInfo;
if (versionInfo.type != "versioninfo") {
return false;
}
vi._rev = versionInfo._rev;
} catch (ex) {
if (ex.status && ex.status == 404) {
// no op.
} else {
throw ex;
}
let versionInfo = (await resolveWithIgnoreKnownError(db.get(VERSIONINFO_DOCID), vi)) as EntryVersionInfo;
if (versionInfo.type != "versioninfo") {
return false;
}
vi._rev = versionInfo._rev;
await db.put(vi);
return true;
};
function isValidPath(filename: string): boolean {
let regex = /[\u0000-\u001f]|[\\"':?<>|*$]/g;
let x = filename.replace(regex, "_");
let win = /(\\|\/)(COM\d|LPT\d|CON|PRN|AUX|NUL|CLOCK$)($|\.)/gi;
let sx = (x = x.replace(win, "/_"));
return sx == filename;
}
// Default Logger.
let Logger: (message: any, levlel?: LOG_LEVEL) => Promise<void> = async (message, _) => {
@ -282,6 +309,7 @@ class LocalPouchDB {
dbname: string;
settings: ObsidianLiveSyncSettings;
localDatabase: PouchDB.Database<EntryDoc>;
nodeid: string = "";
recentModifiedDocs: string[] = [];
h32: (input: string, seed?: number) => string;
@ -294,6 +322,8 @@ class LocalPouchDB {
} = {};
corruptedEntries: { [key: string]: EntryDoc } = {};
remoteLocked = false;
remoteLockedAndDeviceNotAccepted = false;
constructor(settings: ObsidianLiveSyncSettings, dbname: string) {
this.auth = {
@ -345,6 +375,17 @@ class LocalPouchDB {
revs_limit: 100,
deterministic_revs: true,
});
// initialize local node information.
let nodeinfo: EntryNodeInfo = await resolveWithIgnoreKnownError<EntryNodeInfo>(this.localDatabase.get(NODEINFO_DOCID), {
_id: NODEINFO_DOCID,
type: "nodeinfo",
nodeid: "",
});
if (nodeinfo.nodeid == "") {
nodeinfo.nodeid = Math.random().toString(36).slice(-10);
await this.localDatabase.put(nodeinfo);
}
this.nodeid = nodeinfo.nodeid;
// Traceing the leaf id
let changes = this.localDatabase
@ -560,7 +601,54 @@ class LocalPouchDB {
throw ex;
}
}
async deleteDBEntryPrefix(prefix: string): Promise<boolean> {
// delete database entries by prefix.
// it called from folder deletion.
let c = 0;
let readCount = 0;
let delDocs: string[] = [];
do {
let result = await this.localDatabase.allDocs({ include_docs: false, skip: c, limit: 100, conflicts: true });
readCount = result.rows.length;
if (readCount > 0) {
//there are some result
for (let v of result.rows) {
// let doc = v.doc;
if (v.id.startsWith(prefix) || v.id.startsWith("/" + prefix)) {
delDocs.push(v.id);
console.log("!" + v.id);
} else {
if (!v.id.startsWith("h:")) {
console.log("?" + v.id);
}
}
}
}
c += readCount;
} while (readCount != 0);
// items collected.
//bulk docs to delete?
let deleteCount = 0;
let notfound = 0;
for (let v of delDocs) {
try {
let item = await this.localDatabase.get(v);
item._deleted = true;
await this.localDatabase.put(item);
this.updateRecentModifiedDocs(item._id, item._rev, true);
deleteCount++;
} catch (ex) {
if (ex.status && ex.status == 404) {
notfound++;
// NO OP. It should be timing problem.
} else {
throw ex;
}
}
}
Logger(`deleteDBEntryPrefix:deleted ${deleteCount} items, skipped ${notfound}`);
return true;
}
async putDBEntry(note: LoadedEntry) {
let leftData = note.data;
let savenNotes = [];
@ -727,6 +815,47 @@ class LocalPouchDB {
// no op now,
return true;
}
replicateAllToServer(setting: ObsidianLiveSyncSettings) {
return new Promise(async (res, rej) => {
this.closeReplication();
Logger("send all data to server", LOG_LEVEL.NOTICE);
let uri = setting.couchDB_URI;
let auth: Credential = {
username: setting.couchDB_USER,
password: setting.couchDB_PASSWORD,
};
let dbret = await connectRemoteCouchDB(uri, auth);
if (dbret === false) {
Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
return rej(`could not connect to ${uri}`);
}
let syncOptionBase: PouchDB.Replication.SyncOptions = {
batch_size: 250,
batches_limit: 40,
};
let db = dbret.db;
//replicate once
let replicate = this.localDatabase.replicate.to(db, syncOptionBase);
replicate
.on("change", async (e) => {
// no op.
Logger(`sending..:${e.docs.length}`);
})
.on("complete", async (info) => {
Logger("Completed", LOG_LEVEL.NOTICE);
replicate.cancel();
replicate.removeAllListeners();
res(true);
})
.on("error", (e) => {
Logger("Pulling Replication error", LOG_LEVEL.NOTICE);
Logger(e);
rej(e);
});
});
}
async openReplication(setting: ObsidianLiveSyncSettings, keepAlive: boolean, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<{}>[]) => Promise<void>) {
if (setting.versionUpFlash != "") {
new Notice("Open settings and check message, please.");
@ -752,6 +881,26 @@ class LocalPouchDB {
return;
}
let defMilestonePoint: EntryMilestoneInfo = {
_id: MILSTONE_DOCID,
type: "milestoneinfo",
created: (new Date() as any) / 1,
locked: false,
accepted_nodes: [this.nodeid],
};
let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defMilestonePoint);
this.remoteLocked = remoteMilestone.locked;
this.remoteLockedAndDeviceNotAccepted = remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1;
if (remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1) {
Logger("Remote database marked as 'Auto Sync Locked'. And this devide does not marked as resolved device. see settings dialog.", LOG_LEVEL.NOTICE);
return;
}
if (typeof remoteMilestone._rev == "undefined") {
await dbret.db.put(remoteMilestone);
}
let syncOptionBase: PouchDB.Replication.SyncOptions = {
batch_size: 250,
batches_limit: 40,
@ -869,6 +1018,71 @@ class LocalPouchDB {
if (con2 === false) return;
Logger("Remote Database Created or Connected", LOG_LEVEL.NOTICE);
}
async markRemoteLocked(setting: ObsidianLiveSyncSettings, locked: boolean) {
let uri = setting.couchDB_URI;
let auth: Credential = {
username: setting.couchDB_USER,
password: setting.couchDB_PASSWORD,
};
let dbret = await connectRemoteCouchDB(uri, auth);
if (dbret === false) {
Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
return;
}
if (!(await checkRemoteVersion(dbret.db, this.migrate.bind(this), VER))) {
Logger("Remote database is newer or corrupted, make sure to latest version of obsidian-livesync installed", LOG_LEVEL.NOTICE);
return;
}
let defInitPoint: EntryMilestoneInfo = {
_id: MILSTONE_DOCID,
type: "milestoneinfo",
created: (new Date() as any) / 1,
locked: locked,
accepted_nodes: [this.nodeid],
};
let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint);
remoteMilestone.accepted_nodes = [this.nodeid];
remoteMilestone.locked = locked;
if (locked) {
Logger("Lock remote database to prevent data corruption", LOG_LEVEL.NOTICE);
} else {
Logger("Unlock remote database to prevent data corruption", LOG_LEVEL.NOTICE);
}
await dbret.db.put(remoteMilestone);
}
async markRemoteResolved(setting: ObsidianLiveSyncSettings) {
let uri = setting.couchDB_URI;
let auth: Credential = {
username: setting.couchDB_USER,
password: setting.couchDB_PASSWORD,
};
let dbret = await connectRemoteCouchDB(uri, auth);
if (dbret === false) {
Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
return;
}
if (!(await checkRemoteVersion(dbret.db, this.migrate.bind(this), VER))) {
Logger("Remote database is newer or corrupted, make sure to latest version of obsidian-livesync installed", LOG_LEVEL.NOTICE);
return;
}
let defInitPoint: EntryMilestoneInfo = {
_id: MILSTONE_DOCID,
type: "milestoneinfo",
created: (new Date() as any) / 1,
locked: false,
accepted_nodes: [this.nodeid],
};
// check local database hash status and remote replicate hash status
let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint);
// remoteMilestone.locked = false;
remoteMilestone.accepted_nodes = Array.from(new Set([...remoteMilestone.accepted_nodes, this.nodeid]));
// this.remoteLocked = false;
Logger("Mark this device as 'resolved'.", LOG_LEVEL.NOTICE);
await dbret.db.put(remoteMilestone);
}
async garbageCollect() {
// get all documents of NewEntry2
@ -1202,12 +1416,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
if (!isValidPath(doc._id)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
return;
}
await this.ensureDirectory(doc._id);
let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
Logger("live : write to local (newfile:b) " + doc._id);
await this.app.vault.trigger("create", newfile);
}
} else if (doc.datatype == "plain") {
if (!isValidPath(doc._id)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
return;
}
await this.ensureDirectory(doc._id);
let newfile = await this.app.vault.create(doc._id, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
Logger("live : write to local (newfile:p) " + doc._id);
@ -1251,6 +1473,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (doc.datatype == "newnote") {
let bin = base64ToArrayBuffer(doc.data);
if (bin != null) {
if (!isValidPath(doc._id)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
return;
}
await this.ensureDirectory(doc._id);
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg);
@ -1258,6 +1484,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
}
if (doc.datatype == "plain") {
if (!isValidPath(doc._id)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
return;
}
await this.ensureDirectory(doc._id);
await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg);
@ -1298,7 +1528,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return;
}
Logger("replication change arrived", LOG_LEVEL.VERBOSE);
if (change.type != "leaf" && change.type != "versioninfo") {
if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo") {
await this.handleDBChanged(change);
}
if (change.type == "versioninfo") {
@ -1333,6 +1563,18 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
await this.openDatabase();
await this.syncAllFiles();
}
async replicateAllToServer() {
return await this.localDatabase.replicateAllToServer(this.settings);
}
async markRemoteLocked() {
return await this.localDatabase.markRemoteLocked(this.settings, true);
}
async markRemoteUnlocked() {
return await this.localDatabase.markRemoteLocked(this.settings, false);
}
async markRemoteResolved() {
return await this.localDatabase.markRemoteResolved(this.settings);
}
async syncAllFiles() {
// synchronize all files between database and storage.
const filesStorage = this.app.vault.getFiles();
@ -1362,6 +1604,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async deleteFolderOnDB(folder: TFolder) {
Logger(`delete folder:${folder.path}`);
await this.localDatabase.deleteDBEntryPrefix(folder.path + "/");
for (var v of folder.children) {
let entry = v as TFile & TFolder;
Logger(`->entry:${entry.path}`, LOG_LEVEL.VERBOSE);
@ -1374,7 +1617,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (ex.code && ex.code == "ENOENT") {
//NO OP.
} else {
Logger(`error while delete filder:${entry.path}`, LOG_LEVEL.NOTICE);
Logger(`error while delete folder:${entry.path}`, LOG_LEVEL.NOTICE);
Logger(ex);
}
}
@ -1788,14 +2031,17 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
});
text.inputEl.setAttribute("type", "password");
});
new Setting(containerEl).setName("Test DB").addButton((button) =>
button
.setButtonText("Test Database Connection")
.setDisabled(false)
.onClick(async () => {
await this.testConnection();
})
);
new Setting(containerEl)
.setName("Test Database Connection")
.setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.")
.addButton((button) =>
button
.setButtonText("Test")
.setDisabled(false)
.onClick(async () => {
await this.testConnection();
})
);
containerEl.createEl("h3", { text: "Database configuration" });
@ -1931,24 +2177,6 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text.inputEl.setAttribute("type", "number");
});
new Setting(containerEl).setName("Local Database Operations").addButton((button) =>
button
.setButtonText("Reset local database")
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
})
);
new Setting(containerEl).setName("Re-init").addButton((button) =>
button
.setButtonText("Init Database again")
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase();
})
);
new Setting(containerEl).setName("Garbage Collect").addButton((button) =>
button
.setButtonText("Garbage Collection")
@ -1960,6 +2188,60 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
containerEl.createEl("h3", { text: "Hatch" });
if (this.plugin.localDatabase.remoteLockedAndDeviceNotAccepted) {
let c = containerEl.createEl("div", {
text: "To prevent unwanted vault corruption, the remote database has been locked for synchronization, and this device was not marked as 'resolved'. it caused by some operations like this. re-initialized. Local database initialization should be required. please back your vault up, reset local database, and press 'Mark this device as resolved'. ",
});
c.createEl("button", { text: "I'm ready, mark this device 'resolved'" }, (e) => {
e.addEventListener("click", async () => {
await this.plugin.markRemoteResolved();
c.remove();
});
});
c.addClass("op-warn");
} else {
if (this.plugin.localDatabase.remoteLocked) {
let c = containerEl.createEl("div", {
text: "To prevent unwanted vault corruption, the remote database has been locked for synchronization. (This device is marked 'resolved') When all your devices are marked 'resolved', unlock the database.",
});
c.createEl("button", { text: "I'm ready, unlock the database" }, (e) => {
e.addEventListener("click", async () => {
await this.plugin.markRemoteUnlocked();
c.remove();
});
});
c.addClass("op-warn");
}
}
new Setting(containerEl)
.setName("Drop History")
.setDesc("Initialize local and remote database, and create local database from storage and put all into server. And also, lock the database to prevent data corruption.")
.addButton((button) =>
button
.setButtonText("Execute")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer();
})
);
new Setting(containerEl)
.setName("Lock remote database")
.setDesc("Lock remote database for synchronize")
.addButton((button) =>
button
.setButtonText("Lock")
.setDisabled(false)
.onClick(async () => {
await this.plugin.markRemoteLocked();
})
);
new Setting(containerEl)
.setName("Suspend file watching")
.setDesc("if enables it, all file operations are ignored.")
@ -1970,23 +2252,41 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
})
);
new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
button
.setButtonText("Reset remote database")
.setDisabled(false)
.onClick(async () => {
await this.plugin.tryResetRemoteDatabase();
})
);
new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
button
.setButtonText("Create remote database")
.setDisabled(false)
.onClick(async () => {
await this.plugin.tryResetRemoteDatabase();
})
);
new Setting(containerEl)
.setName("Reset remote database")
.setDesc("Reset remote database, this affects only database. If you replicate again, remote database will restored by local database.")
.addButton((button) =>
button
.setButtonText("Reset")
.setDisabled(false)
.onClick(async () => {
await this.plugin.tryResetRemoteDatabase();
})
);
new Setting(containerEl)
.setName("Reset local database")
.setDesc("Reset local database, this affects only database. If you replicate again, local database will restored by remote database.")
.addButton((button) =>
button
.setButtonText("Reset")
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
})
);
new Setting(containerEl)
.setName("Initialize local database again")
.setDesc("WARNING: Reset local database and reconstruct by storage data. It affects local database, but if you replicate remote as is, remote data will be merged or corrupted.")
.addButton((button) =>
button
.setButtonText("INITIALIZE")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase();
})
);
containerEl.createEl("h3", { text: "Corrupted data" });
if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {

View File

@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Obsidian Live sync",
"version": "0.1.6",
"version": "0.1.7",
"minAppVersion": "0.9.12",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.1.6",
"version": "0.1.7",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.1.6",
"version": "0.1.7",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",

View File

@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.1.6",
"version": "0.1.7",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"scripts": {