self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-01-05 12:50:41 +02:00
Code Issues Releases Activity

Fixed:

Browse Source 
- Removed waiting delay on fetching files from the database when launch.
- Database had not been closed right timings.
- Fixed wrong message.

Improved:
- Plugins and Settings sync is not compatible with Cloudant.
- "Restore from file" is added on "Corrupted data".
This commit is contained in:
vorotamoroz 2021-11-26 07:52:15 +09:00
parent 57f91eb407
commit 3435efaf89
2 changed files with 82 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
main.ts
View File

@ -584,6 +584,7 @@ class LocalPouchDB {
// this.initializeDatabase(); // this.initializeDatabase();
} }
close() { close() {
Logger("Database closed (by close)");
this.isReady = false; this.isReady = false;
if (this.changeHandler != null) { if (this.changeHandler != null) {
this.changeHandler.cancel(); this.changeHandler.cancel();
@ -645,6 +646,7 @@ class LocalPouchDB {
await this.localDatabase.put(nodeinfo); await this.localDatabase.put(nodeinfo);
} }
this.localDatabase.on("close", () => { this.localDatabase.on("close", () => {
Logger("Database closed.");
this.isReady = false; this.isReady = false;
}); });
this.nodeid = nodeinfo.nodeid; this.nodeid = nodeinfo.nodeid;
@ -663,6 +665,7 @@ class LocalPouchDB {
}); });
this.changeHandler = changes; this.changeHandler = changes;
this.isReady = true; this.isReady = true;
Logger("Database is now ready.");
} }
async prepareHashFunctions() { async prepareHashFunctions() {
@ -688,7 +691,7 @@ class LocalPouchDB {
waitForLeafReady(id: string): Promise<boolean> { waitForLeafReady(id: string): Promise<boolean> {
return new Promise((res, rej) => { return new Promise((res, rej) => {
// Set timeout. // Set timeout.
let timer = setTimeout(() => rej(false), LEAF_WAIT_TIMEOUT); let timer = setTimeout(() => rej(new Error(`Leaf timed out:${id}`)), LEAF_WAIT_TIMEOUT);
if (typeof this.leafArrivedCallbacks[id] == "undefined") { if (typeof this.leafArrivedCallbacks[id] == "undefined") {
this.leafArrivedCallbacks[id] = []; this.leafArrivedCallbacks[id] = [];
} }
@ -699,7 +702,7 @@ class LocalPouchDB {
}); });
} }
async getDBLeaf(id: string): Promise<string> { async getDBLeaf(id: string, waitForReady: boolean): Promise<string> {
// when in cache, use that. // when in cache, use that.
if (this.hashCacheRev[id]) { if (this.hashCacheRev[id]) {
return this.hashCacheRev[id]; return this.hashCacheRev[id];
@ -721,7 +724,7 @@ class LocalPouchDB {
} }
throw new Error(`retrive leaf, but it was not leaf.`); throw new Error(`retrive leaf, but it was not leaf.`);
} catch (ex) { } catch (ex) {
if (ex.status && ex.status == 404) { if (ex.status && ex.status == 404 && waitForReady) {
// just leaf is not ready. // just leaf is not ready.
// wait for on // wait for on
if ((await this.waitForLeafReady(id)) === false) { if ((await this.waitForLeafReady(id)) === false) {
@ -776,6 +779,10 @@ class LocalPouchDB {
// retrieve metadata only // retrieve metadata only
if (!obj.type || (obj.type && obj.type == "notes") || obj.type == "newnote" || obj.type == "plain") { if (!obj.type || (obj.type && obj.type == "notes") || obj.type == "newnote" || obj.type == "plain") {
let note = obj as Entry; let note = obj as Entry;
let children: string[] = [];
if (obj.type == "newnote" || obj.type == "plain") {
children = obj.children;
}
let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = { let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
data: "", data: "",
_id: note._id, _id: note._id,
@ -785,7 +792,7 @@ class LocalPouchDB {
_deleted: obj._deleted, _deleted: obj._deleted,
_rev: obj._rev, _rev: obj._rev,
_conflicts: obj._conflicts, _conflicts: obj._conflicts,
children: [], children: children,
datatype: "newnote", datatype: "newnote",
}; };
return doc; return doc;
@ -798,7 +805,7 @@ class LocalPouchDB {
} }
return false; return false;
} }
async getDBEntry(path: string, opt?: PouchDB.Core.GetOptions, dump = false): Promise<false | LoadedEntry> { async getDBEntry(path: string, opt?: PouchDB.Core.GetOptions, dump = false, waitForReady = true): Promise<false | LoadedEntry> {
let id = path2id(path); let id = path2id(path);
try { try {
let obj: EntryDocResponse = null; let obj: EntryDocResponse = null;
@ -848,13 +855,14 @@ class LocalPouchDB {
} }
let childrens: string[]; let childrens: string[];
try { try {
childrens = await Promise.all(obj.children.map((e) => this.getDBLeaf(e))); childrens = await Promise.all(obj.children.map((e) => this.getDBLeaf(e, waitForReady)));
if (dump) { if (dump) {
Logger(`childrens:`); Logger(`childrens:`);
Logger(childrens); Logger(childrens);
} }
} catch (ex) { } catch (ex) {
Logger(`Something went wrong on reading elements of ${obj._id} from database.`, LOG_LEVEL.NOTICE); Logger(`Something went wrong on reading elements of ${obj._id} from database.`, LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE);
this.corruptedEntries[obj._id] = obj; this.corruptedEntries[obj._id] = obj;
return false; return false;
} }
@ -1161,7 +1169,7 @@ class LocalPouchDB {
} else { } else {
Logger(`save failed:id:${item.id} rev:${item.rev}`, LOG_LEVEL.NOTICE); Logger(`save failed:id:${item.id} rev:${item.rev}`, LOG_LEVEL.NOTICE);
Logger(item); Logger(item);
this.disposeHashCache(); // this.disposeHashCache();
saved = false; saved = false;
} }
} }
@ -1476,6 +1484,7 @@ class LocalPouchDB {
this.changeHandler.cancel(); this.changeHandler.cancel();
} }
await this.closeReplication(); await this.closeReplication();
Logger("Database closed for reset Database.");
this.isReady = false; this.isReady = false;
await this.localDatabase.destroy(); await this.localDatabase.destroy();
this.localDatabase = null; this.localDatabase = null;
@ -1741,7 +1750,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
name: "Dump informations of this doc ", name: "Dump informations of this doc ",
editorCallback: (editor: Editor, view: MarkdownView) => { editorCallback: (editor: Editor, view: MarkdownView) => {
//this.replicate(); //this.replicate();
this.localDatabase.getDBEntry(view.file.path, {}, true); this.localDatabase.getDBEntry(view.file.path, {}, true, false);
}, },
}); });
this.addCommand({ this.addCommand({
@ -2182,10 +2191,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.refreshStatusText(); this.refreshStatusText();
for (var change of docs) { for (var change of docs) {
if (this.localDatabase.isSelfModified(change._id, change._rev)) { if (this.localDatabase.isSelfModified(change._id, change._rev)) {
return; continue;
}
if (change._id.startsWith("ps:")) {
continue;
}
if (change._id.startsWith("h:")) {
continue;
} }
Logger("replication change arrived", LOG_LEVEL.VERBOSE);
if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo" && change.type != "plugin") { if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo" && change.type != "plugin") {
Logger("replication change arrived", LOG_LEVEL.VERBOSE);
await this.handleDBChanged(change); await this.handleDBChanged(change);
} }
if (change.type == "versioninfo") { if (change.type == "versioninfo") {
@ -2346,7 +2361,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}); });
await runAll("UPDATE STORAGE", onlyInDatabase, async (e) => { await runAll("UPDATE STORAGE", onlyInDatabase, async (e) => {
Logger(`Pull from db:${e}`); Logger(`Pull from db:${e}`);
await this.pullFile(e, filesStorage); await this.pullFile(e, filesStorage, false, null, false);
}); });
await runAll("CHECK FILE STATUS", syncFiles, async (e) => { await runAll("CHECK FILE STATUS", syncFiles, async (e) => {
await this.syncFileBetweenDBandStorage(e, filesStorage); await this.syncFileBetweenDBandStorage(e, filesStorage);
@ -2547,20 +2562,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}).open(); }).open();
} }
} }
async pullFile(filename: string, fileList?: TFile[], force?: boolean, rev?: string) { async pullFile(filename: string, fileList?: TFile[], force?: boolean, rev?: string, waitForReady: boolean = true) {
if (!fileList) { if (!fileList) {
fileList = this.app.vault.getFiles(); fileList = this.app.vault.getFiles();
} }
let targetFiles = fileList.filter((e) => e.path == id2path(filename)); let targetFiles = fileList.filter((e) => e.path == id2path(filename));
if (targetFiles.length == 0) { if (targetFiles.length == 0) {
//have to create; //have to create;
let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null); let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null, false, waitForReady);
if (doc === false) return; if (doc === false) return;
await this.doc2storage_create(doc, force); await this.doc2storage_create(doc, force);
} else if (targetFiles.length == 1) { } else if (targetFiles.length == 1) {
//normal case //normal case
let file = targetFiles[0]; let file = targetFiles[0];
let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null); let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null, false, waitForReady);
if (doc === false) return; if (doc === false) return;
await this.doc2storate_modify(doc, file, force); await this.doc2storate_modify(doc, file, force);
} else { } else {
@ -2572,18 +2587,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async syncFileBetweenDBandStorage(file: TFile, fileList?: TFile[]) { async syncFileBetweenDBandStorage(file: TFile, fileList?: TFile[]) {
let doc = await this.localDatabase.getDBEntryMeta(file.path); let doc = await this.localDatabase.getDBEntryMeta(file.path);
if (doc === false) return; if (doc === false) return;
let storageMtime = ~~(file.stat.mtime / 1000); let storageMtime = ~~(file.stat.mtime / 1000);
let docMtime = ~~(doc.mtime / 1000); let docMtime = ~~(doc.mtime / 1000);
if (storageMtime > docMtime) { if (storageMtime > docMtime) {
//newer local file. //newer local file.
Logger("DB -> STORAGE :" + file.path); Logger("STORAGE -> DB :" + file.path);
Logger(`${storageMtime} > ${docMtime}`); Logger(`${storageMtime} > ${docMtime}`);
await this.updateIntoDB(file); await this.updateIntoDB(file);
} else if (storageMtime < docMtime) { } else if (storageMtime < docMtime) {
//newer database file. //newer database file.
Logger("STORAGE <- DB :" + file.path); Logger("STORAGE <- DB :" + file.path);
Logger(`${storageMtime} < ${docMtime}`); Logger(`${storageMtime} < ${docMtime}`);
let docx = await this.localDatabase.getDBEntry(file.path); let docx = await this.localDatabase.getDBEntry(file.path, null, false, false);
if (docx != false) { if (docx != false) {
await this.doc2storate_modify(docx, file); await this.doc2storate_modify(docx, file);
} }
@ -2616,7 +2632,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
datatype: datatype, datatype: datatype,
}; };
//From here //From here
let old = await this.localDatabase.getDBEntry(fullpath); let old = await this.localDatabase.getDBEntry(fullpath, null, false, false);
if (old !== false) { if (old !== false) {
let oldData = { data: old.data, deleted: old._deleted }; let oldData = { data: old.data, deleted: old._deleted };
let newData = { data: d.data, deleted: d._deleted }; let newData = { data: d.data, deleted: d._deleted };
@ -3324,9 +3340,6 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
// With great respect, thank you TfTHacker! // With great respect, thank you TfTHacker!
// refered: https://github.com/TfTHacker/obsidian42-brat/blob/main/src/features/BetaPlugins.ts // refered: https://github.com/TfTHacker/obsidian42-brat/blob/main/src/features/BetaPlugins.ts
containerEl.createEl("h3", { text: "Plugins and settings (bleeding edge)" }); containerEl.createEl("h3", { text: "Plugins and settings (bleeding edge)" });
containerEl.createEl("div", {
text: "This feature is not compatible with IBM Cloudant and some large plugins (e.g., Self-hosted LiveSync) yet.",
}).addClass("op-warn");
// new Setting(containerEl) // new Setting(containerEl)
// .setName("Use Plugins and settings") // .setName("Use Plugins and settings")
@ -3366,7 +3379,6 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
// @ts-ignore // @ts-ignore
const pl = this.plugin.app.plugins; const pl = this.plugin.app.plugins;
const manifests: PluginManifest[] = Object.values(pl.manifests); const manifests: PluginManifest[] = Object.values(pl.manifests);
console.dir(manifests);
for (let m of manifests) { for (let m of manifests) {
let path = normalizePath(m.dir) + "/"; let path = normalizePath(m.dir) + "/";
const adapter = this.plugin.app.vault.adapter; const adapter = this.plugin.app.vault.adapter;
@ -3375,19 +3387,16 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
for (let file of files) { for (let file of files) {
let thePath = path + file; let thePath = path + file;
if (await adapter.exists(thePath)) { if (await adapter.exists(thePath)) {
// pluginData[file] = await arrayBufferToBase64(await adapter.readBinary(thePath));
pluginData[file] = await adapter.read(thePath); pluginData[file] = await adapter.read(thePath);
} }
} }
console.dir(m.id);
console.dir(pluginData);
let mtime = 0; let mtime = 0;
if (await adapter.exists(path + "/data.json")) { if (await adapter.exists(path + "/data.json")) {
mtime = (await adapter.stat(path + "/data.json")).mtime; mtime = (await adapter.stat(path + "/data.json")).mtime;
} }
let p: PluginDataEntry = { let p: PluginDataEntry = {
_id: `ps:${this.plugin.settings.deviceAndVaultName}-${m.id}`, _id: `ps:${this.plugin.settings.deviceAndVaultName}-${m.id}`,
dataJson: pluginData["data.json"] ? await encrypt(pluginData["data.json"], this.plugin.settings.passphrase) : undefined, dataJson: pluginData["data.json"],
deviceVaultName: this.plugin.settings.deviceAndVaultName, deviceVaultName: this.plugin.settings.deviceAndVaultName,
mainJs: pluginData["main.js"], mainJs: pluginData["main.js"],
styleCss: pluginData["style.css"], styleCss: pluginData["style.css"],
@ -3396,29 +3405,40 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
mtime: mtime, mtime: mtime,
type: "plugin", type: "plugin",
}; };
await db.put(p); let d: LoadedEntry = {
_id: p._id,
data: JSON.stringify(p),
ctime: mtime,
mtime: mtime,
size: 0,
children: [],
datatype: "plain",
};
await this.plugin.localDatabase.putDBEntry(d);
} }
await this.plugin.replicate(true); await this.plugin.replicate(true);
updatePluginPane(); updatePluginPane();
}; };
const updatePluginPane = async () => { const updatePluginPane = async () => {
const db = this.plugin.localDatabase.localDatabase; const db = this.plugin.localDatabase.localDatabase;
let oldDocs = await db.allDocs<PluginDataEntry>({ startkey: `ps:`, endkey: `ps;`, include_docs: true }); let docList = await db.allDocs<PluginDataEntry>({ startkey: `ps:`, endkey: `ps;`, include_docs: false });
let oldDocs: PluginDataEntry[] = ((await Promise.all(docList.rows.map(async (e) => await this.plugin.localDatabase.getDBEntry(e.id)))).filter((e) => e !== false) as LoadedEntry[]).map((e) => JSON.parse(e.data));
let plugins: { [key: string]: PluginDataEntry[] } = {}; let plugins: { [key: string]: PluginDataEntry[] } = {};
let allPlugins: { [key: string]: PluginDataEntry } = {}; let allPlugins: { [key: string]: PluginDataEntry } = {};
let thisDevicePlugins: { [key: string]: PluginDataEntry } = {}; let thisDevicePlugins: { [key: string]: PluginDataEntry } = {};
for (let v of oldDocs.rows) { for (let v of oldDocs) {
if (typeof plugins[v.doc.deviceVaultName] === "undefined") { if (typeof plugins[v.deviceVaultName] === "undefined") {
plugins[v.doc.deviceVaultName] = []; plugins[v.deviceVaultName] = [];
} }
plugins[v.doc.deviceVaultName].push(v.doc); plugins[v.deviceVaultName].push(v);
allPlugins[v.doc._id] = v.doc; allPlugins[v._id] = v;
if (v.doc.deviceVaultName == this.plugin.settings.deviceAndVaultName) { if (v.deviceVaultName == this.plugin.settings.deviceAndVaultName) {
thisDevicePlugins[v.doc.manifest.id] = v.doc; thisDevicePlugins[v.manifest.id] = v;
} }
} }
let html = ` let html = `
<table> <div class='sls-plugins-wrap'>
<table class='sls-plugins-tbl'>
<tr> <tr>
<th>vault</th> <th>vault</th>
<th>plugin</th> <th>plugin</th>
@ -3441,10 +3461,10 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} }
if (thisDevicePlugins[v.manifest.id] && thisDevicePlugins[v.manifest.id].dataJson && v.dataJson) { if (thisDevicePlugins[v.manifest.id] && thisDevicePlugins[v.manifest.id].dataJson && v.dataJson) {
// have this plugin. // have this plugin.
let localSetting = await decrypt(thisDevicePlugins[v.manifest.id].dataJson, this.plugin.settings.passphrase); let localSetting = thisDevicePlugins[v.manifest.id].dataJson;
try { try {
let remoteSetting = await decrypt(v.dataJson, this.plugin.settings.passphrase); let remoteSetting = v.dataJson;
if (localSetting == remoteSetting) { if (localSetting == remoteSetting) {
settingApplyable = "even"; settingApplyable = "even";
} else { } else {
@ -3473,7 +3493,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
html += piece; html += piece;
} }
} }
html += "</table>"; html += "</table></div>";
pluginConfig.innerHTML = html; pluginConfig.innerHTML = html;
pluginConfig.querySelectorAll(".apply-plugin-data").forEach((e) => pluginConfig.querySelectorAll(".apply-plugin-data").forEach((e) =>
e.addEventListener("click", async (evt) => { e.addEventListener("click", async (evt) => {
@ -3488,7 +3508,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.app.plugins.unloadPlugin(plugin.manifest.id); await this.plugin.app.plugins.unloadPlugin(plugin.manifest.id);
Logger(`Unload plugin:${plugin.manifest.id}`, LOG_LEVEL.NOTICE); Logger(`Unload plugin:${plugin.manifest.id}`, LOG_LEVEL.NOTICE);
} }
if (plugin.dataJson) await adapter.write(pluginTargetFolderPath + "data.json", await decrypt(plugin.dataJson, this.plugin.settings.passphrase)); if (plugin.dataJson) await adapter.write(pluginTargetFolderPath + "data.json", plugin.dataJson);
Logger("wrote:" + pluginTargetFolderPath + "data.json", LOG_LEVEL.NOTICE); Logger("wrote:" + pluginTargetFolderPath + "data.json", LOG_LEVEL.NOTICE);
// @ts-ignore // @ts-ignore
if (stat) { if (stat) {
@ -3521,7 +3541,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await adapter.write(pluginTargetFolderPath + "main.js", plugin.mainJs); await adapter.write(pluginTargetFolderPath + "main.js", plugin.mainJs);
await adapter.write(pluginTargetFolderPath + "manifest.json", plugin.manifestJson); await adapter.write(pluginTargetFolderPath + "manifest.json", plugin.manifestJson);
if (plugin.styleCss) await adapter.write(pluginTargetFolderPath + "styles.css", plugin.styleCss); if (plugin.styleCss) await adapter.write(pluginTargetFolderPath + "styles.css", plugin.styleCss);
if (plugin.dataJson) await adapter.write(pluginTargetFolderPath + "data.json", await decrypt(plugin.dataJson, this.plugin.settings.passphrase)); if (plugin.dataJson) await adapter.write(pluginTargetFolderPath + "data.json", plugin.dataJson);
if (stat) { if (stat) {
// @ts-ignore // @ts-ignore
await this.plugin.app.plugins.loadPlugin(plugin.manifest.id); await this.plugin.app.plugins.loadPlugin(plugin.manifest.id);
@ -3553,6 +3573,10 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setButtonText("Save plugins") .setButtonText("Save plugins")
.setDisabled(false) .setDisabled(false)
.onClick(async () => { .onClick(async () => {
if (!this.plugin.settings.encrypt) {
Logger("You have to encrypt the database to use plugin setting sync.", LOG_LEVEL.NOTICE);
return;
}
await sweepPlugin(); await sweepPlugin();
}) })
); );
@ -3570,6 +3594,17 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
xx.remove(); xx.remove();
}); });
}); });
xx.createEl("button", { text: `Restore from file` }, (e) => {
e.addEventListener("click", async () => {
let f = await this.app.vault.getFiles().filter((e) => path2id(e.path) == k);
if (f.length == 0) {
Logger("Not found in vault", LOG_LEVEL.NOTICE);
return;
}
await this.plugin.updateIntoDB(f[0]);
xx.remove();
});
});
} }
} else { } else {
let cx = containerEl.createEl("div", { text: "There's no collupted data." }); let cx = containerEl.createEl("div", { text: "There's no collupted data." });

7
styles.css
View File

@ -31,3 +31,10 @@
.tcenter { .tcenter {
text-align: center; text-align: center;
} }
.sls-plugins-wrap {
display: flex;
flex-grow: 1;
overflow: scroll;
}
.sls-plugins-tbl {
}