diff --git a/package-lock.json b/package-lock.json index 0944a78..32593ea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "fflate": "^0.8.2", "idb": "^8.0.0", "minimatch": "^10.0.1", - "octagonal-wheels": "^0.1.21", + "octagonal-wheels": "^0.1.22", "svelte-check": "^4.0.4", "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2" }, @@ -5104,9 +5104,10 @@ } }, "node_modules/octagonal-wheels": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz", - "integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==", + "version": "0.1.22", + "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz", + "integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==", + "license": "MIT", "dependencies": { "idb": "^8.0.0" } @@ -10308,9 +10309,9 @@ } }, "octagonal-wheels": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz", - "integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==", + "version": "0.1.22", + "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz", + "integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==", "requires": { "idb": "^8.0.0" } diff --git a/package.json b/package.json index eafbe38..c11bd80 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "fflate": "^0.8.2", "idb": "^8.0.0", "minimatch": "^10.0.1", - "octagonal-wheels": "^0.1.21", + "octagonal-wheels": "^0.1.22", "svelte-check": "^4.0.4", "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2" } diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts new file mode 100644 index 0000000..2ac4e1c --- /dev/null +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts @@ -0,0 +1,265 @@ +import { sizeToHumanReadable } from "octagonal-wheels/number"; +import { LOG_LEVEL_NOTICE, type MetaEntry } from "../../lib/src/common/types"; +import { getNoFromRev } from "../../lib/src/pouchdb/LiveSyncLocalDB"; +import type { IObsidianModule } from "../../modules/AbstractObsidianModule"; +import { LiveSyncCommands } from "../LiveSyncCommands"; + +export class LocalDatabaseMaintenance extends LiveSyncCommands implements IObsidianModule { + $everyOnload(): Promise { + return Promise.resolve(true); + } + onunload(): void { + // NO OP. + } + onload(): void | Promise { + // NO OP. + } + async allChunks(includeDeleted: boolean = false) { + const p = this._progress("", LOG_LEVEL_NOTICE); + p.log("Retrieving chunks informations.."); + try { + const ret = await this.localDatabase.allChunks(includeDeleted); + return ret; + } finally { + p.done(); + } + } + get database() { + return this.localDatabase.localDatabase; + } + clearHash() { + this.localDatabase.hashCaches.clear(); + } + + async confirm(title: string, message: string, affirmative = "Yes", negative = "No") { + return ( + (await this.plugin.confirm.askSelectStringDialogue(message, [affirmative, negative], { + title, + defaultAction: affirmative, + })) === affirmative + ); + } + isAvailable() { + if (!this.settings.doNotUseFixedRevisionForChunks) { + this._notice("Please enable 'Compute revisions for chunks' in settings to use Garbage Collection."); + return false; + } + if (this.settings.readChunksOnline) { + this._notice("Please disable 'Read chunks online' in settings to use Garbage Collection."); + return false; + } + return true; + } + /** + * Resurrect deleted chunks that are still used in the database. + */ + async resurrectChunks() { + if (!this.isAvailable()) return; + const { used, existing } = await this.allChunks(true); + const excessiveDeletions = [...existing] + .filter(([key, e]) => e._deleted) + .filter(([key, e]) => used.has(e._id)) + .map(([key, e]) => e); + const completelyLostChunks = [] as string[]; + // Data lost chunks : chunks that are deleted and data is purged. + const dataLostChunks = [...existing] + .filter(([key, e]) => e._deleted && e.data === "") + .map(([key, e]) => e) + .filter((e) => used.has(e._id)); + for (const e of dataLostChunks) { + // Retrieve the data from the previous revision. + const doc = await this.database.get(e._id, { rev: e._rev, revs: true, revs_info: true, conflicts: true }); + const history = doc._revs_info || []; + // Chunks are immutable. So, we can resurrect the chunk by copying the data from any of previous revisions. + let resurrected = null as null | string; + const availableRevs = history + .filter((e) => e.status == "available") + .map((e) => e.rev) + .sort((a, b) => getNoFromRev(a) - getNoFromRev(b)); + for (const rev of availableRevs) { + const revDoc = await this.database.get(e._id, { rev: rev }); + if (revDoc.type == "leaf" && revDoc.data !== "") { + // Found the data. + resurrected = revDoc.data; + break; + } + } + // If the data is not found, we cannot resurrect the chunk, add it to the excessiveDeletions. + if (resurrected !== null) { + excessiveDeletions.push({ ...e, data: resurrected, _deleted: false }); + } else { + completelyLostChunks.push(e._id); + } + } + // Chunks to be resurrected. + const resurrectChunks = excessiveDeletions.filter((e) => e.data !== "").map((e) => ({ ...e, _deleted: false })); + + if (resurrectChunks.length == 0) { + this._notice("No chunks are found to be resurrected."); + return; + } + const message = `We have following chunks that are deleted but still used in the database. + +- Completely lost chunks: ${completelyLostChunks.length} +- Resurrectable chunks: ${resurrectChunks.length} + +Do you want to resurrect these chunks?`; + if (await this.confirm("Resurrect Chunks", message, "Resurrect", "Cancel")) { + const result = await this.database.bulkDocs(resurrectChunks); + this.clearHash(); + const resurrectedChunks = result.filter((e) => "ok" in e).map((e) => e.id); + this._notice(`Resurrected chunks: ${resurrectedChunks.length} / ${resurrectChunks.length}`); + } else { + this._notice("Resurrect operation is cancelled."); + } + } + /** + * Commit deletion of files that are marked as deleted. + * This method makes the deletion permanent, and the files will not be recovered. + * After this, chunks that are used in the deleted files become ready for compaction. + */ + async commitFileDeletion() { + if (!this.isAvailable()) return; + const p = this._progress("", LOG_LEVEL_NOTICE); + p.log("Searching for deleted files.."); + const docs = await this.database.allDocs({ include_docs: true }); + const deletedDocs = docs.rows.filter( + (e) => (e.doc?.type == "newnote" || e.doc?.type == "plain") && e.doc?.deleted + ); + if (deletedDocs.length == 0) { + p.done("No deleted files found."); + return; + } + p.log(`Found ${deletedDocs.length} deleted files.`); + + const message = `We have following files that are marked as deleted. + +- Deleted files: ${deletedDocs.length} + +Are you sure to delete these files permanently? + +Note: **Make sure to synchronise all devices before deletion.** + +> [!Note] +> This operation affects the database permanently. Deleted files will not be recovered after this operation. +> And, the chunks that are used in the deleted files will be ready for compaction.`; + + const deletingDocs = deletedDocs.map((e) => ({ ...e.doc, _deleted: true }) as MetaEntry); + + if (await this.confirm("Delete Files", message, "Delete", "Cancel")) { + const result = await this.database.bulkDocs(deletingDocs); + this.clearHash(); + p.done(`Deleted ${result.filter((e) => "ok" in e).length} / ${deletedDocs.length} files.`); + } else { + p.done("Deletion operation is cancelled."); + } + } + /** + * Commit deletion of chunks that are not used in the database. + * This method makes the deletion permanent, and the chunks will not be recovered if the database run compaction. + * After this, the database can shrink the database size by compaction. + * It is recommended to compact the database after this operation (History should be kept once before compaction). + */ + async commitChunkDeletion() { + if (!this.isAvailable()) return; + const { existing } = await this.allChunks(true); + const deletedChunks = [...existing].filter(([key, e]) => e._deleted && e.data !== "").map(([key, e]) => e); + const deletedNotVacantChunks = deletedChunks.map((e) => ({ ...e, data: "", _deleted: true })); + const size = deletedChunks.reduce((acc, e) => acc + e.data.length, 0); + const humanSize = sizeToHumanReadable(size); + const message = `We have following chunks that are marked as deleted. + +- Deleted chunks: ${deletedNotVacantChunks.length} (${humanSize}) + +Are you sure to delete these chunks permanently? + +Note: **Make sure to synchronise all devices before deletion.** + +> [!Note] +> This operation finally reduces the capacity of the remote.`; + + if (deletedNotVacantChunks.length == 0) { + this._notice("No deleted chunks found."); + return; + } + if (await this.confirm("Delete Chunks", message, "Delete", "Cancel")) { + const result = await this.database.bulkDocs(deletedNotVacantChunks); + this.clearHash(); + this._notice( + `Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deletedNotVacantChunks.length}` + ); + } else { + this._notice("Deletion operation is cancelled."); + } + } + /** + * Compact the database. + * This method removes all deleted chunks that are not used in the database. + * Make sure all devices are synchronized before running this method. + */ + async markUnusedChunks() { + if (!this.isAvailable()) return; + const { used, existing } = await this.allChunks(); + const existChunks = [...existing]; + const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e); + const deleteChunks = unusedChunks.map((e) => ({ + ...e, + _deleted: true, + })); + const size = deleteChunks.reduce((acc, e) => acc + e.data.length, 0); + const humanSize = sizeToHumanReadable(size); + if (deleteChunks.length == 0) { + this._notice("No unused chunks found."); + return; + } + const message = `We have following chunks that are not used from any files. + +- Chunks: ${deleteChunks.length} (${humanSize}) + +Are you sure to mark these chunks to be deleted? + +Note: **Make sure to synchronise all devices before deletion.** + +> [!Note] +> This operation will not reduces the capacity of the remote until permanent deletion.`; + + if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) { + const result = await this.database.bulkDocs(deleteChunks); + this.clearHash(); + this._notice(`Marked chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`); + } + } + + async removeUnusedChunks() { + const { used, existing } = await this.allChunks(); + const existChunks = [...existing]; + const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e); + const deleteChunks = unusedChunks.map((e) => ({ + ...e, + data: "", + _deleted: true, + })); + const size = unusedChunks.reduce((acc, e) => acc + e.data.length, 0); + const humanSize = sizeToHumanReadable(size); + if (deleteChunks.length == 0) { + this._notice("No unused chunks found."); + return; + } + const message = `We have following chunks that are not used from any files. + +- Chunks: ${deleteChunks.length} (${humanSize}) + +Are you sure to delete these chunks? + +Note: **Make sure to synchronise all devices before deletion.** + +> [!Note] +> Chunks referenced from deleted files are not deleted. Please run "Commit File Deletion" before this operation.`; + + if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) { + const result = await this.database.bulkDocs(deleteChunks); + this._notice(`Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`); + this.clearHash(); + } + } +} diff --git a/src/lib b/src/lib index 89e825e..6305e89 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 89e825ef3b26e7299f9ba74b1ffeed67e4103db7 +Subproject commit 6305e8952b7336fabdeec376b22fbae5a66e8381 diff --git a/src/main.ts b/src/main.ts index e3b934c..246b135 100644 --- a/src/main.ts +++ b/src/main.ts @@ -81,6 +81,7 @@ import { ModuleRebuilder } from "./modules/core/ModuleRebuilder.ts"; import { ModuleReplicateTest } from "./modules/extras/ModuleReplicateTest.ts"; import { ModuleLiveSyncMain } from "./modules/main/ModuleLiveSyncMain.ts"; import { ModuleExtraSyncObsidian } from "./modules/extraFeaturesObsidian/ModuleExtraSyncObsidian.ts"; +import { LocalDatabaseMaintenance } from "./features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts"; function throwShouldBeOverridden(): never { throw new Error("This function should be overridden by the module."); @@ -117,7 +118,7 @@ export default class ObsidianLiveSyncPlugin } // Keep order to display the dialogue in order. - addOns = [new ConfigSync(this), new HiddenFileSync(this)] as LiveSyncCommands[]; + addOns = [new ConfigSync(this), new HiddenFileSync(this), new LocalDatabaseMaintenance(this)] as LiveSyncCommands[]; modules = [ new ModuleLiveSyncMain(this), diff --git a/src/modules/core/ModuleFileHandler.ts b/src/modules/core/ModuleFileHandler.ts index bf190fa..15d2572 100644 --- a/src/modules/core/ModuleFileHandler.ts +++ b/src/modules/core/ModuleFileHandler.ts @@ -211,7 +211,7 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule { ? await this.db.fetchEntryMeta(entryInfo, undefined, true) : await this.db.fetchEntryMeta(entryInfo.path, undefined, true); if (!docEntry) { - this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE); + this._log(`File ${file?.path} is not exist on the database`, LOG_LEVEL_VERBOSE); return false; } const path = getPath(docEntry); diff --git a/src/modules/essential/ModuleInitializerFile.ts b/src/modules/essential/ModuleInitializerFile.ts index 5180b62..33baa98 100644 --- a/src/modules/essential/ModuleInitializerFile.ts +++ b/src/modules/essential/ModuleInitializerFile.ts @@ -231,11 +231,6 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule const { file, doc } = e; if (!this.core.$$isFileSizeExceeded(file.stat.size) && !this.core.$$isFileSizeExceeded(doc.size)) { await this.syncFileBetweenDBandStorage(file, doc); - // fireAndForget(() => this.checkAndApplySettingFromMarkdown(getPath(doc), true)); - // eventHub.emitEvent("event-file-changed", { - // file: getPath(doc), - // automated: true, - // }); } else { this._log( `SYNC DATABASE AND STORAGE: ${getPath(doc)} has been skipped due to file size exceeding the limit`, diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 62cc79d..004b6c7 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -77,6 +77,7 @@ import { JournalSyncMinio } from "../../../lib/src/replication/journal/objectsto import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts"; import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts"; import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts"; +import { LocalDatabaseMaintenance } from "../../../features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts"; export type OnUpdateResult = { visibility?: boolean; @@ -118,10 +119,10 @@ function getLevelStr(level: ConfigLevel) { return level == LEVEL_POWER_USER ? " (Power User)" : level == LEVEL_ADVANCED - ? " (Advanced)" - : level == LEVEL_EDGE_CASE - ? " (Edge Case)" - : ""; + ? " (Advanced)" + : level == LEVEL_EDGE_CASE + ? " (Edge Case)" + : ""; } export function findAttrFromParent(el: HTMLElement, attr: string): string { @@ -300,8 +301,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { const syncMode = this.editingSettings?.liveSync ? "LIVESYNC" : this.editingSettings?.periodicReplication - ? "PERIODIC" - : "ONEVENTS"; + ? "PERIODIC" + : "ONEVENTS"; return { syncMode, }; @@ -1651,16 +1652,16 @@ I appreciate you for your great dedication. const options: Record = this.editingSettings.remoteType == REMOTE_COUCHDB ? { - NONE: "", - LIVESYNC: "LiveSync", - PERIODIC: "Periodic w/ batch", - DISABLE: "Disable all automatic", - } + NONE: "", + LIVESYNC: "LiveSync", + PERIODIC: "Periodic w/ batch", + DISABLE: "Disable all automatic", + } : { - NONE: "", - PERIODIC: "Periodic w/ batch", - DISABLE: "Disable all automatic", - }; + NONE: "", + PERIODIC: "Periodic w/ batch", + DISABLE: "Disable all automatic", + }; new Setting(paneEl) .autoWireDropDown("preset", { @@ -1767,10 +1768,10 @@ I appreciate you for your great dedication. const optionsSyncMode = this.editingSettings.remoteType == REMOTE_COUCHDB ? { - ONEVENTS: "On events", - PERIODIC: "Periodic and on events", - LIVESYNC: "LiveSync", - } + ONEVENTS: "On events", + PERIODIC: "Periodic and on events", + LIVESYNC: "LiveSync", + } : { ONEVENTS: "On events", PERIODIC: "Periodic and on events" }; new Setting(paneEl) @@ -2049,7 +2050,7 @@ I appreciate you for your great dedication. text: "Please set device name to identify this device. This name should be unique among your devices. While not configured, we cannot enable this feature.", cls: "op-warn", }, - (c) => {}, + (c) => { }, visibleOnly(() => this.isConfiguredAs("deviceAndVaultName", "")) ); this.createEl( @@ -2059,7 +2060,7 @@ I appreciate you for your great dedication. text: "We cannot change the device name while this feature is enabled. Please disable this feature to change the device name.", cls: "op-warn-info", }, - (c) => {}, + (c) => { }, visibleOnly(() => this.isConfiguredAs("usePluginSync", true)) ); @@ -2151,8 +2152,8 @@ I appreciate you for your great dedication. const scheme = pluginConfig.couchDB_URI.startsWith("http:") ? "(HTTP)" : pluginConfig.couchDB_URI.startsWith("https:") - ? "(HTTPS)" - : ""; + ? "(HTTPS)" + : ""; pluginConfig.couchDB_URI = isCloudantURI(pluginConfig.couchDB_URI) ? "cloudant" : `self-hosted${scheme}`; @@ -2172,8 +2173,8 @@ I appreciate you for your great dedication. const endpointScheme = pluginConfig.endpoint.startsWith("http:") ? "(HTTP)" : pluginConfig.endpoint.startsWith("https:") - ? "(HTTPS)" - : ""; + ? "(HTTPS)" + : ""; pluginConfig.endpoint = `${endpoint.indexOf(".r2.cloudflarestorage.") !== -1 ? "R2" : "self-hosted?"}(${endpointScheme})`; } const obsidianInfo = `Navigator: ${navigator.userAgent} @@ -2384,10 +2385,10 @@ ${stringifyYaml(pluginConfig)}`; Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify"); const files = this.plugin.settings.syncInternalFiles ? await this.plugin.storageAccess.getFilesIncludeHidden( - "/", - undefined, - ignorePatterns - ) + "/", + undefined, + ignorePatterns + ) : await this.plugin.storageAccess.getFileNames(); const documents = [] as FilePath[]; @@ -2880,6 +2881,7 @@ ${stringifyYaml(pluginConfig)}`; }) ) .addOnUpdate(onlyOnCouchDB); + new Setting(paneEl) .setName("Reset journal received history") .setDesc( @@ -2923,7 +2925,53 @@ ${stringifyYaml(pluginConfig)}`; ) .addOnUpdate(onlyOnMinIO); }); + void addPanel(paneEl, "Garbage Collection (Beta)", (e) => e, onlyOnCouchDB).then((paneEl) => { + new Setting(paneEl) + .setName("Remove all orphaned chunks") + .setDesc("Remove all orphaned chunks from the local database.") + .addButton((button) => + button + .setButtonText("Remove") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.plugin + .getAddOn(LocalDatabaseMaintenance.name) + ?.removeUnusedChunks(); + }) + ); + new Setting(paneEl) + .setName("Resurrect deleted chunks") + .setDesc( + "If you have deleted chunks before fully synchronised and missed some chunks, you possibly can resurrect them." + ) + .addButton((button) => + button + .setButtonText("Try resurrect") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.plugin + .getAddOn(LocalDatabaseMaintenance.name) + ?.resurrectChunks(); + }) + ); + new Setting(paneEl) + .setName("Commit File Deletion") + .setDesc("Completely delete all deleted documents from the local database.") + .addButton((button) => + button + .setButtonText("Delete") + .setWarning() + .setDisabled(false) + .onClick(async () => { + await this.plugin + .getAddOn(LocalDatabaseMaintenance.name) + ?.commitFileDeletion(); + }) + ); + }); void addPanel(paneEl, "Rebuilding Operations (Local)").then((paneEl) => { new Setting(paneEl) .setName("Fetch from remote") @@ -3076,33 +3124,6 @@ ${stringifyYaml(pluginConfig)}`; .addOnUpdate(onlyOnMinIO); }); - void addPanel(paneEl, "Deprecated").then((paneEl) => { - new Setting(paneEl) - .setClass("sls-setting-obsolete") - .setName("Run database cleanup") - .setDesc( - "Attempt to shrink the database by deleting unused chunks. This may not work consistently. Use the 'Rebuild everything' under Total Overhaul." - ) - .addButton((button) => - button - .setButtonText("DryRun") - .setDisabled(false) - .onClick(async () => { - await this.dryRunGC(); - }) - ) - .addButton((button) => - button - .setButtonText("Perform cleaning") - .setDisabled(false) - .setWarning() - .onClick(async () => { - this.closeSetting(); - await this.dbGC(); - }) - ) - .addOnUpdate(onlyOnCouchDB); - }); void addPanel(paneEl, "Reset").then((paneEl) => { new Setting(paneEl) .setName("Delete local database to reset or uninstall Self-hosted LiveSync")