1
0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-08-10 22:11:45 +02:00

## 0.24.8

### Fixed
-   Some parallel-processing tasks are now performed more safely.
-   Some error messages has been fixed.
### Improved
-   Synchronisation is now more efficient and faster.
-   Saving chunks is a bit more robust.
### New Feature
-   We can remove orphaned chunks again, now!
This commit is contained in:
vorotamoroz
2025-01-22 11:55:56 +00:00
parent 0629bc04bb
commit 9b1588a65b
8 changed files with 355 additions and 72 deletions

15
package-lock.json generated
View File

@@ -18,7 +18,7 @@
"fflate": "^0.8.2",
"idb": "^8.0.0",
"minimatch": "^10.0.1",
"octagonal-wheels": "^0.1.21",
"octagonal-wheels": "^0.1.22",
"svelte-check": "^4.0.4",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
},
@@ -5104,9 +5104,10 @@
}
},
"node_modules/octagonal-wheels": {
"version": "0.1.21",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz",
"integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==",
"version": "0.1.22",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz",
"integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==",
"license": "MIT",
"dependencies": {
"idb": "^8.0.0"
}
@@ -10308,9 +10309,9 @@
}
},
"octagonal-wheels": {
"version": "0.1.21",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz",
"integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==",
"version": "0.1.22",
"resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz",
"integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==",
"requires": {
"idb": "^8.0.0"
}

View File

@@ -72,7 +72,7 @@
"fflate": "^0.8.2",
"idb": "^8.0.0",
"minimatch": "^10.0.1",
"octagonal-wheels": "^0.1.21",
"octagonal-wheels": "^0.1.22",
"svelte-check": "^4.0.4",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
}

View File

@@ -0,0 +1,265 @@
import { sizeToHumanReadable } from "octagonal-wheels/number";
import { LOG_LEVEL_NOTICE, type MetaEntry } from "../../lib/src/common/types";
import { getNoFromRev } from "../../lib/src/pouchdb/LiveSyncLocalDB";
import type { IObsidianModule } from "../../modules/AbstractObsidianModule";
import { LiveSyncCommands } from "../LiveSyncCommands";
export class LocalDatabaseMaintenance extends LiveSyncCommands implements IObsidianModule {
$everyOnload(): Promise<boolean> {
return Promise.resolve(true);
}
onunload(): void {
// NO OP.
}
onload(): void | Promise<void> {
// NO OP.
}
async allChunks(includeDeleted: boolean = false) {
const p = this._progress("", LOG_LEVEL_NOTICE);
p.log("Retrieving chunks informations..");
try {
const ret = await this.localDatabase.allChunks(includeDeleted);
return ret;
} finally {
p.done();
}
}
get database() {
return this.localDatabase.localDatabase;
}
clearHash() {
this.localDatabase.hashCaches.clear();
}
async confirm(title: string, message: string, affirmative = "Yes", negative = "No") {
return (
(await this.plugin.confirm.askSelectStringDialogue(message, [affirmative, negative], {
title,
defaultAction: affirmative,
})) === affirmative
);
}
isAvailable() {
if (!this.settings.doNotUseFixedRevisionForChunks) {
this._notice("Please enable 'Compute revisions for chunks' in settings to use Garbage Collection.");
return false;
}
if (this.settings.readChunksOnline) {
this._notice("Please disable 'Read chunks online' in settings to use Garbage Collection.");
return false;
}
return true;
}
/**
* Resurrect deleted chunks that are still used in the database.
*/
async resurrectChunks() {
if (!this.isAvailable()) return;
const { used, existing } = await this.allChunks(true);
const excessiveDeletions = [...existing]
.filter(([key, e]) => e._deleted)
.filter(([key, e]) => used.has(e._id))
.map(([key, e]) => e);
const completelyLostChunks = [] as string[];
// Data lost chunks : chunks that are deleted and data is purged.
const dataLostChunks = [...existing]
.filter(([key, e]) => e._deleted && e.data === "")
.map(([key, e]) => e)
.filter((e) => used.has(e._id));
for (const e of dataLostChunks) {
// Retrieve the data from the previous revision.
const doc = await this.database.get(e._id, { rev: e._rev, revs: true, revs_info: true, conflicts: true });
const history = doc._revs_info || [];
// Chunks are immutable. So, we can resurrect the chunk by copying the data from any of previous revisions.
let resurrected = null as null | string;
const availableRevs = history
.filter((e) => e.status == "available")
.map((e) => e.rev)
.sort((a, b) => getNoFromRev(a) - getNoFromRev(b));
for (const rev of availableRevs) {
const revDoc = await this.database.get(e._id, { rev: rev });
if (revDoc.type == "leaf" && revDoc.data !== "") {
// Found the data.
resurrected = revDoc.data;
break;
}
}
// If the data is not found, we cannot resurrect the chunk, add it to the excessiveDeletions.
if (resurrected !== null) {
excessiveDeletions.push({ ...e, data: resurrected, _deleted: false });
} else {
completelyLostChunks.push(e._id);
}
}
// Chunks to be resurrected.
const resurrectChunks = excessiveDeletions.filter((e) => e.data !== "").map((e) => ({ ...e, _deleted: false }));
if (resurrectChunks.length == 0) {
this._notice("No chunks are found to be resurrected.");
return;
}
const message = `We have following chunks that are deleted but still used in the database.
- Completely lost chunks: ${completelyLostChunks.length}
- Resurrectable chunks: ${resurrectChunks.length}
Do you want to resurrect these chunks?`;
if (await this.confirm("Resurrect Chunks", message, "Resurrect", "Cancel")) {
const result = await this.database.bulkDocs(resurrectChunks);
this.clearHash();
const resurrectedChunks = result.filter((e) => "ok" in e).map((e) => e.id);
this._notice(`Resurrected chunks: ${resurrectedChunks.length} / ${resurrectChunks.length}`);
} else {
this._notice("Resurrect operation is cancelled.");
}
}
/**
* Commit deletion of files that are marked as deleted.
* This method makes the deletion permanent, and the files will not be recovered.
* After this, chunks that are used in the deleted files become ready for compaction.
*/
async commitFileDeletion() {
if (!this.isAvailable()) return;
const p = this._progress("", LOG_LEVEL_NOTICE);
p.log("Searching for deleted files..");
const docs = await this.database.allDocs<MetaEntry>({ include_docs: true });
const deletedDocs = docs.rows.filter(
(e) => (e.doc?.type == "newnote" || e.doc?.type == "plain") && e.doc?.deleted
);
if (deletedDocs.length == 0) {
p.done("No deleted files found.");
return;
}
p.log(`Found ${deletedDocs.length} deleted files.`);
const message = `We have following files that are marked as deleted.
- Deleted files: ${deletedDocs.length}
Are you sure to delete these files permanently?
Note: **Make sure to synchronise all devices before deletion.**
> [!Note]
> This operation affects the database permanently. Deleted files will not be recovered after this operation.
> And, the chunks that are used in the deleted files will be ready for compaction.`;
const deletingDocs = deletedDocs.map((e) => ({ ...e.doc, _deleted: true }) as MetaEntry);
if (await this.confirm("Delete Files", message, "Delete", "Cancel")) {
const result = await this.database.bulkDocs(deletingDocs);
this.clearHash();
p.done(`Deleted ${result.filter((e) => "ok" in e).length} / ${deletedDocs.length} files.`);
} else {
p.done("Deletion operation is cancelled.");
}
}
/**
* Commit deletion of chunks that are not used in the database.
* This method makes the deletion permanent, and the chunks will not be recovered if the database run compaction.
* After this, the database can shrink the database size by compaction.
* It is recommended to compact the database after this operation (History should be kept once before compaction).
*/
async commitChunkDeletion() {
if (!this.isAvailable()) return;
const { existing } = await this.allChunks(true);
const deletedChunks = [...existing].filter(([key, e]) => e._deleted && e.data !== "").map(([key, e]) => e);
const deletedNotVacantChunks = deletedChunks.map((e) => ({ ...e, data: "", _deleted: true }));
const size = deletedChunks.reduce((acc, e) => acc + e.data.length, 0);
const humanSize = sizeToHumanReadable(size);
const message = `We have following chunks that are marked as deleted.
- Deleted chunks: ${deletedNotVacantChunks.length} (${humanSize})
Are you sure to delete these chunks permanently?
Note: **Make sure to synchronise all devices before deletion.**
> [!Note]
> This operation finally reduces the capacity of the remote.`;
if (deletedNotVacantChunks.length == 0) {
this._notice("No deleted chunks found.");
return;
}
if (await this.confirm("Delete Chunks", message, "Delete", "Cancel")) {
const result = await this.database.bulkDocs(deletedNotVacantChunks);
this.clearHash();
this._notice(
`Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deletedNotVacantChunks.length}`
);
} else {
this._notice("Deletion operation is cancelled.");
}
}
/**
* Compact the database.
* This method removes all deleted chunks that are not used in the database.
* Make sure all devices are synchronized before running this method.
*/
async markUnusedChunks() {
if (!this.isAvailable()) return;
const { used, existing } = await this.allChunks();
const existChunks = [...existing];
const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e);
const deleteChunks = unusedChunks.map((e) => ({
...e,
_deleted: true,
}));
const size = deleteChunks.reduce((acc, e) => acc + e.data.length, 0);
const humanSize = sizeToHumanReadable(size);
if (deleteChunks.length == 0) {
this._notice("No unused chunks found.");
return;
}
const message = `We have following chunks that are not used from any files.
- Chunks: ${deleteChunks.length} (${humanSize})
Are you sure to mark these chunks to be deleted?
Note: **Make sure to synchronise all devices before deletion.**
> [!Note]
> This operation will not reduces the capacity of the remote until permanent deletion.`;
if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) {
const result = await this.database.bulkDocs(deleteChunks);
this.clearHash();
this._notice(`Marked chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`);
}
}
async removeUnusedChunks() {
const { used, existing } = await this.allChunks();
const existChunks = [...existing];
const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e);
const deleteChunks = unusedChunks.map((e) => ({
...e,
data: "",
_deleted: true,
}));
const size = unusedChunks.reduce((acc, e) => acc + e.data.length, 0);
const humanSize = sizeToHumanReadable(size);
if (deleteChunks.length == 0) {
this._notice("No unused chunks found.");
return;
}
const message = `We have following chunks that are not used from any files.
- Chunks: ${deleteChunks.length} (${humanSize})
Are you sure to delete these chunks?
Note: **Make sure to synchronise all devices before deletion.**
> [!Note]
> Chunks referenced from deleted files are not deleted. Please run "Commit File Deletion" before this operation.`;
if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) {
const result = await this.database.bulkDocs(deleteChunks);
this._notice(`Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`);
this.clearHash();
}
}
}

Submodule src/lib updated: 89e825ef3b...6305e8952b

View File

@@ -81,6 +81,7 @@ import { ModuleRebuilder } from "./modules/core/ModuleRebuilder.ts";
import { ModuleReplicateTest } from "./modules/extras/ModuleReplicateTest.ts";
import { ModuleLiveSyncMain } from "./modules/main/ModuleLiveSyncMain.ts";
import { ModuleExtraSyncObsidian } from "./modules/extraFeaturesObsidian/ModuleExtraSyncObsidian.ts";
import { LocalDatabaseMaintenance } from "./features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts";
function throwShouldBeOverridden(): never {
throw new Error("This function should be overridden by the module.");
@@ -117,7 +118,7 @@ export default class ObsidianLiveSyncPlugin
}
// Keep order to display the dialogue in order.
addOns = [new ConfigSync(this), new HiddenFileSync(this)] as LiveSyncCommands[];
addOns = [new ConfigSync(this), new HiddenFileSync(this), new LocalDatabaseMaintenance(this)] as LiveSyncCommands[];
modules = [
new ModuleLiveSyncMain(this),

View File

@@ -211,7 +211,7 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
? await this.db.fetchEntryMeta(entryInfo, undefined, true)
: await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
if (!docEntry) {
this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE);
this._log(`File ${file?.path} is not exist on the database`, LOG_LEVEL_VERBOSE);
return false;
}
const path = getPath(docEntry);

View File

@@ -231,11 +231,6 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
const { file, doc } = e;
if (!this.core.$$isFileSizeExceeded(file.stat.size) && !this.core.$$isFileSizeExceeded(doc.size)) {
await this.syncFileBetweenDBandStorage(file, doc);
// fireAndForget(() => this.checkAndApplySettingFromMarkdown(getPath(doc), true));
// eventHub.emitEvent("event-file-changed", {
// file: getPath(doc),
// automated: true,
// });
} else {
this._log(
`SYNC DATABASE AND STORAGE: ${getPath(doc)} has been skipped due to file size exceeding the limit`,

View File

@@ -77,6 +77,7 @@ import { JournalSyncMinio } from "../../../lib/src/replication/journal/objectsto
import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts";
import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
import { LocalDatabaseMaintenance } from "../../../features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts";
export type OnUpdateResult = {
visibility?: boolean;
@@ -118,10 +119,10 @@ function getLevelStr(level: ConfigLevel) {
return level == LEVEL_POWER_USER
? " (Power User)"
: level == LEVEL_ADVANCED
? " (Advanced)"
: level == LEVEL_EDGE_CASE
? " (Edge Case)"
: "";
? " (Advanced)"
: level == LEVEL_EDGE_CASE
? " (Edge Case)"
: "";
}
export function findAttrFromParent(el: HTMLElement, attr: string): string {
@@ -300,8 +301,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
const syncMode = this.editingSettings?.liveSync
? "LIVESYNC"
: this.editingSettings?.periodicReplication
? "PERIODIC"
: "ONEVENTS";
? "PERIODIC"
: "ONEVENTS";
return {
syncMode,
};
@@ -1651,16 +1652,16 @@ I appreciate you for your great dedication.
const options: Record<string, string> =
this.editingSettings.remoteType == REMOTE_COUCHDB
? {
NONE: "",
LIVESYNC: "LiveSync",
PERIODIC: "Periodic w/ batch",
DISABLE: "Disable all automatic",
}
NONE: "",
LIVESYNC: "LiveSync",
PERIODIC: "Periodic w/ batch",
DISABLE: "Disable all automatic",
}
: {
NONE: "",
PERIODIC: "Periodic w/ batch",
DISABLE: "Disable all automatic",
};
NONE: "",
PERIODIC: "Periodic w/ batch",
DISABLE: "Disable all automatic",
};
new Setting(paneEl)
.autoWireDropDown("preset", {
@@ -1767,10 +1768,10 @@ I appreciate you for your great dedication.
const optionsSyncMode =
this.editingSettings.remoteType == REMOTE_COUCHDB
? {
ONEVENTS: "On events",
PERIODIC: "Periodic and on events",
LIVESYNC: "LiveSync",
}
ONEVENTS: "On events",
PERIODIC: "Periodic and on events",
LIVESYNC: "LiveSync",
}
: { ONEVENTS: "On events", PERIODIC: "Periodic and on events" };
new Setting(paneEl)
@@ -2049,7 +2050,7 @@ I appreciate you for your great dedication.
text: "Please set device name to identify this device. This name should be unique among your devices. While not configured, we cannot enable this feature.",
cls: "op-warn",
},
(c) => {},
(c) => { },
visibleOnly(() => this.isConfiguredAs("deviceAndVaultName", ""))
);
this.createEl(
@@ -2059,7 +2060,7 @@ I appreciate you for your great dedication.
text: "We cannot change the device name while this feature is enabled. Please disable this feature to change the device name.",
cls: "op-warn-info",
},
(c) => {},
(c) => { },
visibleOnly(() => this.isConfiguredAs("usePluginSync", true))
);
@@ -2151,8 +2152,8 @@ I appreciate you for your great dedication.
const scheme = pluginConfig.couchDB_URI.startsWith("http:")
? "(HTTP)"
: pluginConfig.couchDB_URI.startsWith("https:")
? "(HTTPS)"
: "";
? "(HTTPS)"
: "";
pluginConfig.couchDB_URI = isCloudantURI(pluginConfig.couchDB_URI)
? "cloudant"
: `self-hosted${scheme}`;
@@ -2172,8 +2173,8 @@ I appreciate you for your great dedication.
const endpointScheme = pluginConfig.endpoint.startsWith("http:")
? "(HTTP)"
: pluginConfig.endpoint.startsWith("https:")
? "(HTTPS)"
: "";
? "(HTTPS)"
: "";
pluginConfig.endpoint = `${endpoint.indexOf(".r2.cloudflarestorage.") !== -1 ? "R2" : "self-hosted?"}(${endpointScheme})`;
}
const obsidianInfo = `Navigator: ${navigator.userAgent}
@@ -2384,10 +2385,10 @@ ${stringifyYaml(pluginConfig)}`;
Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
const files = this.plugin.settings.syncInternalFiles
? await this.plugin.storageAccess.getFilesIncludeHidden(
"/",
undefined,
ignorePatterns
)
"/",
undefined,
ignorePatterns
)
: await this.plugin.storageAccess.getFileNames();
const documents = [] as FilePath[];
@@ -2880,6 +2881,7 @@ ${stringifyYaml(pluginConfig)}`;
})
)
.addOnUpdate(onlyOnCouchDB);
new Setting(paneEl)
.setName("Reset journal received history")
.setDesc(
@@ -2923,7 +2925,53 @@ ${stringifyYaml(pluginConfig)}`;
)
.addOnUpdate(onlyOnMinIO);
});
void addPanel(paneEl, "Garbage Collection (Beta)", (e) => e, onlyOnCouchDB).then((paneEl) => {
new Setting(paneEl)
.setName("Remove all orphaned chunks")
.setDesc("Remove all orphaned chunks from the local database.")
.addButton((button) =>
button
.setButtonText("Remove")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin
.getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
?.removeUnusedChunks();
})
);
new Setting(paneEl)
.setName("Resurrect deleted chunks")
.setDesc(
"If you have deleted chunks before fully synchronised and missed some chunks, you possibly can resurrect them."
)
.addButton((button) =>
button
.setButtonText("Try resurrect")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin
.getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
?.resurrectChunks();
})
);
new Setting(paneEl)
.setName("Commit File Deletion")
.setDesc("Completely delete all deleted documents from the local database.")
.addButton((button) =>
button
.setButtonText("Delete")
.setWarning()
.setDisabled(false)
.onClick(async () => {
await this.plugin
.getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
?.commitFileDeletion();
})
);
});
void addPanel(paneEl, "Rebuilding Operations (Local)").then((paneEl) => {
new Setting(paneEl)
.setName("Fetch from remote")
@@ -3076,33 +3124,6 @@ ${stringifyYaml(pluginConfig)}`;
.addOnUpdate(onlyOnMinIO);
});
void addPanel(paneEl, "Deprecated").then((paneEl) => {
new Setting(paneEl)
.setClass("sls-setting-obsolete")
.setName("Run database cleanup")
.setDesc(
"Attempt to shrink the database by deleting unused chunks. This may not work consistently. Use the 'Rebuild everything' under Total Overhaul."
)
.addButton((button) =>
button
.setButtonText("DryRun")
.setDisabled(false)
.onClick(async () => {
await this.dryRunGC();
})
)
.addButton((button) =>
button
.setButtonText("Perform cleaning")
.setDisabled(false)
.setWarning()
.onClick(async () => {
this.closeSetting();
await this.dbGC();
})
)
.addOnUpdate(onlyOnCouchDB);
});
void addPanel(paneEl, "Reset").then((paneEl) => {
new Setting(paneEl)
.setName("Delete local database to reset or uninstall Self-hosted LiveSync")