self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-08-10 22:11:45 +02:00
Code Issues Releases Activity

### Fixed

Browse Source 
- Storage scanning no longer occurs when `Suspend file watching` is enabled (including boot-sequence).

### Improved
- Saving notes and files now consumes less memory.
- Chunk caching is now more efficient.
- Both of them (may) are effective for #692, #680, and some more.

### Changed
- `Incubate Chunks in Document` (also known as `Eden`) is now fully sunset.
- The `Compute revisions for chunks` setting has also been removed.
- As mentioned, `Memory cache size (by total characters)` has been removed.

### Refactored
- A significant refactoring of the core codebase is underway (please refer the release-note).
This commit is contained in:
vorotamoroz
2025-08-09 01:45:41 +09:00
parent 1073ee9e30
commit f996e056af
16 changed files with 760 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

836
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
package.json
View File

@@ -23,7 +23,8 @@
"pretty": "npm run prettyNoWrite -- --write --log-level error",
"prettyCheck": "npm run prettyNoWrite -- --check",
"prettyNoWrite": "prettier --config ./.prettierrc \"**/*.js\" \"**/*.ts\" \"**/*.json\" ",
"check": "npm run lint && npm run svelte-check && npm run tsc-check"
"check": "npm run lint && npm run svelte-check && npm run tsc-check",
"unittest": "deno test -A --no-check --coverage=cov_profile --v8-flags=--expose-gc --trace-leaks ./src/"
},
"keywords": [],
"author": "vorotamoroz",
@@ -60,6 +61,7 @@
"pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-idb": "^9.0.0",
"pouchdb-adapter-indexeddb": "^9.0.0",
"pouchdb-adapter-memory": "^9.0.0",
"pouchdb-core": "^9.0.0",
"pouchdb-errors": "^9.0.0",
"pouchdb-find": "^9.0.0",
@@ -75,7 +77,8 @@
"tslib": "^2.8.1",
"tsx": "^4.19.4",
"typescript": "5.7.3",
"yaml": "^2.8.0"
"yaml": "^2.8.0",
"@types/deno": "^2.3.0"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.808.0",

10
src/common/KeyValueDB.ts
View File

@@ -1,13 +1,5 @@
import { deleteDB, type IDBPDatabase, openDB } from "idb";
export interface KeyValueDatabase {
get<T>(key: IDBValidKey): Promise<T>;
set<T>(key: IDBValidKey, value: T): Promise<IDBValidKey>;
del(key: IDBValidKey): Promise<void>;
clear(): Promise<void>;
keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>;
close(): void;
destroy(): Promise<void>;
}
import type { KeyValueDatabase } from "../lib/src/interfaces/KeyValueDatabase.ts";
const databaseCache: { [key: string]: IDBPDatabase<any> } = {};
export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatabase> => {
if (dbKey in databaseCache) {

3
src/common/utils.ts
View File

@@ -28,11 +28,12 @@ import type ObsidianLiveSyncPlugin from "../main.ts";
import { writeString } from "../lib/src/string_and_binary/convert.ts";
import { fireAndForget } from "../lib/src/common/utils.ts";
import { sameChangePairs } from "./stores.ts";
import type { KeyValueDatabase } from "./KeyValueDB.ts";
import { scheduleTask } from "octagonal-wheels/concurrency/task";
import { EVENT_PLUGIN_UNLOADED, eventHub } from "./events.ts";
import { promiseWithResolver, type PromiseWithResolvers } from "octagonal-wheels/promises";
import { AuthorizationHeaderGenerator } from "../lib/src/replication/httplib.ts";
import type { KeyValueDatabase } from "../lib/src/interfaces/KeyValueDatabase.ts";
export { scheduleTask, cancelTask, cancelAllTasks } from "../lib/src/concurrency/task.ts";

2
src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts
View File

@@ -28,7 +28,7 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands implements IObsid
return this.localDatabase.localDatabase;
}
clearHash() {
this.localDatabase.hashCaches.clear();
this.localDatabase.clearCaches();
}
async confirm(title: string, message: string, affirmative = "Yes", negative = "No") {

2
src/lib

Submodule src/lib updated: a5ac735c6f...7d1597edcf

10
src/main.ts
View File

@@ -27,7 +27,7 @@ import {
LiveSyncAbstractReplicator,
type LiveSyncReplicatorEnv,
} from "./lib/src/replication/LiveSyncAbstractReplicator.js";
import { type KeyValueDatabase } from "./common/KeyValueDB.ts";
import { type KeyValueDatabase } from "./lib/src/interfaces/KeyValueDatabase.ts";
import { LiveSyncCommands } from "./features/LiveSyncCommands.ts";
import { HiddenFileSync } from "./features/HiddenFileSync/CmdHiddenFileSync.ts";
import { ConfigSync } from "./features/ConfigSync/CmdConfigSync.ts";
@@ -591,7 +591,11 @@ export default class ObsidianLiveSyncPlugin
throwShouldBeOverridden();
}
$$initializeDatabase(showingNotice: boolean = false, reopenDatabase = true): Promise<boolean> {
$$initializeDatabase(
showingNotice: boolean = false,
reopenDatabase = true,
ignoreSuspending: boolean = false
): Promise<boolean> {
throwShouldBeOverridden();
}
@@ -628,7 +632,7 @@ export default class ObsidianLiveSyncPlugin
throwShouldBeOverridden();
}
$$performFullScan(showingNotice?: boolean): Promise<void> {
$$performFullScan(showingNotice?: boolean, ignoreSuspending?: boolean): Promise<void> {
throwShouldBeOverridden();
}

2
src/modules/core/ModuleDatabaseFileAccess.ts
View File

@@ -53,7 +53,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
async () => await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)
);
// For test, we need to clear the caches.
await this.localDatabase.hashCaches.clear();
this.localDatabase.clearCaches();
await this._test("readContent", async () => {
const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
if (!content) return "File not found";

4
src/modules/core/ModuleRebuilder.ts
View File

@@ -73,7 +73,7 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
await this.core.$$realizeSettingSyncMode();
await this.resetLocalDatabase();
await delay(1000);
await this.core.$$initializeDatabase(true);
await this.core.$$initializeDatabase(true, true, true);
await this.core.$$markRemoteLocked();
await this.core.$$tryResetRemoteDatabase();
await this.core.$$markRemoteLocked();
@@ -190,7 +190,7 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
if (makeLocalChunkBeforeSync) {
await this.core.fileHandler.createAllChunks(true);
} else if (!preventMakeLocalFilesBeforeSync) {
await this.core.$$initializeDatabase(true);
await this.core.$$initializeDatabase(true, true, true);
} else {
// Do not create local file entries before sync (Means use remote information)
}

8
src/modules/core/ModuleReplicator.ts
View File

@@ -30,7 +30,7 @@ import {
import { isAnyNote } from "../../lib/src/common/utils";
import { EVENT_FILE_SAVED, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
import { globalSlipBoard } from "../../lib/src/bureau/bureau";
import { $msg } from "../../lib/src/common/i18n";
import { clearHandlers } from "../../lib/src/replication/SyncParamsHandler";
@@ -150,12 +150,12 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
}
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
this.localDatabase.clearCaches();
// Perform the synchronisation once.
if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) {
await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
this.localDatabase.clearCaches();
await this.core.$$getReplicator().markRemoteResolved(this.settings);
Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
} else {
@@ -310,7 +310,7 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
const change = docs[0];
if (!change) return;
if (isChunk(change._id)) {
globalSlipBoard.submit("read-chunk", change._id, change as EntryLeaf);
this.localDatabase.onNewLeaf(change as EntryLeaf);
return;
}
if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;

20
src/modules/essential/ModuleInitializerFile.ts
View File

@@ -20,7 +20,7 @@ import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
import { withConcurrency } from "octagonal-wheels/iterable/map";
export class ModuleInitializerFile extends AbstractModule implements ICoreModule {
async $$performFullScan(showingNotice?: boolean): Promise<void> {
async $$performFullScan(showingNotice?: boolean, ignoreSuspending: boolean = false): Promise<void> {
this._log("Opening the key-value database", LOG_LEVEL_VERBOSE);
const isInitialized = (await this.core.kvDB.get<boolean>("initialized")) || false;
// synchronize all files between database and storage.
@@ -34,6 +34,16 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
}
return;
}
if (!ignoreSuspending && this.settings.suspendFileWatching) {
if (showingNotice) {
this._log(
"Now suspending file watching. Synchronising between the storage and the local database is now prevented.",
LOG_LEVEL_NOTICE,
"syncAll"
);
}
return;
}
if (showingNotice) {
this._log("Initializing", LOG_LEVEL_NOTICE, "syncAll");
@@ -355,11 +365,15 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
this._log(`Checking expired file history done`);
}
async $$initializeDatabase(showingNotice: boolean = false, reopenDatabase = true): Promise<boolean> {
async $$initializeDatabase(
showingNotice: boolean = false,
reopenDatabase = true,
ignoreSuspending: boolean = false
): Promise<boolean> {
this.core.$$resetIsReady();
if (!reopenDatabase || (await this.core.$$openDatabase())) {
if (this.localDatabase.isReady) {
await this.core.$$performFullScan(showingNotice);
await this.core.$$performFullScan(showingNotice, ignoreSuspending);
}
if (!(await this.core.$everyOnDatabaseInitialized(showingNotice))) {
this._log(`Initializing database has been failed on some module`, LOG_LEVEL_NOTICE);

4
src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts
View File

@@ -875,7 +875,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}
await purgeUnreferencedChunks(remoteDBConn.db, true, this.plugin.settings, false);
await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, true);
this.plugin.localDatabase.hashCaches.clear();
this.plugin.localDatabase.clearCaches();
});
}
@@ -895,7 +895,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}
await purgeUnreferencedChunks(remoteDBConnection.db, false, this.plugin.settings, true);
await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, false);
this.plugin.localDatabase.hashCaches.clear();
this.plugin.localDatabase.clearCaches();
await balanceChunkPurgedDBs(this.plugin.localDatabase.localDatabase, remoteDBConnection.db);
this.plugin.localDatabase.refreshSettings();
Logger(

2
src/modules/features/SettingDialogue/PaneAdvanced.ts
View File

@@ -6,7 +6,7 @@ import type { PageFunctions } from "./SettingPane.ts";
export function paneAdvanced(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void {
void addPanel(paneEl, "Memory cache").then((paneEl) => {
new Setting(paneEl).autoWireNumeric("hashCacheMaxCount", { clampMin: 10 });
new Setting(paneEl).autoWireNumeric("hashCacheMaxAmount", { clampMin: 1 });
// new Setting(paneEl).autoWireNumeric("hashCacheMaxAmount", { clampMin: 1 });
});
void addPanel(paneEl, "Local Database Tweak").then((paneEl) => {
paneEl.addClass("wizardHidden");

2
src/modules/features/SettingDialogue/PaneHatch.ts
View File

@@ -335,7 +335,7 @@ ${stringifyYaml({
Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns");
const targetPatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns");
this.plugin.localDatabase.hashCaches.clear();
this.plugin.localDatabase.clearCaches();
Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
const files = this.plugin.settings.syncInternalFiles
? await this.plugin.storageAccess.getFilesIncludeHidden("/", targetPatterns, ignorePatterns)

20
src/modules/features/SettingDialogue/PanePatches.ts
View File

@@ -28,9 +28,9 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen
void addPanel(paneEl, "Compatibility (Database structure)").then((paneEl) => {
new Setting(paneEl).autoWireToggle("useIndexedDBAdapter", { invert: true, holdValue: true });
new Setting(paneEl)
.autoWireToggle("doNotUseFixedRevisionForChunks", { holdValue: true })
.setClass("wizardHidden");
// new Setting(paneEl)
// .autoWireToggle("doNotUseFixedRevisionForChunks", { holdValue: true })
// .setClass("wizardHidden");
new Setting(paneEl).autoWireToggle("handleFilenameCaseSensitive", { holdValue: true }).setClass("wizardHidden");
this.addOnSaved("useIndexedDBAdapter", async () => {
@@ -99,13 +99,13 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen
});
void addPanel(paneEl, "Remote Database Tweak (In sunset)").then((paneEl) => {
new Setting(paneEl).autoWireToggle("useEden").setClass("wizardHidden");
const onlyUsingEden = visibleOnly(() => this.isConfiguredAs("useEden", true));
new Setting(paneEl).autoWireNumeric("maxChunksInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
new Setting(paneEl)
.autoWireNumeric("maxTotalLengthInEden", { onUpdate: onlyUsingEden })
.setClass("wizardHidden");
new Setting(paneEl).autoWireNumeric("maxAgeInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
// new Setting(paneEl).autoWireToggle("useEden").setClass("wizardHidden");
// const onlyUsingEden = visibleOnly(() => this.isConfiguredAs("useEden", true));
// new Setting(paneEl).autoWireNumeric("maxChunksInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
// new Setting(paneEl)
// .autoWireNumeric("maxTotalLengthInEden", { onUpdate: onlyUsingEden })
// .setClass("wizardHidden");
// new Setting(paneEl).autoWireNumeric("maxAgeInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
new Setting(paneEl).autoWireToggle("enableCompression").setClass("wizardHidden");
});

2
tsconfig.json
View File

@@ -23,5 +23,5 @@
}
},
"include": ["**/*.ts"],
"exclude": ["pouchdb-browser-webpack", "utils", "src/lib/apps"]
"exclude": ["pouchdb-browser-webpack", "utils", "src/lib/apps", "**/*.test.ts"]
}