self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2024-12-15 09:14:07 +02:00
Code Issues Releases Activity

- Changes and performance improvements:

Browse Source 
- Now the saving files are processed by Blob.
    - The V2-Format has been reverted.
    - New encoding format has been enabled in default.
    - WARNING: Since this version, the compatibilities with older Filesystem LiveSync have been lost.
This commit is contained in:
vorotamoroz 2023-11-24 10:31:58 +00:00
parent ac9428e96b
commit b644fb791d
7 changed files with 43 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/CmdConfigSync.ts
View File

@ -1,13 +1,13 @@
import { writable } from 'svelte/store';
import { Notice, type PluginManifest, parseYaml, normalizePath } from "./deps";
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry } from "./lib/src/types";
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
import { delay, getDocData } from "./lib/src/utils";
import { createTextBlob, delay, getDocData } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { WrappedNotice } from "./lib/src/wrapper";
import { readString, crc32CKHash, decodeBinary, encodeBinary } from "./lib/src/strbin";
import { readString, crc32CKHash, decodeBinary, arrayBufferToBase64 } from "./lib/src/strbin";
import { serialized } from "./lib/src/lock";
import { LiveSyncCommands } from "./LiveSyncCommands";
import { stripAllPrefixes } from "./lib/src/path";
@ -549,7 +549,7 @@ export class ConfigSync extends LiveSyncCommands {
const contentBin = await this.app.vault.adapter.readBinary(path);
let content: string[];
try {
content = await encodeBinary(contentBin, this.settings.useV1);
content = await arrayBufferToBase64(contentBin);
if (path.toLowerCase().endsWith("/manifest.json")) {
const v = readString(new Uint8Array(contentBin));
try {
@ -652,10 +652,10 @@ export class ConfigSync extends LiveSyncCommands {
return
}
const content = serialize(dt);
const content = createTextBlob(serialize(dt));
try {
const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, false);
let saveData: LoadedEntry;
let saveData: SavingEntry;
if (old === false) {
saveData = {
_id: id,
@ -664,7 +664,7 @@ export class ConfigSync extends LiveSyncCommands {
mtime,
ctime: mtime,
datatype: "newnote",
size: content.length,
size: content.size,
children: [],
deleted: false,
type: "newnote",
@ -679,7 +679,7 @@ export class ConfigSync extends LiveSyncCommands {
...old,
data: content,
mtime,
size: content.length,
size: content.size,
datatype: "newnote",
children: [],
deleted: false,

12
src/CmdHiddenFileSync.ts
View File

@ -1,7 +1,7 @@
import { normalizePath, type PluginManifest } from "./deps";
import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED } from "./lib/src/types";
import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry } from "./lib/src/types";
import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
import { Parallels, delay, isDocContentSame } from "./lib/src/utils";
import { Parallels, createBinaryBlob, delay, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils";
@ -409,9 +409,9 @@ export class HiddenFileSync extends LiveSyncCommands {
const id = await this.path2id(file.path, ICHeader);
const prefixedFileName = addPrefix(file.path, ICHeader);
const contentBin = await this.app.vault.adapter.readBinary(file.path);
let content: string[];
let content: Blob;
try {
content = await encodeBinary(contentBin, this.settings.useV1);
content = createBinaryBlob(contentBin);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
@ -421,7 +421,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return await serialized("file-" + prefixedFileName, async () => {
try {
const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false);
let saveData: LoadedEntry;
let saveData: SavingEntry;
if (old === false) {
saveData = {
_id: id,
@ -559,7 +559,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return true;
} else {
const contentBin = await this.app.vault.adapter.readBinary(filename);
const content = await encodeBinary(contentBin, this.settings.useV1);
const content = await encodeBinary(contentBin);
if (isDocContentSame(content, fileOnDB.data) && !force) {
// Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
return true;

11
src/CmdPluginAndTheirSettings.ts
View File

@ -1,8 +1,8 @@
import { normalizePath, type PluginManifest } from "./deps";
import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry } from "./lib/src/types";
import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry, SavingEntry } from "./lib/src/types";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types";
import { type PluginDataEntry, PERIODIC_PLUGIN_SWEEP, type PluginList, type DevicePluginList, PSCHeader, PSCHeaderEnd } from "./types";
import { getDocData, isDocContentSame } from "./lib/src/utils";
import { createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { isPluginMetadata, PeriodicProcessor } from "./utils";
@ -211,13 +211,14 @@ export class PluginAndTheirSettings extends LiveSyncCommands {
mtime: mtime,
type: "plugin",
};
const d: LoadedEntry = {
const blob = createTextBlob(JSON.stringify(p));
const d: SavingEntry = {
_id: p._id,
path: p._id as string as FilePathWithPrefix,
data: JSON.stringify(p),
data: blob,
ctime: mtime,
mtime: mtime,
size: 0,
size: blob.size,
children: [],
datatype: "plain",
type: "plain"

4
src/CmdSetupLiveSync.ts
View File

@ -60,7 +60,7 @@ export class SetupLiveSync extends LiveSyncCommands {
delete setting[k];
}
}
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true));
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
@ -70,7 +70,7 @@ export class SetupLiveSync extends LiveSyncCommands {
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true));
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);

9
src/ObsidianLiveSyncSettingTab.ts
View File

@ -1832,15 +1832,6 @@ ${stringifyYaml(pluginConfig)}`;
await this.plugin.saveSettings();
})
);
new Setting(containerHatchEl)
.setName("Use binary and encryption version 1")
.setDesc("Use the previous data format for other products which shares the remote database.")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.useV1).onChange(async (value) => {
this.plugin.settings.useV1 = value;
await this.plugin.saveSettings();
})
);
addScreenElement("50", containerHatchEl);

2
src/lib

@ -1 +1 @@
Subproject commit 4a4e7a26b4bfd6eb33c156235b0543576f1761ba
Subproject commit 2ea80a40c11c367efcb6861dbf3c4ee076b705b9

38
src/main.ts
View File

@ -2,9 +2,9 @@ const isDebug = false;
import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, } from "./lib/src/types";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, } from "./lib/src/types";
import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
import { arrayToChunkedArray, getDocData, isDocContentSame } from "./lib/src/utils";
import { arrayToChunkedArray, createBinaryBlob, createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger, setGlobalLogFunction } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { ConflictResolveModal } from "./ConflictResolveModal";
@ -16,7 +16,7 @@ import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissin
import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store";
import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores";
import { setNoticeClass } from "./lib/src/wrapper";
import { versionNumberString2Number, writeString, decodeBinary, encodeBinary, readString } from "./lib/src/strbin";
import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/strbin";
import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path";
import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock";
import { Semaphore } from "./lib/src/semaphore";
@ -204,7 +204,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
if (passphrase !== "false" && typeof passphrase === "string") {
enableEncryption(db, passphrase, useDynamicIterationCount, false, this.settings.useV1);
enableEncryption(db, passphrase, useDynamicIterationCount, false);
}
if (skipInfo) {
return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } };
@ -404,10 +404,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin
Logger(`Checking expired file history done`);
}
async onLayoutReady() {
if (this.settings.useV1 === undefined) {
this.settings.useV1 = await this.askEnableV2();
await this.saveSettingData();
}
this.registerFileWatchEvents();
if (!this.localDatabase.isReady) {
Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE);
@ -567,8 +563,13 @@ Note: We can always able to read V1 format. It will be progressively converted.
this.addCommand({
id: "livesync-dump",
name: "Dump information of this doc ",
editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => {
const file = view.file;
// editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => {
// const file = view.file;
// if (!file) return;
// this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false);
// },
callback: () => {
const file = this.app.workspace.getActiveFile();
if (!file) return;
this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false);
},
@ -682,6 +683,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
}
async onload() {
logStore.subscribe(e => this.addLog(e.message, e.level, e.key));
Logger("loading plugin");
this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
@ -826,7 +828,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) {
if (this.usedPassphrase != "") {
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false, true);
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false);
}
const passphrase = await this.getPassphrase(settings);
@ -834,7 +836,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT);
return "";
}
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false, true);
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false);
if (dec) {
this.usedPassphrase = passphrase;
return dec;
@ -2497,7 +2499,7 @@ Or if you are sure know what had been happened, we can unlock the database from
if (shouldBeIgnored(file.path)) {
return true;
}
let content: string | string[];
let content: Blob;
let datatype: "plain" | "newnote" = "newnote";
if (!cache) {
if (!isPlainText(file.name)) {
@ -2505,7 +2507,7 @@ Or if you are sure know what had been happened, we can unlock the database from
const contentBin = await this.app.vault.readBinary(file);
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try {
content = await encodeBinary(contentBin, this.settings.useV1);
content = createBinaryBlob(contentBin);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
@ -2513,14 +2515,14 @@ Or if you are sure know what had been happened, we can unlock the database from
}
datatype = "newnote";
} else {
content = await this.app.vault.read(file);
content = createTextBlob(await this.app.vault.read(file));
datatype = "plain";
}
} else {
if (cache instanceof ArrayBuffer) {
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try {
content = await encodeBinary(cache, this.settings.useV1);
content = createBinaryBlob(cache);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
@ -2528,13 +2530,13 @@ Or if you are sure know what had been happened, we can unlock the database from
}
datatype = "newnote"
} else {
content = cache;
content = createTextBlob(cache);
datatype = "plain";
}
}
const fullPath = getPathFromTFile(file);
const id = await this.path2id(fullPath);
const d: LoadedEntry = {
const d: SavingEntry = {
_id: id,
path: getPathFromTFile(file),
data: content,