From b644fb791d2bc2f8bc76d966fcf6d54b0d5a63b4 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 24 Nov 2023 10:31:58 +0000 Subject: [PATCH] - Changes and performance improvements: - Now the saving files are processed by Blob. - The V2-Format has been reverted. - New encoding format has been enabled in default. - WARNING: Since this version, the compatibilities with older Filesystem LiveSync have been lost. --- src/CmdConfigSync.ts | 16 ++++++------- src/CmdHiddenFileSync.ts | 12 +++++----- src/CmdPluginAndTheirSettings.ts | 11 +++++---- src/CmdSetupLiveSync.ts | 4 ++-- src/ObsidianLiveSyncSettingTab.ts | 9 -------- src/lib | 2 +- src/main.ts | 38 ++++++++++++++++--------------- 7 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/CmdConfigSync.ts b/src/CmdConfigSync.ts index 225b212..d2fb1dc 100644 --- a/src/CmdConfigSync.ts +++ b/src/CmdConfigSync.ts @@ -1,13 +1,13 @@ import { writable } from 'svelte/store'; import { Notice, type PluginManifest, parseYaml, normalizePath } from "./deps"; -import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry } from "./lib/src/types"; +import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types"; -import { delay, getDocData } from "./lib/src/utils"; +import { createTextBlob, delay, getDocData } from "./lib/src/utils"; import { Logger } from "./lib/src/logger"; import { WrappedNotice } from "./lib/src/wrapper"; -import { readString, crc32CKHash, decodeBinary, encodeBinary } from "./lib/src/strbin"; +import { readString, crc32CKHash, decodeBinary, arrayBufferToBase64 } from "./lib/src/strbin"; import { serialized } from "./lib/src/lock"; import { LiveSyncCommands } from "./LiveSyncCommands"; import { stripAllPrefixes } from "./lib/src/path"; @@ -549,7 +549,7 @@ export class ConfigSync extends LiveSyncCommands { const contentBin = await this.app.vault.adapter.readBinary(path); let content: string[]; try { - content = await encodeBinary(contentBin, this.settings.useV1); + content = await arrayBufferToBase64(contentBin); if (path.toLowerCase().endsWith("/manifest.json")) { const v = readString(new Uint8Array(contentBin)); try { @@ -652,10 +652,10 @@ export class ConfigSync extends LiveSyncCommands { return } - const content = serialize(dt); + const content = createTextBlob(serialize(dt)); try { const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, false); - let saveData: LoadedEntry; + let saveData: SavingEntry; if (old === false) { saveData = { _id: id, @@ -664,7 +664,7 @@ export class ConfigSync extends LiveSyncCommands { mtime, ctime: mtime, datatype: "newnote", - size: content.length, + size: content.size, children: [], deleted: false, type: "newnote", @@ -679,7 +679,7 @@ export class ConfigSync extends LiveSyncCommands { ...old, data: content, mtime, - size: content.length, + size: content.size, datatype: "newnote", children: [], deleted: false, diff --git a/src/CmdHiddenFileSync.ts b/src/CmdHiddenFileSync.ts index 6fcfaef..b276a9c 100644 --- a/src/CmdHiddenFileSync.ts +++ b/src/CmdHiddenFileSync.ts @@ -1,7 +1,7 @@ import { normalizePath, type PluginManifest } from "./deps"; -import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED } from "./lib/src/types"; +import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry } from "./lib/src/types"; import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "./types"; -import { Parallels, delay, isDocContentSame } from "./lib/src/utils"; +import { Parallels, createBinaryBlob, delay, isDocContentSame } from "./lib/src/utils"; import { Logger } from "./lib/src/logger"; import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils"; @@ -409,9 +409,9 @@ export class HiddenFileSync extends LiveSyncCommands { const id = await this.path2id(file.path, ICHeader); const prefixedFileName = addPrefix(file.path, ICHeader); const contentBin = await this.app.vault.adapter.readBinary(file.path); - let content: string[]; + let content: Blob; try { - content = await encodeBinary(contentBin, this.settings.useV1); + content = createBinaryBlob(contentBin); } catch (ex) { Logger(`The file ${file.path} could not be encoded`); Logger(ex, LOG_LEVEL_VERBOSE); @@ -421,7 +421,7 @@ export class HiddenFileSync extends LiveSyncCommands { return await serialized("file-" + prefixedFileName, async () => { try { const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false); - let saveData: LoadedEntry; + let saveData: SavingEntry; if (old === false) { saveData = { _id: id, @@ -559,7 +559,7 @@ export class HiddenFileSync extends LiveSyncCommands { return true; } else { const contentBin = await this.app.vault.adapter.readBinary(filename); - const content = await encodeBinary(contentBin, this.settings.useV1); + const content = await encodeBinary(contentBin); if (isDocContentSame(content, fileOnDB.data) && !force) { // Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE); return true; diff --git a/src/CmdPluginAndTheirSettings.ts b/src/CmdPluginAndTheirSettings.ts index 2c48c0b..814ca37 100644 --- a/src/CmdPluginAndTheirSettings.ts +++ b/src/CmdPluginAndTheirSettings.ts @@ -1,8 +1,8 @@ import { normalizePath, type PluginManifest } from "./deps"; -import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry } from "./lib/src/types"; +import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry, SavingEntry } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types"; import { type PluginDataEntry, PERIODIC_PLUGIN_SWEEP, type PluginList, type DevicePluginList, PSCHeader, PSCHeaderEnd } from "./types"; -import { getDocData, isDocContentSame } from "./lib/src/utils"; +import { createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils"; import { Logger } from "./lib/src/logger"; import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { isPluginMetadata, PeriodicProcessor } from "./utils"; @@ -211,13 +211,14 @@ export class PluginAndTheirSettings extends LiveSyncCommands { mtime: mtime, type: "plugin", }; - const d: LoadedEntry = { + const blob = createTextBlob(JSON.stringify(p)); + const d: SavingEntry = { _id: p._id, path: p._id as string as FilePathWithPrefix, - data: JSON.stringify(p), + data: blob, ctime: mtime, mtime: mtime, - size: 0, + size: blob.size, children: [], datatype: "plain", type: "plain" diff --git a/src/CmdSetupLiveSync.ts b/src/CmdSetupLiveSync.ts index 92bd022..857b83d 100644 --- a/src/CmdSetupLiveSync.ts +++ b/src/CmdSetupLiveSync.ts @@ -60,7 +60,7 @@ export class SetupLiveSync extends LiveSyncCommands { delete setting[k]; } } - const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true)); + const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false)); const uri = `${configURIBase}${encryptedSetting}`; await navigator.clipboard.writeText(uri); Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE); @@ -70,7 +70,7 @@ export class SetupLiveSync extends LiveSyncCommands { if (encryptingPassphrase === false) return; const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" }; - const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true)); + const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false)); const uri = `${configURIBase}${encryptedSetting}`; await navigator.clipboard.writeText(uri); Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE); diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts index 6da3fc4..eaec37a 100644 --- a/src/ObsidianLiveSyncSettingTab.ts +++ b/src/ObsidianLiveSyncSettingTab.ts @@ -1832,15 +1832,6 @@ ${stringifyYaml(pluginConfig)}`; await this.plugin.saveSettings(); }) ); - new Setting(containerHatchEl) - .setName("Use binary and encryption version 1") - .setDesc("Use the previous data format for other products which shares the remote database.") - .addToggle((toggle) => - toggle.setValue(this.plugin.settings.useV1).onChange(async (value) => { - this.plugin.settings.useV1 = value; - await this.plugin.saveSettings(); - }) - ); addScreenElement("50", containerHatchEl); diff --git a/src/lib b/src/lib index 4a4e7a2..2ea80a4 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 4a4e7a26b4bfd6eb33c156235b0543576f1761ba +Subproject commit 2ea80a40c11c367efcb6861dbf3c4ee076b705b9 diff --git a/src/main.ts b/src/main.ts index 8cd63d8..34a937c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,9 +2,9 @@ const isDebug = false; import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps"; import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; -import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, } from "./lib/src/types"; +import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, } from "./lib/src/types"; import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types"; -import { arrayToChunkedArray, getDocData, isDocContentSame } from "./lib/src/utils"; +import { arrayToChunkedArray, createBinaryBlob, createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils"; import { Logger, setGlobalLogFunction } from "./lib/src/logger"; import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { ConflictResolveModal } from "./ConflictResolveModal"; @@ -16,7 +16,7 @@ import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissin import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store"; import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores"; import { setNoticeClass } from "./lib/src/wrapper"; -import { versionNumberString2Number, writeString, decodeBinary, encodeBinary, readString } from "./lib/src/strbin"; +import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/strbin"; import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path"; import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock"; import { Semaphore } from "./lib/src/semaphore"; @@ -204,7 +204,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin const db: PouchDB.Database = new PouchDB(uri, conf); if (passphrase !== "false" && typeof passphrase === "string") { - enableEncryption(db, passphrase, useDynamicIterationCount, false, this.settings.useV1); + enableEncryption(db, passphrase, useDynamicIterationCount, false); } if (skipInfo) { return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } }; @@ -404,10 +404,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin Logger(`Checking expired file history done`); } async onLayoutReady() { - if (this.settings.useV1 === undefined) { - this.settings.useV1 = await this.askEnableV2(); - await this.saveSettingData(); - } this.registerFileWatchEvents(); if (!this.localDatabase.isReady) { Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE); @@ -567,8 +563,13 @@ Note: We can always able to read V1 format. It will be progressively converted. this.addCommand({ id: "livesync-dump", name: "Dump information of this doc ", - editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => { - const file = view.file; + // editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => { + // const file = view.file; + // if (!file) return; + // this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false); + // }, + callback: () => { + const file = this.app.workspace.getActiveFile(); if (!file) return; this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false); }, @@ -682,6 +683,7 @@ Note: We can always able to read V1 format. It will be progressively converted. } async onload() { + logStore.subscribe(e => this.addLog(e.message, e.level, e.key)); Logger("loading plugin"); this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this)); @@ -826,7 +828,7 @@ Note: We can always able to read V1 format. It will be progressively converted. async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) { if (this.usedPassphrase != "") { - return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false, true); + return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false); } const passphrase = await this.getPassphrase(settings); @@ -834,7 +836,7 @@ Note: We can always able to read V1 format. It will be progressively converted. Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT); return ""; } - const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false, true); + const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false); if (dec) { this.usedPassphrase = passphrase; return dec; @@ -2497,7 +2499,7 @@ Or if you are sure know what had been happened, we can unlock the database from if (shouldBeIgnored(file.path)) { return true; } - let content: string | string[]; + let content: Blob; let datatype: "plain" | "newnote" = "newnote"; if (!cache) { if (!isPlainText(file.name)) { @@ -2505,7 +2507,7 @@ Or if you are sure know what had been happened, we can unlock the database from const contentBin = await this.app.vault.readBinary(file); Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE); try { - content = await encodeBinary(contentBin, this.settings.useV1); + content = createBinaryBlob(contentBin); } catch (ex) { Logger(`The file ${file.path} could not be encoded`); Logger(ex, LOG_LEVEL_VERBOSE); @@ -2513,14 +2515,14 @@ Or if you are sure know what had been happened, we can unlock the database from } datatype = "newnote"; } else { - content = await this.app.vault.read(file); + content = createTextBlob(await this.app.vault.read(file)); datatype = "plain"; } } else { if (cache instanceof ArrayBuffer) { Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE); try { - content = await encodeBinary(cache, this.settings.useV1); + content = createBinaryBlob(cache); } catch (ex) { Logger(`The file ${file.path} could not be encoded`); Logger(ex, LOG_LEVEL_VERBOSE); @@ -2528,13 +2530,13 @@ Or if you are sure know what had been happened, we can unlock the database from } datatype = "newnote" } else { - content = cache; + content = createTextBlob(cache); datatype = "plain"; } } const fullPath = getPathFromTFile(file); const id = await this.path2id(fullPath); - const d: LoadedEntry = { + const d: SavingEntry = { _id: id, path: getPathFromTFile(file), data: content,