self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2024-12-15 09:14:07 +02:00
Code Issues Releases Activity

- Changes and performance improvements:

Browse Source 
- Now the saving files are processed by Blob.
    - The V2-Format has been reverted.
    - New encoding format has been enabled in default.
    - WARNING: Since this version, the compatibilities with older Filesystem LiveSync have been lost.
This commit is contained in:
vorotamoroz 2023-11-24 10:31:58 +00:00
parent ac9428e96b
commit b644fb791d
7 changed files with 43 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/CmdConfigSync.ts
View File

@ -1,13 +1,13 @@
import { writable } from 'svelte/store'; import { writable } from 'svelte/store';
import { Notice, type PluginManifest, parseYaml, normalizePath } from "./deps"; import { Notice, type PluginManifest, parseYaml, normalizePath } from "./deps";
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry } from "./lib/src/types"; import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
import { delay, getDocData } from "./lib/src/utils"; import { createTextBlob, delay, getDocData } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { WrappedNotice } from "./lib/src/wrapper"; import { WrappedNotice } from "./lib/src/wrapper";
import { readString, crc32CKHash, decodeBinary, encodeBinary } from "./lib/src/strbin"; import { readString, crc32CKHash, decodeBinary, arrayBufferToBase64 } from "./lib/src/strbin";
import { serialized } from "./lib/src/lock"; import { serialized } from "./lib/src/lock";
import { LiveSyncCommands } from "./LiveSyncCommands"; import { LiveSyncCommands } from "./LiveSyncCommands";
import { stripAllPrefixes } from "./lib/src/path"; import { stripAllPrefixes } from "./lib/src/path";
@ -549,7 +549,7 @@ export class ConfigSync extends LiveSyncCommands {
const contentBin = await this.app.vault.adapter.readBinary(path); const contentBin = await this.app.vault.adapter.readBinary(path);
let content: string[]; let content: string[];
try { try {
content = await encodeBinary(contentBin, this.settings.useV1); content = await arrayBufferToBase64(contentBin);
if (path.toLowerCase().endsWith("/manifest.json")) { if (path.toLowerCase().endsWith("/manifest.json")) {
const v = readString(new Uint8Array(contentBin)); const v = readString(new Uint8Array(contentBin));
try { try {
@ -652,10 +652,10 @@ export class ConfigSync extends LiveSyncCommands {
return return
} }
const content = serialize(dt); const content = createTextBlob(serialize(dt));
try { try {
const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, false); const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, false);
let saveData: LoadedEntry; let saveData: SavingEntry;
if (old === false) { if (old === false) {
saveData = { saveData = {
_id: id, _id: id,
@ -664,7 +664,7 @@ export class ConfigSync extends LiveSyncCommands {
mtime, mtime,
ctime: mtime, ctime: mtime,
datatype: "newnote", datatype: "newnote",
size: content.length, size: content.size,
children: [], children: [],
deleted: false, deleted: false,
type: "newnote", type: "newnote",
@ -679,7 +679,7 @@ export class ConfigSync extends LiveSyncCommands {
...old, ...old,
data: content, data: content,
mtime, mtime,
size: content.length, size: content.size,
datatype: "newnote", datatype: "newnote",
children: [], children: [],
deleted: false, deleted: false,

12
src/CmdHiddenFileSync.ts
View File

@ -1,7 +1,7 @@
import { normalizePath, type PluginManifest } from "./deps"; import { normalizePath, type PluginManifest } from "./deps";
import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED } from "./lib/src/types"; import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry } from "./lib/src/types";
import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "./types"; import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
import { Parallels, delay, isDocContentSame } from "./lib/src/utils"; import { Parallels, createBinaryBlob, delay, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils"; import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils";
@ -409,9 +409,9 @@ export class HiddenFileSync extends LiveSyncCommands {
const id = await this.path2id(file.path, ICHeader); const id = await this.path2id(file.path, ICHeader);
const prefixedFileName = addPrefix(file.path, ICHeader); const prefixedFileName = addPrefix(file.path, ICHeader);
const contentBin = await this.app.vault.adapter.readBinary(file.path); const contentBin = await this.app.vault.adapter.readBinary(file.path);
let content: string[]; let content: Blob;
try { try {
content = await encodeBinary(contentBin, this.settings.useV1); content = createBinaryBlob(contentBin);
} catch (ex) { } catch (ex) {
Logger(`The file ${file.path} could not be encoded`); Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE); Logger(ex, LOG_LEVEL_VERBOSE);
@ -421,7 +421,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return await serialized("file-" + prefixedFileName, async () => { return await serialized("file-" + prefixedFileName, async () => {
try { try {
const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false); const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false);
let saveData: LoadedEntry; let saveData: SavingEntry;
if (old === false) { if (old === false) {
saveData = { saveData = {
_id: id, _id: id,
@ -559,7 +559,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return true; return true;
} else { } else {
const contentBin = await this.app.vault.adapter.readBinary(filename); const contentBin = await this.app.vault.adapter.readBinary(filename);
const content = await encodeBinary(contentBin, this.settings.useV1); const content = await encodeBinary(contentBin);
if (isDocContentSame(content, fileOnDB.data) && !force) { if (isDocContentSame(content, fileOnDB.data) && !force) {
// Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE); // Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
return true; return true;

11
src/CmdPluginAndTheirSettings.ts
View File

@ -1,8 +1,8 @@
import { normalizePath, type PluginManifest } from "./deps"; import { normalizePath, type PluginManifest } from "./deps";
import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry } from "./lib/src/types"; import type { DocumentID, EntryDoc, FilePathWithPrefix, LoadedEntry, SavingEntry } from "./lib/src/types";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types";
import { type PluginDataEntry, PERIODIC_PLUGIN_SWEEP, type PluginList, type DevicePluginList, PSCHeader, PSCHeaderEnd } from "./types"; import { type PluginDataEntry, PERIODIC_PLUGIN_SWEEP, type PluginList, type DevicePluginList, PSCHeader, PSCHeaderEnd } from "./types";
import { getDocData, isDocContentSame } from "./lib/src/utils"; import { createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { isPluginMetadata, PeriodicProcessor } from "./utils"; import { isPluginMetadata, PeriodicProcessor } from "./utils";
@ -211,13 +211,14 @@ export class PluginAndTheirSettings extends LiveSyncCommands {
mtime: mtime, mtime: mtime,
type: "plugin", type: "plugin",
}; };
const d: LoadedEntry = { const blob = createTextBlob(JSON.stringify(p));
const d: SavingEntry = {
_id: p._id, _id: p._id,
path: p._id as string as FilePathWithPrefix, path: p._id as string as FilePathWithPrefix,
data: JSON.stringify(p), data: blob,
ctime: mtime, ctime: mtime,
mtime: mtime, mtime: mtime,
size: 0, size: blob.size,
children: [], children: [],
datatype: "plain", datatype: "plain",
type: "plain" type: "plain"

4
src/CmdSetupLiveSync.ts
View File

@ -60,7 +60,7 @@ export class SetupLiveSync extends LiveSyncCommands {
delete setting[k]; delete setting[k];
} }
} }
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true)); const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`; const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri); await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE); Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
@ -70,7 +70,7 @@ export class SetupLiveSync extends LiveSyncCommands {
if (encryptingPassphrase === false) if (encryptingPassphrase === false)
return; return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" }; const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, true)); const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`; const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri); await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE); Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);

9
src/ObsidianLiveSyncSettingTab.ts
View File

@ -1832,15 +1832,6 @@ ${stringifyYaml(pluginConfig)}`;
await this.plugin.saveSettings(); await this.plugin.saveSettings();
}) })
); );
new Setting(containerHatchEl)
.setName("Use binary and encryption version 1")
.setDesc("Use the previous data format for other products which shares the remote database.")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.useV1).onChange(async (value) => {
this.plugin.settings.useV1 = value;
await this.plugin.saveSettings();
})
);
addScreenElement("50", containerHatchEl); addScreenElement("50", containerHatchEl);

2
src/lib

@ -1 +1 @@
Subproject commit 4a4e7a26b4bfd6eb33c156235b0543576f1761ba Subproject commit 2ea80a40c11c367efcb6861dbf3c4ee076b705b9

38
src/main.ts
View File

@ -2,9 +2,9 @@ const isDebug = false;
import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps"; import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, } from "./lib/src/types"; import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, } from "./lib/src/types";
import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types"; import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
import { arrayToChunkedArray, getDocData, isDocContentSame } from "./lib/src/utils"; import { arrayToChunkedArray, createBinaryBlob, createTextBlob, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger, setGlobalLogFunction } from "./lib/src/logger"; import { Logger, setGlobalLogFunction } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { ConflictResolveModal } from "./ConflictResolveModal"; import { ConflictResolveModal } from "./ConflictResolveModal";
@ -16,7 +16,7 @@ import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissin
import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store"; import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store";
import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores"; import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores";
import { setNoticeClass } from "./lib/src/wrapper"; import { setNoticeClass } from "./lib/src/wrapper";
import { versionNumberString2Number, writeString, decodeBinary, encodeBinary, readString } from "./lib/src/strbin"; import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/strbin";
import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path"; import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path";
import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock"; import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock";
import { Semaphore } from "./lib/src/semaphore"; import { Semaphore } from "./lib/src/semaphore";
@ -204,7 +204,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf); const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
if (passphrase !== "false" && typeof passphrase === "string") { if (passphrase !== "false" && typeof passphrase === "string") {
enableEncryption(db, passphrase, useDynamicIterationCount, false, this.settings.useV1); enableEncryption(db, passphrase, useDynamicIterationCount, false);
} }
if (skipInfo) { if (skipInfo) {
return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } }; return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } };
@ -404,10 +404,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin
Logger(`Checking expired file history done`); Logger(`Checking expired file history done`);
} }
async onLayoutReady() { async onLayoutReady() {
if (this.settings.useV1 === undefined) {
this.settings.useV1 = await this.askEnableV2();
await this.saveSettingData();
}
this.registerFileWatchEvents(); this.registerFileWatchEvents();
if (!this.localDatabase.isReady) { if (!this.localDatabase.isReady) {
Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE); Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE);
@ -567,8 +563,13 @@ Note: We can always able to read V1 format. It will be progressively converted.
this.addCommand({ this.addCommand({
id: "livesync-dump", id: "livesync-dump",
name: "Dump information of this doc ", name: "Dump information of this doc ",
editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => { // editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => {
const file = view.file; // const file = view.file;
// if (!file) return;
// this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false);
// },
callback: () => {
const file = this.app.workspace.getActiveFile();
if (!file) return; if (!file) return;
this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false); this.localDatabase.getDBEntry(getPathFromTFile(file), {}, true, false);
}, },
@ -682,6 +683,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
} }
async onload() { async onload() {
logStore.subscribe(e => this.addLog(e.message, e.level, e.key)); logStore.subscribe(e => this.addLog(e.message, e.level, e.key));
Logger("loading plugin"); Logger("loading plugin");
this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this)); this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
@ -826,7 +828,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) { async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) {
if (this.usedPassphrase != "") { if (this.usedPassphrase != "") {
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false, true); return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false);
} }
const passphrase = await this.getPassphrase(settings); const passphrase = await this.getPassphrase(settings);
@ -834,7 +836,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT); Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT);
return ""; return "";
} }
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false, true); const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false);
if (dec) { if (dec) {
this.usedPassphrase = passphrase; this.usedPassphrase = passphrase;
return dec; return dec;
@ -2497,7 +2499,7 @@ Or if you are sure know what had been happened, we can unlock the database from
if (shouldBeIgnored(file.path)) { if (shouldBeIgnored(file.path)) {
return true; return true;
} }
let content: string | string[]; let content: Blob;
let datatype: "plain" | "newnote" = "newnote"; let datatype: "plain" | "newnote" = "newnote";
if (!cache) { if (!cache) {
if (!isPlainText(file.name)) { if (!isPlainText(file.name)) {
@ -2505,7 +2507,7 @@ Or if you are sure know what had been happened, we can unlock the database from
const contentBin = await this.app.vault.readBinary(file); const contentBin = await this.app.vault.readBinary(file);
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE); Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try { try {
content = await encodeBinary(contentBin, this.settings.useV1); content = createBinaryBlob(contentBin);
} catch (ex) { } catch (ex) {
Logger(`The file ${file.path} could not be encoded`); Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE); Logger(ex, LOG_LEVEL_VERBOSE);
@ -2513,14 +2515,14 @@ Or if you are sure know what had been happened, we can unlock the database from
} }
datatype = "newnote"; datatype = "newnote";
} else { } else {
content = await this.app.vault.read(file); content = createTextBlob(await this.app.vault.read(file));
datatype = "plain"; datatype = "plain";
} }
} else { } else {
if (cache instanceof ArrayBuffer) { if (cache instanceof ArrayBuffer) {
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE); Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try { try {
content = await encodeBinary(cache, this.settings.useV1); content = createBinaryBlob(cache);
} catch (ex) { } catch (ex) {
Logger(`The file ${file.path} could not be encoded`); Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE); Logger(ex, LOG_LEVEL_VERBOSE);
@ -2528,13 +2530,13 @@ Or if you are sure know what had been happened, we can unlock the database from
} }
datatype = "newnote" datatype = "newnote"
} else { } else {
content = cache; content = createTextBlob(cache);
datatype = "plain"; datatype = "plain";
} }
} }
const fullPath = getPathFromTFile(file); const fullPath = getPathFromTFile(file);
const id = await this.path2id(fullPath); const id = await this.path2id(fullPath);
const d: LoadedEntry = { const d: SavingEntry = {
_id: id, _id: id,
path: getPathFromTFile(file), path: getPathFromTFile(file),
data: content, data: content,