self-hosted/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2024-12-12 09:04:06 +02:00
Code Issues Releases Activity

- Improved:

Browse Source 
- A New binary file handling implemented
  - A new encrypted format has been implemented
  - Now the chunk sizes will be adjusted for efficient sync
- Fixed:
  - levels of exception in some logs have been fixed
- Tidied:
  - Some Lint warnings have been suppressed.
This commit is contained in:
vorotamoroz 2023-09-29 18:55:46 +09:00
parent 395b7fbc42
commit d91c4f50b4
10 changed files with 56 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/CmdConfigSync.ts
View File

@ -7,7 +7,7 @@ import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
import { delay, getDocData } from "./lib/src/utils";
import { Logger } from "./lib/src/logger";
import { WrappedNotice } from "./lib/src/wrapper";
import { base64ToArrayBuffer, arrayBufferToBase64, readString, crc32CKHash } from "./lib/src/strbin";
import { readString, crc32CKHash, decodeBinary, encodeBinary } from "./lib/src/strbin";
import { serialized } from "./lib/src/lock";
import { LiveSyncCommands } from "./LiveSyncCommands";
import { stripAllPrefixes } from "./lib/src/path";
@ -328,7 +328,7 @@ export class ConfigSync extends LiveSyncCommands {
const path = `${baseDir}/${f.filename}`;
await this.ensureDirectoryEx(path);
if (!content) {
const dt = base64ToArrayBuffer(f.data);
const dt = decodeBinary(f.data);
await this.app.vault.adapter.writeBinary(path, dt);
} else {
await this.app.vault.adapter.write(path, content);
@ -460,7 +460,7 @@ export class ConfigSync extends LiveSyncCommands {
const contentBin = await this.app.vault.adapter.readBinary(path);
let content: string[];
try {
content = await arrayBufferToBase64(contentBin);
content = await encodeBinary(contentBin, this.settings.useV1);
if (path.toLowerCase().endsWith("/manifest.json")) {
const v = readString(new Uint8Array(contentBin));
try {

10
src/CmdHiddenFileSync.ts
View File

@ -6,7 +6,7 @@ import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js";
import { scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils";
import { WrappedNotice } from "./lib/src/wrapper";
import { base64ToArrayBuffer, arrayBufferToBase64 } from "./lib/src/strbin";
import { decodeBinary, encodeBinary } from "./lib/src/strbin";
import { serialized } from "./lib/src/lock";
import { JsonResolveModal } from "./JsonResolveModal";
import { LiveSyncCommands } from "./LiveSyncCommands";
@ -411,7 +411,7 @@ export class HiddenFileSync extends LiveSyncCommands {
const contentBin = await this.app.vault.adapter.readBinary(file.path);
let content: string[];
try {
content = await arrayBufferToBase64(contentBin);
content = await encodeBinary(contentBin, this.settings.useV1);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
@ -547,7 +547,7 @@ export class HiddenFileSync extends LiveSyncCommands {
}
if (!isExists) {
await this.ensureDirectoryEx(filename);
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
await this.app.vault.adapter.writeBinary(filename, decodeBinary(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);
@ -559,12 +559,12 @@ export class HiddenFileSync extends LiveSyncCommands {
return true;
} else {
const contentBin = await this.app.vault.adapter.readBinary(filename);
const content = await arrayBufferToBase64(contentBin);
const content = await encodeBinary(contentBin, this.settings.useV1);
if (isDocContentSame(content, fileOnDB.data) && !force) {
// Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
return true;
}
await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
await this.app.vault.adapter.writeBinary(filename, decodeBinary(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);

4
src/CmdSetupLiveSync.ts
View File

@ -60,7 +60,7 @@ export class SetupLiveSync extends LiveSyncCommands {
delete setting[k];
}
}
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
@ -70,7 +70,7 @@ export class SetupLiveSync extends LiveSyncCommands {
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);

8
src/DocumentHistoryModal.ts
View File

@ -1,6 +1,6 @@
import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
import { getPathFromTFile, isValidPath } from "./utils";
import { base64ToArrayBuffer, base64ToString, escapeStringToHTML } from "./lib/src/strbin";
import { decodeBinary, escapeStringToHTML, readString } from "./lib/src/strbin";
import ObsidianLiveSyncPlugin from "./main";
import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "./lib/src/types";
import { Logger } from "./lib/src/logger";
@ -89,7 +89,7 @@ export class DocumentHistoryModal extends Modal {
this.currentDoc = w;
this.info.innerHTML = `Modified:${new Date(w.mtime).toLocaleString()}`;
let result = "";
const w1data = w.datatype == "plain" ? getDocData(w.data) : base64ToString(w.data);
const w1data = w.datatype == "plain" ? getDocData(w.data) : readString(new Uint8Array(decodeBinary(w.data)));
this.currentDeleted = !!w.deleted;
this.currentText = w1data;
if (this.showDiff) {
@ -99,7 +99,7 @@ export class DocumentHistoryModal extends Modal {
const w2 = await db.getDBEntry(this.file, { rev: oldRev }, false, false, true);
if (w2 != false) {
const dmp = new diff_match_patch();
const w2data = w2.datatype == "plain" ? getDocData(w2.data) : base64ToString(w2.data);
const w2data = w2.datatype == "plain" ? getDocData(w2.data) : readString(new Uint8Array(decodeBinary(w.data)));
const diff = dmp.diff_main(w2data, w1data);
dmp.diff_cleanupSemantic(diff);
for (const v of diff) {
@ -204,7 +204,7 @@ export class DocumentHistoryModal extends Modal {
await focusFile(pathToWrite);
this.close();
} else if (this.currentDoc?.datatype == "newnote") {
await this.app.vault.adapter.writeBinary(pathToWrite, base64ToArrayBuffer(this.currentDoc.data));
await this.app.vault.adapter.writeBinary(pathToWrite, decodeBinary(this.currentDoc.data));
await focusFile(pathToWrite);
this.close();
} else {

4
src/GlobalHistory.svelte
View File

@ -7,7 +7,7 @@
import { DocumentHistoryModal } from "./DocumentHistoryModal";
import { isPlainText, stripAllPrefixes } from "./lib/src/path";
import { TFile } from "./deps";
import { arrayBufferToBase64 } from "./lib/src/strbin";
import { encodeBinary } from "./lib/src/strbin";
export let plugin: ObsidianLiveSyncPlugin;
let showDiffInfo = false;
@ -116,7 +116,7 @@
result = isDocContentSame(data, doc.data);
} else {
const data = await plugin.app.vault.readBinary(abs);
const dataEEncoded = await arrayBufferToBase64(data);
const dataEEncoded = await encodeBinary(data, plugin.settings.useV1);
result = isDocContentSame(dataEEncoded, doc.data);
}
if (result) {

4
src/JsonResolvePane.svelte
View File

@ -1,7 +1,7 @@
<script lang="ts">
import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "./deps";
import type { FilePath, LoadedEntry } from "./lib/src/types";
import { base64ToString } from "./lib/src/strbin";
import { decodeBinary, readString } from "./lib/src/strbin";
import { getDocData } from "./lib/src/utils";
import { mergeObject } from "./utils";
@ -26,7 +26,7 @@
let mode: SelectModes = defaultSelect as SelectModes;
function docToString(doc: LoadedEntry) {
return doc.datatype == "plain" ? getDocData(doc.data) : base64ToString(doc.data);
return doc.datatype == "plain" ? getDocData(doc.data) : readString(new Uint8Array(decodeBinary(doc.data)));
}
function revStringToRevNumber(rev: string) {
return rev.split("-")[0];

13
src/ObsidianLiveSyncSettingTab.ts
View File

@ -1270,7 +1270,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
new Setting(containerSyncSettingEl)
.setName("Enhance chunk size")
.setDesc("Enhance chunk size for binary files (0.1MBytes). This cannot be increased when using IBM Cloudant.")
.setDesc("Enhance chunk size for binary files (Ratio). This cannot be increased when using IBM Cloudant.")
.setClass("wizardHidden")
.addText((text) => {
text.setPlaceholder("")
@ -1278,7 +1278,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 1) {
v = 1;
v = 0;
}
this.plugin.settings.customChunkSize = v;
await this.plugin.saveSettings();
@ -1816,6 +1816,15 @@ ${stringifyYaml(pluginConfig)}`;
await this.plugin.saveSettings();
})
);
new Setting(containerHatchEl)
.setName("Use binary and encryption version 1")
.setDesc("Use the previous data format for other products which shares the remote database.")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.useV1).onChange(async (value) => {
this.plugin.settings.useV1 = value;
await this.plugin.saveSettings();
})
);
addScreenElement("50", containerHatchEl);

1
src/dialogs.ts
View File

@ -182,7 +182,6 @@ export class MessageBox extends Modal {
this.timer = undefined;
}
})
contentEl.createEl("h1", { text: this.title });
const div = contentEl.createDiv();
MarkdownRenderer.render(this.plugin.app, this.contentMd, div, "/", this.plugin);
const buttonSetting = new Setting(contentEl);

2
src/lib

@ -1 +1 @@
Subproject commit 6548bd3ed75dc087d74acdac5a473f95aaccf6a9
Subproject commit 4a955add9c274ea5fb3965e441e0202819f87249

37
src/main.ts
View File

@ -2,7 +2,7 @@ const isDebug = false;
import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE } from "./lib/src/types";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, } from "./lib/src/types";
import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
import { arrayToChunkedArray, getDocData, isDocContentSame } from "./lib/src/utils";
import { Logger, setGlobalLogFunction } from "./lib/src/logger";
@ -16,7 +16,7 @@ import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissin
import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store";
import { lockStore, logMessageStore, logStore, type LogEntry } from "./lib/src/stores";
import { setNoticeClass } from "./lib/src/wrapper";
import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayBufferToBase64, writeString } from "./lib/src/strbin";
import { versionNumberString2Number, writeString, decodeBinary, encodeBinary, readString } from "./lib/src/strbin";
import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/path";
import { isLockAcquired, serialized, skipIfDuplicated } from "./lib/src/lock";
import { Semaphore } from "./lib/src/semaphore";
@ -64,8 +64,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin
replicator!: LiveSyncDBReplicator;
statusBar?: HTMLElement;
suspended: boolean = false;
deviceAndVaultName: string = "";
suspended = false;
deviceAndVaultName = "";
isMobile = false;
isReady = false;
packageVersion = "";
@ -204,7 +204,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
if (passphrase !== "false" && typeof passphrase === "string") {
enableEncryption(db, passphrase, useDynamicIterationCount);
enableEncryption(db, passphrase, useDynamicIterationCount, false, this.settings.useV1);
}
if (skipInfo) {
return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } };
@ -404,6 +404,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin
Logger(`Checking expired file history done`);
}
async onLayoutReady() {
if (this.settings.useV1 === undefined) {
this.settings.useV1 = await this.askEnableV2();
await this.saveSettingData();
}
this.registerFileWatchEvents();
if (!this.localDatabase.isReady) {
Logger(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE);
@ -500,6 +504,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
}
Logger(`Additional safety scan done`, LOG_LEVEL_VERBOSE);
}
async askEnableV2() {
const message = `Since v0.20.0, Self-hosted LiveSync uses a new format for binary files and encrypted things. In the new format, files are split at meaningful delimitations, increasing the effectiveness of deduplication.
However, the new format lacks compatibility with LiveSync before v0.20.0 and related projects. Basically enabling V2 is recommended. but If you are using some related products, stay in a while, please!
Note: We can always able to read V1 format. It will be progressively converted. And, we can change this toggle later.`
const CHOICE_V2 = "Enable v2";
const CHOICE_V1 = "Keep v1";
const ret = await confirmWithMessage(this, "binary and encryption", message, [CHOICE_V2, CHOICE_V1], CHOICE_V1, 40);
return ret == CHOICE_V1;
}
async onload() {
logStore.subscribe(e => this.addLog(e.message, e.level, e.key));
@ -521,6 +535,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
if (lastVersion > this.settings.lastReadUpdates) {
Logger("Self-hosted LiveSync has undergone a major upgrade. Please open the setting dialog, and check the information pane.", LOG_LEVEL_NOTICE);
}
//@ts-ignore
if (this.app.isMobile) {
this.isMobile = true;
@ -797,7 +812,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) {
if (this.usedPassphrase != "") {
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false);
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false, true);
}
const passphrase = await this.getPassphrase(settings);
@ -805,7 +820,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
Logger("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT);
return "";
}
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false);
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false, true);
if (dec) {
this.usedPassphrase = passphrase;
return dec;
@ -1270,7 +1285,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL_NOTICE);
return;
}
const writeData = doc.datatype == "newnote" ? base64ToArrayBuffer(doc.data) : getDocData(doc.data);
const writeData = doc.datatype == "newnote" ? decodeBinary(doc.data) : getDocData(doc.data);
await this.ensureDirectoryEx(path);
try {
let outFile;
@ -1953,7 +1968,7 @@ Or if you are sure know what had been happened, we can unlock the database from
if (doc === false) return false;
let data = getDocData(doc.data)
if (doc.datatype == "newnote") {
data = base64ToString(data);
data = readString(new Uint8Array(decodeBinary(doc.data)));
} else if (doc.datatype == "plain") {
// NO OP.
}
@ -2473,7 +2488,7 @@ Or if you are sure know what had been happened, we can unlock the database from
const contentBin = await this.app.vault.readBinary(file);
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try {
content = await arrayBufferToBase64(contentBin);
content = await encodeBinary(contentBin, this.settings.useV1);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
@ -2488,7 +2503,7 @@ Or if you are sure know what had been happened, we can unlock the database from
if (cache instanceof ArrayBuffer) {
Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
try {
content = await arrayBufferToBase64(cache);
content = await encodeBinary(cache, this.settings.useV1);
} catch (ex) {
Logger(`The file ${file.path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);