self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-21 09:38:01 +02:00
Code Issues Releases Activity

Chore: Migrate file system sync to TypeScript (#10361)

Browse Source
This commit is contained in:
Henry Heino 2024-04-25 05:31:48 -07:00 committed by GitHub
parent c7c4371902
commit 83b50aaa8e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 124 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintignore
View File

@ -777,6 +777,7 @@ packages/lib/ObjectUtils.js
packages/lib/PoorManIntervals.js
packages/lib/RotatingLogs.test.js
packages/lib/RotatingLogs.js
packages/lib/SyncTargetFilesystem.js
packages/lib/SyncTargetJoplinCloud.js
packages/lib/SyncTargetJoplinServer.js
packages/lib/SyncTargetNone.js
@ -816,6 +817,7 @@ packages/lib/errorUtils.js
packages/lib/errors.js
packages/lib/eventManager.js
packages/lib/file-api-driver-joplinServer.js
packages/lib/file-api-driver-local.js
packages/lib/file-api-driver-memory.js
packages/lib/file-api-driver.test.js
packages/lib/file-api.test.js

2
.gitignore vendored
View File

@ -757,6 +757,7 @@ packages/lib/ObjectUtils.js
packages/lib/PoorManIntervals.js
packages/lib/RotatingLogs.test.js
packages/lib/RotatingLogs.js
packages/lib/SyncTargetFilesystem.js
packages/lib/SyncTargetJoplinCloud.js
packages/lib/SyncTargetJoplinServer.js
packages/lib/SyncTargetNone.js
@ -796,6 +797,7 @@ packages/lib/errorUtils.js
packages/lib/errors.js
packages/lib/eventManager.js
packages/lib/file-api-driver-joplinServer.js
packages/lib/file-api-driver-local.js
packages/lib/file-api-driver-memory.js
packages/lib/file-api-driver.test.js
packages/lib/file-api.test.js

2
packages/app-cli/app/main.js
View File

@ -26,7 +26,7 @@ const sharp = require('sharp');
const { shimInit } = require('@joplin/lib/shim-init-node.js');
const shim = require('@joplin/lib/shim').default;
const { _ } = require('@joplin/lib/locale');
const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
const FileApiDriverLocal = require('@joplin/lib/file-api-driver-local').default;
const EncryptionService = require('@joplin/lib/services/e2ee/EncryptionService').default;
const envFromArgs = require('@joplin/lib/envFromArgs');
const nodeSqlite = require('sqlite3');

2
packages/app-desktop/main-html.js
View File

@ -26,7 +26,7 @@ const shim = require('@joplin/lib/shim').default;
const { shimInit } = require('@joplin/lib/shim-init-node.js');
const bridge = require('@electron/remote').require('./bridge').default;
const EncryptionService = require('@joplin/lib/services/e2ee/EncryptionService').default;
const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
const FileApiDriverLocal = require('@joplin/lib/file-api-driver-local').default;
const React = require('react');
const nodeSqlite = require('sqlite3');
const initLib = require('@joplin/lib/initLib').default;

4
packages/app-mobile/root.tsx
View File

@ -72,13 +72,13 @@ const { SideMenuContentNote } = require('./components/side-menu-content-note.js'
const { DatabaseDriverReactNative } = require('./utils/database-driver-react-native');
import { reg } from '@joplin/lib/registry';
const { defaultState } = require('@joplin/lib/reducer');
const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
import FileApiDriverLocal from '@joplin/lib/file-api-driver-local';
import ResourceFetcher from '@joplin/lib/services/ResourceFetcher';
import SearchEngine from '@joplin/lib/services/search/SearchEngine';
import WelcomeUtils from '@joplin/lib/WelcomeUtils';
import { themeStyle } from './components/global-style';
import SyncTargetRegistry from '@joplin/lib/SyncTargetRegistry';
const SyncTargetFilesystem = require('@joplin/lib/SyncTargetFilesystem.js');
import SyncTargetFilesystem from '@joplin/lib/SyncTargetFilesystem';
const SyncTargetNextcloud = require('@joplin/lib/SyncTargetNextcloud.js');
const SyncTargetWebDAV = require('@joplin/lib/SyncTargetWebDAV.js');
const SyncTargetDropbox = require('@joplin/lib/SyncTargetDropbox.js');

2
packages/lib/BaseApplication.ts
View File

@ -30,7 +30,7 @@ import fs = require('fs-extra');
const EventEmitter = require('events');
const syswidecas = require('./vendor/syswide-cas');
import SyncTargetRegistry from './SyncTargetRegistry';
const SyncTargetFilesystem = require('./SyncTargetFilesystem.js');
import SyncTargetFilesystem from './SyncTargetFilesystem';
const SyncTargetNextcloud = require('./SyncTargetNextcloud.js');
const SyncTargetWebDAV = require('./SyncTargetWebDAV.js');
const SyncTargetDropbox = require('./SyncTargetDropbox.js');

44
packages/lib/SyncTargetFilesystem.js
View File

@ -1,44 +0,0 @@
const BaseSyncTarget = require('./BaseSyncTarget').default;
const { _ } = require('./locale');
const Setting = require('./models/Setting').default;
const { FileApi } = require('./file-api.js');
const { FileApiDriverLocal } = require('./file-api-driver-local');
const Synchronizer = require('./Synchronizer').default;
class SyncTargetFilesystem extends BaseSyncTarget {
static id() {
return 2;
}
static targetName() {
return 'filesystem';
}
static label() {
return _('File system');
}
static unsupportedPlatforms() {
return ['ios'];
}
async isAuthenticated() {
return true;
}
async initFileApi() {
const syncPath = Setting.value('sync.2.path');
const driver = new FileApiDriverLocal();
const fileApi = new FileApi(syncPath, driver);
fileApi.setLogger(this.logger());
fileApi.setSyncTargetId(SyncTargetFilesystem.id());
await driver.mkdir(syncPath);
return fileApi;
}
async initSynchronizer() {
return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
}
}
module.exports = SyncTargetFilesystem;

43
packages/lib/SyncTargetFilesystem.ts Normal file
View File

@ -0,0 +1,43 @@
import BaseSyncTarget from './BaseSyncTarget';
import { _ } from './locale';
import Setting from './models/Setting';
import { FileApi } from './file-api';
import FileApiDriverLocal from './file-api-driver-local';
import Synchronizer from './Synchronizer';
export default class SyncTargetFilesystem extends BaseSyncTarget {
public static id() {
return 2;
}
public static targetName() {
return 'filesystem';
}
public static label() {
return _('File system');
}
public static unsupportedPlatforms() {
return ['ios'];
}
public async isAuthenticated() {
return true;
}
public async initFileApi() {
const syncPath = Setting.value('sync.2.path');
const driver = new FileApiDriverLocal();
const fileApi = new FileApi(syncPath, driver);
fileApi.setLogger(this.logger());
fileApi.setSyncTargetId(SyncTargetFilesystem.id());
await driver.mkdir(syncPath);
return fileApi;
}
public async initSynchronizer() {
return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
}
}

48
packages/lib/file-api-driver-local.js → packages/lib/file-api-driver-local.ts
View File

@ -1,4 +1,7 @@
const { basicDelta } = require('./file-api');
import JoplinError from './JoplinError';
import { DeltaOptions, GetOptions, ItemStat, PutOptions, basicDelta } from './file-api';
import FsDriverBase, { Stat } from './fs-driver-base';
// NOTE: when synchronising with the file system the time resolution is the second (unlike milliseconds for OneDrive for instance).
// What it means is that if, for example, client 1 changes a note at time t, and client 2 changes the same note within the same second,
@ -13,21 +16,24 @@ const { basicDelta } = require('./file-api');
// check that it is indeed the problem, check log-database.txt of both clients, search for the note ID, and most likely both notes
// will have been modified at the same exact second at some point. If not, it's another bug that needs to be investigated.
class FileApiDriverLocal {
fsErrorToJsError_(error, path = null) {
export default class FileApiDriverLocal {
public static fsDriver_: FsDriverBase;
private fsErrorToJsError_(error: JoplinError, path: string|null = null) {
let msg = error.toString();
if (path !== null) msg += `. Path: ${path}`;
const output = new Error(msg);
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Partial refactor of old coe from before rule was applied
const output: any = new Error(msg);
if (error.code) output.code = error.code;
return output;
}
fsDriver() {
public fsDriver() {
if (!FileApiDriverLocal.fsDriver_) { throw new Error('FileApiDriverLocal.fsDriver_ not set!'); }
return FileApiDriverLocal.fsDriver_;
}
async stat(path) {
public async stat(path: string) {
try {
const s = await this.fsDriver().stat(path);
if (!s) return null;
@ -37,7 +43,7 @@ class FileApiDriverLocal {
}
}
metadataFromStat_(stat) {
private metadataFromStat_(stat: Stat): ItemStat {
return {
path: stat.path,
// created_time: stat.birthtime.getTime(),
@ -46,7 +52,7 @@ class FileApiDriverLocal {
};
}
metadataFromStats_(stats) {
private metadataFromStats_(stats: Stat[]): ItemStat[] {
const output = [];
for (let i = 0; i < stats.length; i++) {
const mdStat = this.metadataFromStat_(stats[i]);
@ -55,7 +61,7 @@ class FileApiDriverLocal {
return output;
}
async setTimestamp(path, timestampMs) {
public async setTimestamp(path: string, timestampMs: number) {
try {
await this.fsDriver().setTimestamp(path, new Date(timestampMs));
} catch (error) {
@ -63,8 +69,8 @@ class FileApiDriverLocal {
}
}
async delta(path, options) {
const getStatFn = async path => {
public async delta(path: string, options: DeltaOptions) {
const getStatFn = async (path: string) => {
const stats = await this.fsDriver().readDirStats(path);
return this.metadataFromStats_(stats);
};
@ -77,7 +83,7 @@ class FileApiDriverLocal {
}
}
async list(path) {
public async list(path: string) {
try {
const stats = await this.fsDriver().readDirStats(path);
const output = this.metadataFromStats_(stats);
@ -85,14 +91,14 @@ class FileApiDriverLocal {
return {
items: output,
hasMore: false,
context: null,
context: null as unknown,
};
} catch (error) {
throw this.fsErrorToJsError_(error, path);
}
}
async get(path, options) {
public async get(path: string, options: GetOptions) {
if (!options) options = {};
let output = null;
@ -112,7 +118,7 @@ class FileApiDriverLocal {
return output;
}
async mkdir(path) {
public async mkdir(path: string) {
if (await this.fsDriver().exists(path)) return;
try {
@ -139,7 +145,7 @@ class FileApiDriverLocal {
// });
}
async put(path, content, options = null) {
public async put(path: string, content: string, options: PutOptions = null) {
if (!options) options = {};
try {
@ -167,7 +173,7 @@ class FileApiDriverLocal {
// });
}
async delete(path) {
public async delete(path: string) {
try {
await this.fsDriver().unlink(path);
} catch (error) {
@ -190,7 +196,7 @@ class FileApiDriverLocal {
// });
}
async move(oldPath, newPath) {
public async move(oldPath: string, newPath: string) {
try {
await this.fsDriver().move(oldPath, newPath);
} catch (error) {
@ -218,11 +224,11 @@ class FileApiDriverLocal {
// throw lastError;
}
format() {
public format() {
throw new Error('Not supported');
}
async clearRoot(baseDir) {
public async clearRoot(baseDir: string) {
if (baseDir.startsWith('content://')) {
const result = await this.list(baseDir);
for (const item of result.items) {
@ -234,5 +240,3 @@ class FileApiDriverLocal {
}
}
}
module.exports = { FileApiDriverLocal };

44
packages/lib/file-api.ts
View File

@ -1,4 +1,4 @@
import Logger from '@joplin/utils/Logger';
import Logger, { LoggerWrapper } from '@joplin/utils/Logger';
import shim from './shim';
import BaseItem from './models/BaseItem';
import time from './time';
@ -100,6 +100,36 @@ async function tryAndRepeat(fn: Function, count: number) {
}
}
export interface DeltaOptions {
allItemIdsHandler(): Promise<string[]>;
logger?: LoggerWrapper;
wipeOutFailSafe: boolean;
}
export enum GetOptionsTarget {
String = 'string',
File = 'file',
}
export interface GetOptions {
target?: GetOptionsTarget;
path?: string;
encoding?: string;
source?: string;
}
export interface PutOptions {
path?: string;
source?: string;
}
export interface ItemStat {
path: string;
updated_time: number;
isDir: boolean;
}
class FileApi {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
@ -316,7 +346,7 @@ class FileApi {
return tryAndRepeat(() => this.driver_.mkdir(this.fullPath(path)), this.requestRepeatCount());
}
public async stat(path: string) {
public async stat(path: string): Promise<ItemStat> {
logger.debug(`stat ${this.fullPath(path)}`);
const output = await tryAndRepeat(() => this.driver_.stat(this.fullPath(path)), this.requestRepeatCount());
@ -327,8 +357,7 @@ class FileApi {
}
// Returns UTF-8 encoded string by default, or a Response if `options.target = 'file'`
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
public get(path: string, options: any = null) {
public get(path: string, options: GetOptions = null) {
if (!options) options = {};
if (!options.encoding) options.encoding = 'utf8';
logger.debug(`get ${this.fullPath(path)}`);
@ -336,7 +365,7 @@ class FileApi {
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
public async put(path: string, content: any, options: any = null) {
public async put(path: string, content: any, options: PutOptions = null) {
logger.debug(`put ${this.fullPath(path)}`, options);
if (options && options.source === 'file') {
@ -372,8 +401,7 @@ class FileApi {
return tryAndRepeat(() => this.driver_.clearRoot(this.baseDir()), this.requestRepeatCount());
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
public delta(path: string, options: any = null): Promise<PaginatedList> {
public delta(path: string, options: DeltaOptions|null = null): Promise<PaginatedList> {
logger.debug(`delta ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
}
@ -423,7 +451,7 @@ function basicDeltaContextFromOptions_(options: any) {
// a built-in delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
// the file system do not.
// eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/no-explicit-any -- Old code before rule was applied, Old code before rule was applied
async function basicDelta(path: string, getDirStatFn: Function, options: any) {
async function basicDelta(path: string, getDirStatFn: Function, options: DeltaOptions) {
const outputLimit = 50;
const itemIds = await options.allItemIdsHandler();
if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');

4
packages/lib/fs-driver-base.ts
View File

@ -84,6 +84,10 @@ export default class FsDriverBase {
throw new Error('Not implemented: remove');
}
public async setTimestamp(_path: string, _timestampDate: Date): Promise<void> {
throw new Error('Not implemented: setTimestamp');
}
public async isDirectory(path: string) {
const stat = await this.stat(path);
return !stat ? false : stat.isDirectory();

1
packages/lib/services/synchronizer/Synchronizer.resources.test.ts
View File

@ -375,6 +375,7 @@ describe('Synchronizer.resources', () => {
const resource: ResourceEntity = (await Resource.all())[0];
const resourcePath = `.resource/${resource.id}`;
await synchronizer().api().mkdir('.resource/');
await synchronizer().api().put(resourcePath, 'before upload');
expect(await synchronizer().api().get(resourcePath)).toBe('before upload');
await synchronizerStart();

2
packages/lib/shim-init-node.ts
View File

@ -13,7 +13,7 @@ import { DownloadController } from './downloadController';
import { TextItem } from 'pdfjs-dist/types/src/display/api';
import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
const { FileApiDriverLocal } = require('./file-api-driver-local');
import FileApiDriverLocal from './file-api-driver-local';
const mimeUtils = require('./mime-utils.js').mime;
const { _ } = require('./locale');
const http = require('http');

4
packages/lib/shim.ts
View File

@ -1,6 +1,7 @@
import * as React from 'react';
import { NoteEntity, ResourceEntity } from './services/database/types';
import type FsDriverBase from './fs-driver-base';
import type FileApiDriverLocal from './file-api-driver-local';
export interface CreateResourceFromPathOptions {
resizeLargeImages?: 'always' | 'never' | 'ask';
@ -260,8 +261,7 @@ const shim = {
throw new Error('Not implemented: fsDriver');
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
FileApiDriverLocal: null as any,
FileApiDriverLocal: null as typeof FileApiDriverLocal,
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
readLocalFileBase64: (_path: string): any => {

4
packages/lib/testing/test-utils.ts
View File

@ -30,13 +30,13 @@ import MasterKey from '../models/MasterKey';
import BaseItem from '../models/BaseItem';
import { FileApi } from '../file-api';
const FileApiDriverMemory = require('../file-api-driver-memory').default;
const { FileApiDriverLocal } = require('../file-api-driver-local');
import FileApiDriverLocal from '../file-api-driver-local';
const { FileApiDriverWebDav } = require('../file-api-driver-webdav.js');
const { FileApiDriverDropbox } = require('../file-api-driver-dropbox.js');
const { FileApiDriverOneDrive } = require('../file-api-driver-onedrive.js');
import SyncTargetRegistry from '../SyncTargetRegistry';
const SyncTargetMemory = require('../SyncTargetMemory.js');
const SyncTargetFilesystem = require('../SyncTargetFilesystem.js');
import SyncTargetFilesystem from '../SyncTargetFilesystem';
const SyncTargetNextcloud = require('../SyncTargetNextcloud.js');
const SyncTargetDropbox = require('../SyncTargetDropbox.js');
const SyncTargetAmazonS3 = require('../SyncTargetAmazonS3.js');