Chore: Migrate file system sync to TypeScript (#10361)

.eslintignore

@@ -777,6 +777,7 @@ packages/lib/ObjectUtils.js
 packages/lib/PoorManIntervals.js
 packages/lib/RotatingLogs.test.js
 packages/lib/RotatingLogs.js
+packages/lib/SyncTargetFilesystem.js
 packages/lib/SyncTargetJoplinCloud.js
 packages/lib/SyncTargetJoplinServer.js
 packages/lib/SyncTargetNone.js
@@ -816,6 +817,7 @@ packages/lib/errorUtils.js
 packages/lib/errors.js
 packages/lib/eventManager.js
 packages/lib/file-api-driver-joplinServer.js
+packages/lib/file-api-driver-local.js
 packages/lib/file-api-driver-memory.js
 packages/lib/file-api-driver.test.js
 packages/lib/file-api.test.js

.gitignore

@@ -757,6 +757,7 @@ packages/lib/ObjectUtils.js
 packages/lib/PoorManIntervals.js
 packages/lib/RotatingLogs.test.js
 packages/lib/RotatingLogs.js
+packages/lib/SyncTargetFilesystem.js
 packages/lib/SyncTargetJoplinCloud.js
 packages/lib/SyncTargetJoplinServer.js
 packages/lib/SyncTargetNone.js
@@ -796,6 +797,7 @@ packages/lib/errorUtils.js
 packages/lib/errors.js
 packages/lib/eventManager.js
 packages/lib/file-api-driver-joplinServer.js
+packages/lib/file-api-driver-local.js
 packages/lib/file-api-driver-memory.js
 packages/lib/file-api-driver.test.js
 packages/lib/file-api.test.js

packages/app-cli/app/main.js

@@ -26,7 +26,7 @@ const sharp = require('sharp');
 const { shimInit } = require('@joplin/lib/shim-init-node.js');
 const shim = require('@joplin/lib/shim').default;
 const { _ } = require('@joplin/lib/locale');
-const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
+const FileApiDriverLocal = require('@joplin/lib/file-api-driver-local').default;
 const EncryptionService = require('@joplin/lib/services/e2ee/EncryptionService').default;
 const envFromArgs = require('@joplin/lib/envFromArgs');
 const nodeSqlite = require('sqlite3');

packages/app-desktop/main-html.js

@@ -26,7 +26,7 @@ const shim = require('@joplin/lib/shim').default;
 const { shimInit } = require('@joplin/lib/shim-init-node.js');
 const bridge = require('@electron/remote').require('./bridge').default;
 const EncryptionService = require('@joplin/lib/services/e2ee/EncryptionService').default;
-const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
+const FileApiDriverLocal = require('@joplin/lib/file-api-driver-local').default;
 const React = require('react');
 const nodeSqlite = require('sqlite3');
 const initLib = require('@joplin/lib/initLib').default;

packages/app-mobile/root.tsx

@@ -72,13 +72,13 @@ const { SideMenuContentNote } = require('./components/side-menu-content-note.js'
 const { DatabaseDriverReactNative } = require('./utils/database-driver-react-native');
 import { reg } from '@joplin/lib/registry';
 const { defaultState } = require('@joplin/lib/reducer');
-const { FileApiDriverLocal } = require('@joplin/lib/file-api-driver-local');
+import FileApiDriverLocal from '@joplin/lib/file-api-driver-local';
 import ResourceFetcher from '@joplin/lib/services/ResourceFetcher';
 import SearchEngine from '@joplin/lib/services/search/SearchEngine';
 import WelcomeUtils from '@joplin/lib/WelcomeUtils';
 import { themeStyle } from './components/global-style';
 import SyncTargetRegistry from '@joplin/lib/SyncTargetRegistry';
-const SyncTargetFilesystem = require('@joplin/lib/SyncTargetFilesystem.js');
+import SyncTargetFilesystem from '@joplin/lib/SyncTargetFilesystem';
 const SyncTargetNextcloud = require('@joplin/lib/SyncTargetNextcloud.js');
 const SyncTargetWebDAV = require('@joplin/lib/SyncTargetWebDAV.js');
 const SyncTargetDropbox = require('@joplin/lib/SyncTargetDropbox.js');

packages/lib/BaseApplication.ts

@@ -30,7 +30,7 @@ import fs = require('fs-extra');
 const EventEmitter = require('events');
 const syswidecas = require('./vendor/syswide-cas');
 import SyncTargetRegistry from './SyncTargetRegistry';
-const SyncTargetFilesystem = require('./SyncTargetFilesystem.js');
+import SyncTargetFilesystem from './SyncTargetFilesystem';
 const SyncTargetNextcloud = require('./SyncTargetNextcloud.js');
 const SyncTargetWebDAV = require('./SyncTargetWebDAV.js');
 const SyncTargetDropbox = require('./SyncTargetDropbox.js');

packages/lib/SyncTargetFilesystem.js

@@ -1,44 +0,0 @@
-const BaseSyncTarget = require('./BaseSyncTarget').default;
-const { _ } = require('./locale');
-const Setting = require('./models/Setting').default;
-const { FileApi } = require('./file-api.js');
-const { FileApiDriverLocal } = require('./file-api-driver-local');
-const Synchronizer = require('./Synchronizer').default;
-
-class SyncTargetFilesystem extends BaseSyncTarget {
-	static id() {
-		return 2;
-	}
-
-	static targetName() {
-		return 'filesystem';
-	}
-
-	static label() {
-		return _('File system');
-	}
-
-	static unsupportedPlatforms() {
-		return ['ios'];
-	}
-
-	async isAuthenticated() {
-		return true;
-	}
-
-	async initFileApi() {
-		const syncPath = Setting.value('sync.2.path');
-		const driver = new FileApiDriverLocal();
-		const fileApi = new FileApi(syncPath, driver);
-		fileApi.setLogger(this.logger());
-		fileApi.setSyncTargetId(SyncTargetFilesystem.id());
-		await driver.mkdir(syncPath);
-		return fileApi;
-	}
-
-	async initSynchronizer() {
-		return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
-	}
-}
-
-module.exports = SyncTargetFilesystem;

packages/lib/SyncTargetFilesystem.ts

@@ -0,0 +1,43 @@
+import BaseSyncTarget from './BaseSyncTarget';
+import { _ } from './locale';
+import Setting from './models/Setting';
+import { FileApi } from './file-api';
+import FileApiDriverLocal from './file-api-driver-local';
+import Synchronizer from './Synchronizer';
+
+export default class SyncTargetFilesystem extends BaseSyncTarget {
+	public static id() {
+		return 2;
+	}
+
+	public static targetName() {
+		return 'filesystem';
+	}
+
+	public static label() {
+		return _('File system');
+	}
+
+	public static unsupportedPlatforms() {
+		return ['ios'];
+	}
+
+	public async isAuthenticated() {
+		return true;
+	}
+
+	public async initFileApi() {
+		const syncPath = Setting.value('sync.2.path');
+		const driver = new FileApiDriverLocal();
+		const fileApi = new FileApi(syncPath, driver);
+		fileApi.setLogger(this.logger());
+		fileApi.setSyncTargetId(SyncTargetFilesystem.id());
+		await driver.mkdir(syncPath);
+		return fileApi;
+	}
+
+	public async initSynchronizer() {
+		return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
+	}
+}
+

packages/lib/file-api-driver-local.ts

@@ -1,4 +1,7 @@
-const { basicDelta } = require('./file-api');
+import JoplinError from './JoplinError';
+
+import { DeltaOptions, GetOptions, ItemStat, PutOptions, basicDelta } from './file-api';
+import FsDriverBase, { Stat } from './fs-driver-base';
 
 // NOTE: when synchronising with the file system the time resolution is the second (unlike milliseconds for OneDrive for instance).
 // What it means is that if, for example, client 1 changes a note at time t, and client 2 changes the same note within the same second,
@@ -13,21 +16,24 @@ const { basicDelta } = require('./file-api');
 // check that it is indeed the problem, check log-database.txt of both clients, search for the note ID, and most likely both notes
 // will have been modified at the same exact second at some point. If not, it's another bug that needs to be investigated.
 
-class FileApiDriverLocal {
+export default class FileApiDriverLocal {
-	fsErrorToJsError_(error, path = null) {
+	public static fsDriver_: FsDriverBase;
+
+	private fsErrorToJsError_(error: JoplinError, path: string|null = null) {
 		let msg = error.toString();
 		if (path !== null) msg += `. Path: ${path}`;
-		const output = new Error(msg);
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Partial refactor of old coe from before rule was applied
+		const output: any = new Error(msg);
 		if (error.code) output.code = error.code;
 		return output;
 	}
 
-	fsDriver() {
+	public fsDriver() {
 		if (!FileApiDriverLocal.fsDriver_) { throw new Error('FileApiDriverLocal.fsDriver_ not set!'); }
 		return FileApiDriverLocal.fsDriver_;
 	}
 
-	async stat(path) {
+	public async stat(path: string) {
 		try {
 			const s = await this.fsDriver().stat(path);
 			if (!s) return null;
@@ -37,7 +43,7 @@ class FileApiDriverLocal {
 		}
 	}
 
-	metadataFromStat_(stat) {
+	private metadataFromStat_(stat: Stat): ItemStat {
 		return {
 			path: stat.path,
 			// created_time: stat.birthtime.getTime(),
@@ -46,7 +52,7 @@ class FileApiDriverLocal {
 		};
 	}
 
-	metadataFromStats_(stats) {
+	private metadataFromStats_(stats: Stat[]): ItemStat[] {
 		const output = [];
 		for (let i = 0; i < stats.length; i++) {
 			const mdStat = this.metadataFromStat_(stats[i]);
@@ -55,7 +61,7 @@ class FileApiDriverLocal {
 		return output;
 	}
 
-	async setTimestamp(path, timestampMs) {
+	public async setTimestamp(path: string, timestampMs: number) {
 		try {
 			await this.fsDriver().setTimestamp(path, new Date(timestampMs));
 		} catch (error) {
@@ -63,8 +69,8 @@ class FileApiDriverLocal {
 		}
 	}
 
-	async delta(path, options) {
+	public async delta(path: string, options: DeltaOptions) {
-		const getStatFn = async path => {
+		const getStatFn = async (path: string) => {
 			const stats = await this.fsDriver().readDirStats(path);
 			return this.metadataFromStats_(stats);
 		};
@@ -77,7 +83,7 @@ class FileApiDriverLocal {
 		}
 	}
 
-	async list(path) {
+	public async list(path: string) {
 		try {
 			const stats = await this.fsDriver().readDirStats(path);
 			const output = this.metadataFromStats_(stats);
@@ -85,14 +91,14 @@ class FileApiDriverLocal {
 			return {
 				items: output,
 				hasMore: false,
-				context: null,
+				context: null as unknown,
 			};
 		} catch (error) {
 			throw this.fsErrorToJsError_(error, path);
 		}
 	}
 
-	async get(path, options) {
+	public async get(path: string, options: GetOptions) {
 		if (!options) options = {};
 		let output = null;
 
@@ -112,7 +118,7 @@ class FileApiDriverLocal {
 		return output;
 	}
 
-	async mkdir(path) {
+	public async mkdir(path: string) {
 		if (await this.fsDriver().exists(path)) return;
 
 		try {
@@ -139,7 +145,7 @@ class FileApiDriverLocal {
 		// });
 	}
 
-	async put(path, content, options = null) {
+	public async put(path: string, content: string, options: PutOptions = null) {
 		if (!options) options = {};
 
 		try {
@@ -167,7 +173,7 @@ class FileApiDriverLocal {
 		// });
 	}
 
-	async delete(path) {
+	public async delete(path: string) {
 		try {
 			await this.fsDriver().unlink(path);
 		} catch (error) {
@@ -190,7 +196,7 @@ class FileApiDriverLocal {
 		// });
 	}
 
-	async move(oldPath, newPath) {
+	public async move(oldPath: string, newPath: string) {
 		try {
 			await this.fsDriver().move(oldPath, newPath);
 		} catch (error) {
@@ -218,11 +224,11 @@ class FileApiDriverLocal {
 		// throw lastError;
 	}
 
-	format() {
+	public format() {
 		throw new Error('Not supported');
 	}
 
-	async clearRoot(baseDir) {
+	public async clearRoot(baseDir: string) {
 		if (baseDir.startsWith('content://')) {
 			const result = await this.list(baseDir);
 			for (const item of result.items) {
@@ -234,5 +240,3 @@ class FileApiDriverLocal {
 		}
 	}
 }
-
-module.exports = { FileApiDriverLocal };

packages/lib/file-api.ts

@@ -1,4 +1,4 @@
-import Logger from '@joplin/utils/Logger';
+import Logger, { LoggerWrapper } from '@joplin/utils/Logger';
 import shim from './shim';
 import BaseItem from './models/BaseItem';
 import time from './time';
@@ -100,6 +100,36 @@ async function tryAndRepeat(fn: Function, count: number) {
 	}
 }
 
+export interface DeltaOptions {
+	allItemIdsHandler(): Promise<string[]>;
+	logger?: LoggerWrapper;
+	wipeOutFailSafe: boolean;
+}
+
+export enum GetOptionsTarget {
+	String = 'string',
+	File = 'file',
+}
+
+export interface GetOptions {
+	target?: GetOptionsTarget;
+	path?: string;
+	encoding?: string;
+
+	source?: string;
+}
+
+export interface PutOptions {
+	path?: string;
+	source?: string;
+}
+
+export interface ItemStat {
+	path: string;
+	updated_time: number;
+	isDir: boolean;
+}
+
 class FileApi {
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
@@ -316,7 +346,7 @@ class FileApi {
 		return tryAndRepeat(() => this.driver_.mkdir(this.fullPath(path)), this.requestRepeatCount());
 	}
 
-	public async stat(path: string) {
+	public async stat(path: string): Promise<ItemStat> {
 		logger.debug(`stat ${this.fullPath(path)}`);
 
 		const output = await tryAndRepeat(() => this.driver_.stat(this.fullPath(path)), this.requestRepeatCount());
@@ -327,8 +357,7 @@ class FileApi {
 	}
 
 	// Returns UTF-8 encoded string by default, or a Response if `options.target = 'file'`
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
+	public get(path: string, options: GetOptions = null) {
-	public get(path: string, options: any = null) {
 		if (!options) options = {};
 		if (!options.encoding) options.encoding = 'utf8';
 		logger.debug(`get ${this.fullPath(path)}`);
@@ -336,7 +365,7 @@ class FileApi {
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
-	public async put(path: string, content: any, options: any = null) {
+	public async put(path: string, content: any, options: PutOptions = null) {
 		logger.debug(`put ${this.fullPath(path)}`, options);
 
 		if (options && options.source === 'file') {
@@ -372,8 +401,7 @@ class FileApi {
 		return tryAndRepeat(() => this.driver_.clearRoot(this.baseDir()), this.requestRepeatCount());
 	}
 
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
+	public delta(path: string, options: DeltaOptions|null = null): Promise<PaginatedList> {
-	public delta(path: string, options: any = null): Promise<PaginatedList> {
 		logger.debug(`delta ${this.fullPath(path)}`);
 		return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
 	}
@@ -423,7 +451,7 @@ function basicDeltaContextFromOptions_(options: any) {
 // a built-in delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
 // the file system do not.
 // eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/no-explicit-any -- Old code before rule was applied, Old code before rule was applied
-async function basicDelta(path: string, getDirStatFn: Function, options: any) {
+async function basicDelta(path: string, getDirStatFn: Function, options: DeltaOptions) {
 	const outputLimit = 50;
 	const itemIds = await options.allItemIdsHandler();
 	if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');

packages/lib/fs-driver-base.ts

@@ -84,6 +84,10 @@ export default class FsDriverBase {
 		throw new Error('Not implemented: remove');
 	}
 
+	public async setTimestamp(_path: string, _timestampDate: Date): Promise<void> {
+		throw new Error('Not implemented: setTimestamp');
+	}
+
 	public async isDirectory(path: string) {
 		const stat = await this.stat(path);
 		return !stat ? false : stat.isDirectory();

packages/lib/services/synchronizer/Synchronizer.resources.test.ts

@@ -375,6 +375,7 @@ describe('Synchronizer.resources', () => {
 		const resource: ResourceEntity = (await Resource.all())[0];
 		const resourcePath = `.resource/${resource.id}`;
 
+		await synchronizer().api().mkdir('.resource/');
 		await synchronizer().api().put(resourcePath, 'before upload');
 		expect(await synchronizer().api().get(resourcePath)).toBe('before upload');
 		await synchronizerStart();

packages/lib/shim-init-node.ts

@@ -13,7 +13,7 @@ import { DownloadController } from './downloadController';
 import { TextItem } from 'pdfjs-dist/types/src/display/api';
 import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
 
-const { FileApiDriverLocal } = require('./file-api-driver-local');
+import FileApiDriverLocal from './file-api-driver-local';
 const mimeUtils = require('./mime-utils.js').mime;
 const { _ } = require('./locale');
 const http = require('http');

packages/lib/shim.ts

@@ -1,6 +1,7 @@
 import * as React from 'react';
 import { NoteEntity, ResourceEntity } from './services/database/types';
 import type FsDriverBase from './fs-driver-base';
+import type FileApiDriverLocal from './file-api-driver-local';
 
 export interface CreateResourceFromPathOptions {
 	resizeLargeImages?: 'always' | 'never' | 'ask';
@@ -260,8 +261,7 @@ const shim = {
 		throw new Error('Not implemented: fsDriver');
 	},
 
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
+	FileApiDriverLocal: null as typeof FileApiDriverLocal,
-	FileApiDriverLocal: null as any,
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 	readLocalFileBase64: (_path: string): any => {

packages/lib/testing/test-utils.ts

@@ -30,13 +30,13 @@ import MasterKey from '../models/MasterKey';
 import BaseItem from '../models/BaseItem';
 import { FileApi } from '../file-api';
 const FileApiDriverMemory = require('../file-api-driver-memory').default;
-const { FileApiDriverLocal } = require('../file-api-driver-local');
+import FileApiDriverLocal from '../file-api-driver-local';
 const { FileApiDriverWebDav } = require('../file-api-driver-webdav.js');
 const { FileApiDriverDropbox } = require('../file-api-driver-dropbox.js');
 const { FileApiDriverOneDrive } = require('../file-api-driver-onedrive.js');
 import SyncTargetRegistry from '../SyncTargetRegistry';
 const SyncTargetMemory = require('../SyncTargetMemory.js');
-const SyncTargetFilesystem = require('../SyncTargetFilesystem.js');
+import SyncTargetFilesystem from '../SyncTargetFilesystem';
 const SyncTargetNextcloud = require('../SyncTargetNextcloud.js');
 const SyncTargetDropbox = require('../SyncTargetDropbox.js');
 const SyncTargetAmazonS3 = require('../SyncTargetAmazonS3.js');