test

packages/app-cli/tests/support/plugins/selected_text/api/JoplinInterop.d.ts

@@ -1,17 +0,0 @@
-import { ExportModule, ImportModule } from './types';
-/**
- * Provides a way to create modules to import external data into Joplin or to export notes into any arbitrary format.
- *
- * [View the demo plugin](https://github.com/laurent22/joplin/tree/dev/packages/app-cli/tests/support/plugins/json_export)
- *
- * To implement an import or export module, you would simply define an object with various event handlers that are called
- * by the application during the import/export process.
- *
- * See the documentation of the [[ExportModule]] and [[ImportModule]] for more information.
- *
- * You may also want to refer to the Joplin API documentation to see the list of properties for each item (note, notebook, etc.) - https://joplinapp.org/api/references/rest_api/
- */
-export default class JoplinInterop {
-    registerExportModule(module: ExportModule): Promise<void>;
-    registerImportModule(module: ImportModule): Promise<void>;
-}

packages/lib/file-api-driver-joplinServer.ts

@@ -42,9 +42,6 @@ export default class FileApiDriverJoplinServer {
 			isDeleted: isDeleted,
 		};
 
-		// TODO - HANDLE DELETED
-		// if (md['.tag'] === 'deleted') output.isDeleted = true;
-
 		return output;
 	}
 

packages/lib/models/Setting.ts

@@ -461,7 +461,7 @@ class Setting extends BaseModel {
 					return settings['sync.target'] == SyncTargetRegistry.nameToId('joplinServer');
 				},
 				public: true,
-				label: () => _('Joplin Server username'),
+				label: () => _('Joplin Server email'),
 				storage: SettingStorage.File,
 			},
 			'sync.9.password': {

packages/server/src/apps/joplin/Application.ts

@@ -3,12 +3,12 @@ import Logger from '@joplin/lib/Logger';
 import BaseModel, { ModelType } from '@joplin/lib/BaseModel';
 import BaseItem from '@joplin/lib/models/BaseItem';
 import Note from '@joplin/lib/models/Note';
-import { File, Share, Uuid } from '../../db';
+import { File, JoplinFileContent, Share, Uuid } from '../../db';
 import { NoteEntity } from '@joplin/lib/services/database/types';
 import { MarkupToHtml } from '@joplin/renderer';
 import Setting from '@joplin/lib/models/Setting';
 import Resource from '@joplin/lib/models/Resource';
-import FileModel from '../../models/FileModel';
+import FileModel, { FileContent, FileWithContent, LoadContentHandlerEvent, SaveContentHandlerEvent } from '../../models/FileModel';
 import { ErrorNotFound } from '../../utils/errors';
 import BaseApplication from '../../services/BaseApplication';
 import { formatDateTime } from '../../utils/time';
@@ -63,6 +63,24 @@ export default class Application extends BaseApplication {
 		// resources.
 		BaseItem.loadClass('Note', Note);
 		BaseItem.loadClass('Resource', Resource);
+
+		this.fileModel_saveContentHandler = this.fileModel_saveContentHandler.bind(this);
+		this.fileModel_loadContentHandler = this.fileModel_loadContentHandler.bind(this);
+		FileModel.registerSaveContentHandler(this.fileModel_saveContentHandler);
+		FileModel.registerLoadContentHandler(this.fileModel_loadContentHandler);
+	}
+
+	// TODO: also do loadContentHandler
+
+	private async fileModel_saveContentHandler(event: SaveContentHandlerEvent): Promise<File> | null {
+		const fileContent = await this.fileToJoplinItem({ file: event.file, content: event.content });
+		if (!fileContent) return null;
+		const result = await event.models.joplinFileContent({ userId: event.file.owner_id }).saveFileAndContent(event.file, fileContent, event.options);
+		return result;
+	}
+
+	private async fileModel_loadContentHandler(event: LoadContentHandlerEvent): Promise<any> | null {
+		return this.joplinItemToFile(event.content);
 	}
 
 	public async localFileFromUrl(url: string): Promise<string> {
@@ -79,7 +97,7 @@ export default class Application extends BaseApplication {
 		return `${itemId}.md`;
 	}
 
-	private async itemMetadataFile(parentId: Uuid, itemId: string): Promise<File | null> {
+	private async itemMetadataFile(parentId: Uuid, itemId: string): Promise<FileWithContent | null> {
 		const file = await this.models.file().fileByName(parentId, this.itemIdFilename(itemId), { skipPermissionCheck: true });
 		if (!file) {
 			// We don't throw an error because it can happen if the note
@@ -90,9 +108,8 @@ export default class Application extends BaseApplication {
 		return this.models.file().loadWithContent(file.id, { skipPermissionCheck: true });
 	}
 
-	private async unserializeItem(file: File): Promise<any> {
-		const content = file.content.toString();
-		return BaseItem.unserialize(content);
+	private async unserializeItem(fileContent: FileContent): Promise<any> {
+		return BaseItem.unserialize(fileContent.toString());
 	}
 
 	private async resourceInfos(linkedItemInfos: LinkedItemInfos): Promise<ResourceInfos> {
@@ -119,12 +136,12 @@ export default class Application extends BaseApplication {
 		const output: LinkedItemInfos = {};
 
 		for (const itemId of itemIds) {
-			const itemFile = await this.itemMetadataFile(noteFileParentId, itemId);
-			if (!itemFile) continue;
+			const itemFileWithContent = await this.itemMetadataFile(noteFileParentId, itemId);
+			if (!itemFileWithContent) continue;
 
 			output[itemId] = {
-				item: await this.unserializeItem(itemFile),
-				file: itemFile,
+				item: await this.unserializeItem(itemFileWithContent.content),
+				file: itemFileWithContent.file,
 			};
 		}
 
@@ -138,7 +155,7 @@ export default class Application extends BaseApplication {
 		return fileModel.pathToFile(dirPath);
 	}
 
-	private async itemFile(fileModel: FileModel, parentId: Uuid, itemType: ModelType, itemId: string): Promise<File> {
+	private async itemFile(fileModel: FileModel, parentId: Uuid, itemType: ModelType, itemId: string): Promise<FileWithContent> {
 		let output: File = null;
 
 		if (itemType === ModelType.Resource) {
@@ -153,9 +170,9 @@ export default class Application extends BaseApplication {
 		return fileModel.loadWithContent(output.id);
 	}
 
-	private async renderResource(file: File): Promise<FileViewerResponse> {
+	private async renderResource(file: File, content: FileContent): Promise<FileViewerResponse> {
 		return {
-			body: file.content,
+			body: content,
 			mime: file.mime_type,
 			size: file.size,
 		};
@@ -221,30 +238,28 @@ export default class Application extends BaseApplication {
 		};
 	}
 
-	public async renderFile(file: File, share: Share, query: Record<string, any>): Promise<FileViewerResponse> {
+	public async renderFile(fileWithContent: FileWithContent, share: Share, query: Record<string, any>): Promise<FileViewerResponse> {
+		const { file, content } = fileWithContent;
+
 		const fileModel = this.models.file({ userId: file.owner_id });
 
-		const rootNote: NoteEntity = await this.unserializeItem(file);
+		const rootNote: NoteEntity = await this.unserializeItem(content);
 		const linkedItemInfos = await this.noteLinkedItemInfos(file.parent_id, rootNote);
 		const resourceInfos = await this.resourceInfos(linkedItemInfos);
 
 		const fileToRender = {
 			file: file,
+			content: null as any,
 			itemId: rootNote.id,
 		};
 
 		if (query.resource_id) {
-			fileToRender.file = await this.itemFile(fileModel, file.parent_id, ModelType.Resource, query.resource_id);
+			const withContent = await this.itemFile(fileModel, file.parent_id, ModelType.Resource, query.resource_id);
+			fileToRender.file = withContent.file;
+			fileToRender.content = withContent.content;
 			fileToRender.itemId = query.resource_id;
 		}
 
-		// No longer supported - need to decide what to do about note links.
-
-		// if (query.note_id) {
-		// 	fileToRender.file = await this.itemFile(fileModel, file.parent_id, ModelType.Note, query.note_id);
-		// 	fileToRender.itemId = query.note_id;
-		// }
-
 		if (fileToRender.file !== file && !linkedItemInfos[fileToRender.itemId]) {
 			throw new ErrorNotFound(`Item "${fileToRender.itemId}" does not belong to this note`);
 		}
@@ -253,7 +268,7 @@ export default class Application extends BaseApplication {
 		const itemType: ModelType = itemToRender.type_;
 
 		if (itemType === ModelType.Resource) {
-			return this.renderResource(fileToRender.file);
+			return this.renderResource(fileToRender.file, fileToRender.content);
 		} else if (itemType === ModelType.Note) {
 			return this.renderNote(share, itemToRender, resourceInfos, linkedItemInfos);
 		} else {
@@ -261,17 +276,49 @@ export default class Application extends BaseApplication {
 		}
 	}
 
-	public async isItemFile(file: File): Promise<boolean> {
-		if (file.mime_type !== 'text/markdown') return false;
+	public async fileToJoplinItem(fileWithContent: FileWithContent): Promise<JoplinFileContent> | null {
+		if (fileWithContent.file.mime_type !== 'text/markdown') return null;
 
 		try {
-			await this.unserializeItem(file);
+			const rawItem: any = await this.unserializeItem(fileWithContent.content);
+
+			const dbItem: JoplinFileContent = {
+				id: rawItem.id,
+				parent_id: rawItem.parent_id || '',
+				encryption_applied: rawItem.encryption_applied || 0,
+				type: rawItem.type_,
+				updated_time: rawItem.updated_time,
+				created_time: rawItem.created_time,
+				owner_id: fileWithContent.file.owner_id,
+			};
+
+			delete rawItem.id;
+			delete rawItem.parent_id;
+			delete rawItem.encryption_applied;
+			delete rawItem.type_;
+			delete rawItem.updated_time;
+			delete rawItem.created_time;
+
+			dbItem.content = JSON.stringify(rawItem);
+
+			return dbItem;
 		} catch (error) {
 			// No need to log - it means it's not a note file
-			return false;
+			return null;
 		}
+	}
 
-		return true;
+	public async joplinItemToFile(fileContent: JoplinFileContent): Promise<string> {
+		const item = JSON.parse(fileContent.content);
+		item.id = fileContent.id;
+		item.type_ = fileContent.type;
+		item.parent_id = fileContent.parent_id;
+		item.encryption_applied = fileContent.encryption_applied;
+		item.updated_time = fileContent.updated_time;
+		item.created_time = fileContent.created_time;
+
+		const ItemClass = BaseItem.itemClass(item);
+		return ItemClass.serialize(item);
 	}
 
 }

packages/server/src/apps/joplin/JoplinFileContentModel.ts

@@ -0,0 +1,30 @@
+import { File, FileContentType, JoplinFileContent } from '../../db';
+import BaseModel, { SaveOptions } from '../../models/BaseModel';
+
+export default class JoplinFileContentModel extends BaseModel<JoplinFileContent> {
+
+	public get tableName(): string {
+		return 'joplin_file_contents';
+	}
+
+	protected autoTimestampEnabled(): boolean {
+		return false;
+	}
+
+	public async saveFileAndContent(file: File, joplinFileContent: JoplinFileContent, options: SaveOptions): Promise<File> {
+		let modFile: File = { ...file };
+
+		await this.withTransaction(async () => {
+			// For now will always overwrite the complete content with the new one
+			await this.delete(joplinFileContent.id, { allowNoOp: true });
+			await this.save(joplinFileContent, { isNew: true });
+			delete modFile.content;
+			modFile.content_id = joplinFileContent.id;
+			modFile.content_type = FileContentType.JoplinItem;
+			modFile = await this.models().file({ userId: this.userId }).save(modFile, options);
+		}, 'saveJoplinFileContent');
+
+		return modFile;
+	}
+
+}

packages/server/src/db.ts

@@ -211,6 +211,11 @@ export enum ChangeType {
 	Delete = 3,
 }
 
+export enum FileContentType {
+	Any = 1,
+	JoplinItem = 2,
+}
+
 export function changeTypeToString(t: ChangeType): string {
 	if (t === ChangeType.Create) return 'create';
 	if (t === ChangeType.Update) return 'update';
@@ -275,6 +280,9 @@ export interface File extends WithDates, WithUuid {
 	is_directory?: number;
 	is_root?: number;
 	parent_id?: Uuid;
+	source_file_id?: Uuid;
+	content_type?: FileContentType;
+	content_id?: Uuid;
 }
 
 export interface Change extends WithDates, WithUuid {
@@ -307,6 +315,23 @@ export interface Share extends WithDates, WithUuid {
 	type?: ShareType;
 }
 
+export interface ShareUser extends WithDates, WithUuid {
+	share_id?: Uuid;
+	user_id?: Uuid;
+	is_accepted?: number;
+}
+
+export interface JoplinFileContent {
+	id?: Uuid;
+	owner_id?: Uuid;
+	parent_id?: Uuid;
+	type?: number;
+	updated_time?: string;
+	created_time?: string;
+	encryption_applied?: number;
+	content?: any;
+}
+
 export const databaseSchema: DatabaseTables = {
 	users: {
 		id: { type: 'string' },
@@ -346,6 +371,9 @@ export const databaseSchema: DatabaseTables = {
 		parent_id: { type: 'string' },
 		updated_time: { type: 'string' },
 		created_time: { type: 'string' },
+		source_file_id: { type: 'string' },
+		content_type: { type: 'number' },
+		content_id: { type: 'string' },
 	},
 	changes: {
 		counter: { type: 'number' },
@@ -385,5 +413,23 @@ export const databaseSchema: DatabaseTables = {
 		updated_time: { type: 'string' },
 		created_time: { type: 'string' },
 	},
+	share_users: {
+		id: { type: 'string' },
+		share_id: { type: 'string' },
+		user_id: { type: 'string' },
+		is_accepted: { type: 'number' },
+		updated_time: { type: 'string' },
+		created_time: { type: 'string' },
+	},
+	joplin_file_contents: {
+		id: { type: 'string' },
+		owner_id: { type: 'string' },
+		parent_id: { type: 'string' },
+		type: { type: 'number' },
+		updated_time: { type: 'string' },
+		created_time: { type: 'string' },
+		encryption_applied: { type: 'number' },
+		content: { type: 'any' },
+	},
 };
 // AUTO-GENERATED-TYPES

packages/server/src/middleware/routeHandler.ts

@@ -1,7 +1,6 @@
 import routes from '../routes/routes';
-import { ErrorForbidden, ErrorNotFound } from '../utils/errors';
-import { routeResponseFormat, findMatchingRoute, Response, RouteResponseFormat, MatchedRoute } from '../utils/routeUtils';
-import { AppContext, Env, HttpMethod } from '../utils/types';
+import { routeResponseFormat, Response, RouteResponseFormat, execRequest } from '../utils/routeUtils';
+import { AppContext, Env } from '../utils/types';
 import MustacheService, { isView, View } from '../services/MustacheService';
 import config from '../config';
 
@@ -16,38 +15,21 @@ function mustache(): MustacheService {
 export default async function(ctx: AppContext) {
 	ctx.appLogger().info(`${ctx.request.method} ${ctx.path}`);
 
-	const match: MatchedRoute = null;
-
 	try {
-		const match = findMatchingRoute(ctx.path, routes);
+		const responseObject = await execRequest(routes, ctx);
 
-		if (match) {
-			let responseObject = null;
-
-			const routeHandler = match.route.findEndPoint(ctx.request.method as HttpMethod, match.subPath.schema);
-
-			// This is a generic catch-all for all private end points - if we
-			// couldn't get a valid session, we exit now. Individual end points
-			// might have additional permission checks depending on the action.
-			if (!match.route.public && !ctx.owner) throw new ErrorForbidden();
-
-			responseObject = await routeHandler(match.subPath, ctx);
-
-			if (responseObject instanceof Response) {
-				ctx.response = responseObject.response;
-			} else if (isView(responseObject)) {
-				ctx.response.status = 200;
-				ctx.response.body = await mustache().renderView(responseObject, {
-					notifications: ctx.notifications || [],
-					hasNotifications: !!ctx.notifications && !!ctx.notifications.length,
-					owner: ctx.owner,
-				});
-			} else {
-				ctx.response.status = 200;
-				ctx.response.body = [undefined, null].includes(responseObject) ? '' : responseObject;
-			}
+		if (responseObject instanceof Response) {
+			ctx.response = responseObject.response;
+		} else if (isView(responseObject)) {
+			ctx.response.status = 200;
+			ctx.response.body = await mustache().renderView(responseObject, {
+				notifications: ctx.notifications || [],
+				hasNotifications: !!ctx.notifications && !!ctx.notifications.length,
+				owner: ctx.owner,
+			});
 		} else {
-			throw new ErrorNotFound();
+			ctx.response.status = 200;
+			ctx.response.body = [undefined, null].includes(responseObject) ? '' : responseObject;
 		}
 	} catch (error) {
 		if (error.httpCode >= 400 && error.httpCode < 500) {
@@ -58,7 +40,7 @@ export default async function(ctx: AppContext) {
 
 		ctx.response.status = error.httpCode ? error.httpCode : 500;
 
-		const responseFormat = routeResponseFormat(match, ctx);
+		const responseFormat = routeResponseFormat(ctx);
 
 		if (responseFormat === RouteResponseFormat.Html) {
 			ctx.response.set('Content-Type', 'text/html');

packages/server/src/migrations/20210201143859_app_share.ts

@@ -0,0 +1,34 @@
+import * as Knex from 'knex';
+import { DbConnection } from '../db';
+
+export async function up(db: DbConnection): Promise<any> {
+	await db.schema.createTable('share_users', function(table: Knex.CreateTableBuilder) {
+		table.string('id', 32).unique().primary().notNullable();
+		table.string('share_id', 32).notNullable();
+		table.string('user_id', 32).notNullable();
+		table.integer('is_accepted').defaultTo(0).notNullable();
+		table.bigInteger('updated_time').notNullable();
+		table.bigInteger('created_time').notNullable();
+	});
+
+	await db.schema.alterTable('share_users', function(table: Knex.CreateTableBuilder) {
+		table.unique(['share_id', 'user_id']);
+	});
+
+	await db.schema.table('files', function(table: Knex.CreateTableBuilder) {
+		table.string('source_file_id', 32).defaultTo('').notNullable();
+	});
+
+	await db.schema.alterTable('files', function(table: Knex.CreateTableBuilder) {
+		table.index(['owner_id']);
+		table.index(['source_file_id']);
+	});
+
+	await db.schema.alterTable('changes', function(table: Knex.CreateTableBuilder) {
+		table.index(['item_id']);
+	});
+}
+
+export async function down(db: DbConnection): Promise<any> {
+	await db.schema.dropTable('share_users');
+}

packages/server/src/migrations/20210321112923_joplin_items.ts

@@ -0,0 +1,24 @@
+import * as Knex from 'knex';
+import { DbConnection } from '../db';
+
+export async function up(db: DbConnection): Promise<any> {
+	await db.schema.createTable('joplin_file_contents', function(table: Knex.CreateTableBuilder) {
+		table.string('id', 32).unique().primary().notNullable();
+		table.string('owner_id', 32).notNullable();
+		table.string('parent_id', 32).defaultTo('').notNullable();
+		table.integer('type', 2).notNullable();
+		table.bigInteger('updated_time').notNullable();
+		table.bigInteger('created_time').notNullable();
+		table.integer('encryption_applied', 1).notNullable();
+		table.binary('content').defaultTo('').notNullable();
+	});
+
+	await db.schema.alterTable('files', function(table: Knex.CreateTableBuilder) {
+		table.integer('content_type', 2).defaultTo(1).notNullable();
+		table.string('content_id', 32).defaultTo('').notNullable();
+	});
+}
+
+export async function down(db: DbConnection): Promise<any> {
+	await db.schema.dropTable('joplin_items');
+}

packages/server/src/models/BaseModel.ts

@@ -3,9 +3,11 @@ import TransactionHandler from '../utils/TransactionHandler';
 import uuidgen from '../utils/uuidgen';
 import { ErrorUnprocessableEntity, ErrorBadRequest } from '../utils/errors';
 import { Models } from './factory';
+import Applications from '../services/Applications';
 
 export interface ModelOptions {
 	userId?: string;
+	apps?: Applications;
 }
 
 export interface SaveOptions {
@@ -15,8 +17,13 @@ export interface SaveOptions {
 	trackChanges?: boolean;
 }
 
+export interface LoadOptions {
+	fields?: string[];
+}
+
 export interface DeleteOptions {
 	validationRules?: any;
+	allowNoOp?: boolean;
 }
 
 export interface ValidateOptions {
@@ -63,6 +70,10 @@ export default abstract class BaseModel<T> {
 		return this.options.userId;
 	}
 
+	protected get apps(): Applications {
+		return this.options.apps;
+	}
+
 	protected get db(): DbConnection {
 		if (this.transactionHandler_.activeTransaction) return this.transactionHandler_.activeTransaction;
 		return this.db_;
@@ -91,7 +102,7 @@ export default abstract class BaseModel<T> {
 		return true;
 	}
 
-	protected hasDateProperties(): boolean {
+	protected autoTimestampEnabled(): boolean {
 		return true;
 	}
 
@@ -121,7 +132,7 @@ export default abstract class BaseModel<T> {
 	//
 	// The `name` argument is only for debugging, so that any stuck transaction
 	// can be more easily identified.
-	protected async withTransaction(fn: Function, name: string = null): Promise<void> {
+	protected async withTransaction<T>(fn: Function, name: string = null): Promise<T> {
 		const debugTransaction = false;
 
 		const debugTimerId = debugTransaction ? setTimeout(() => {
@@ -132,8 +143,10 @@ export default abstract class BaseModel<T> {
 
 		if (debugTransaction) console.info('START', name, txIndex);
 
+		let output: T = null;
+
 		try {
-			await fn();
+			output = await fn();
 		} catch (error) {
 			await this.transactionHandler_.rollback(txIndex);
 
@@ -151,6 +164,7 @@ export default abstract class BaseModel<T> {
 		}
 
 		await this.transactionHandler_.commit(txIndex);
+		return output;
 	}
 
 	public async all(): Promise<T[]> {
@@ -182,6 +196,7 @@ export default abstract class BaseModel<T> {
 	protected async isNew(object: T, options: SaveOptions): Promise<boolean> {
 		if (options.isNew === false) return false;
 		if (options.isNew === true) return true;
+		if ('id' in object && !(object as WithUuid).id) throw new Error('ID cannot be undefined or null');
 		return !(object as WithUuid).id;
 	}
 
@@ -218,7 +233,7 @@ export default abstract class BaseModel<T> {
 			(toSave as WithUuid).id = uuidgen();
 		}
 
-		if (this.hasDateProperties()) {
+		if (this.autoTimestampEnabled()) {
 			const timestamp = Date.now();
 			if (isNew) {
 				(toSave as WithDates).created_time = timestamp;
@@ -249,18 +264,18 @@ export default abstract class BaseModel<T> {
 		return toSave;
 	}
 
-	public async loadByIds(ids: string[]): Promise<T[]> {
+	public async loadByIds(ids: string[], options: LoadOptions = {}): Promise<T[]> {
 		if (!ids.length) return [];
-		return this.db(this.tableName).select(this.defaultFields).whereIn('id', ids);
+		return this.db(this.tableName).select(options.fields || this.defaultFields).whereIn('id', ids);
 	}
 
-	public async load(id: string): Promise<T> {
+	public async load(id: string, options: LoadOptions = {}): Promise<T> {
 		if (!id) throw new Error('id cannot be empty');
 
-		return this.db(this.tableName).select(this.defaultFields).where({ id: id }).first();
+		return this.db(this.tableName).select(options.fields || this.defaultFields).where({ id: id }).first();
 	}
 
-	public async delete(id: string | string[]): Promise<void> {
+	public async delete(id: string | string[], options: DeleteOptions = {}): Promise<void> {
 		if (!id) throw new Error('id cannot be empty');
 
 		const ids = typeof id === 'string' ? [id] : id;
@@ -281,7 +296,7 @@ export default abstract class BaseModel<T> {
 			}
 
 			const deletedCount = await query.del();
-			if (deletedCount !== ids.length) throw new Error(`${ids.length} row(s) should have been deleted by ${deletedCount} row(s) were deleted`);
+			if (!options.allowNoOp && deletedCount !== ids.length) throw new Error(`${ids.length} row(s) should have been deleted by ${deletedCount} row(s) were deleted`);
 
 			if (trackChanges) {
 				for (const item of itemsWithParentIds) await this.handleChangeTracking({}, item, ChangeType.Delete);

packages/server/src/models/ChangeModel.ts

@@ -9,6 +9,10 @@ export interface ChangeWithItem {
 	type: ChangeType;
 }
 
+export interface ChangeWithDestFile extends Change {
+	dest_file_id: Uuid;
+}
+
 export interface PaginatedChanges extends PaginatedResults {
 	items: ChangeWithItem[];
 }
@@ -88,27 +92,82 @@ export default class ChangeModel extends BaseModel<Change> {
 		const directory = await fileModel.load(dirId);
 		if (!directory.is_directory) throw new ErrorUnprocessableEntity(`Item with id "${dirId}" is not a directory.`);
 
-		// Rather than query the changes, then use JS to compress them, it might
-		// be possible to do both in one query.
-		// https://stackoverflow.com/questions/65348794
-		const query = this.db(this.tableName)
+		// Retrieves the IDs of all the files that have been shared with the
+		// current user.
+		const linkedFilesQuery = this
+			.db('files')
+			.select('source_file_id')
+			.where('source_file_id', '!=', '')
+			.andWhere('parent_id', '=', dirId)
+			.andWhere('owner_id', '=', this.userId);
+
+		// Retrieves all the changes for the files that belong to the current
+		// user.
+		const ownChangesQuery = this.db(this.tableName)
 			.select([
 				'counter',
 				'id',
 				'item_id',
 				'item_name',
 				'type',
+				this.db.raw('"" as dest_file_id'),
 			])
-			.where('parent_id', dirId)
-			.orderBy('counter', 'asc')
-			.limit(pagination.limit);
+			.where('parent_id', dirId);
 
+		// Retrieves all the changes for the files that have been shared with
+		// this user.
+		//
+		// Each row will have an additional "dest_file_id" property that points
+		// to the destination of the link. For example:
+		//
+		// - User 1 shares a file with ID 123 with user 2.
+		// - User 2 get a new file with ID 456 that links to file 123.
+		// - User 1 changes file 123
+		// - When user 2 retrieves all the changes, they'll get a change for
+		//   item_id = 123, and dest_file_id = 456
+		//
+		// We need this dest_file_id because when sending the list of files, we
+		// want to send back metadata for file 456, and not 123, as that belongs
+		// to a different user.
+		const sharedChangesQuery = this.db(this.tableName)
+			.select([
+				'counter',
+				'changes.id',
+				'item_id',
+				'item_name',
+				'type',
+				'files.id as dest_file_id',
+			])
+			.join('files', 'changes.item_id', 'files.source_file_id')
+			.whereIn('changes.item_id', linkedFilesQuery);
+
+		// If a cursor was provided, apply it to both queries.
 		if (changeAtCursor) {
-			void query.where('counter', '>', changeAtCursor.counter);
+			void ownChangesQuery.where('counter', '>', changeAtCursor.counter);
+			void sharedChangesQuery.where('counter', '>', changeAtCursor.counter);
 		}
 
-		const changes: Change[] = await query;
+		// This will give the list of all changes for shared and not shared
+		// files for the provided directory ID. Knexjs TypeScript support seems
+		// to be buggy here as it reports that will return `any[][]` so we fix
+		// that by forcing `any[]`
+		const changesWithDestFile: ChangeWithDestFile[] = await ownChangesQuery
+			.union(sharedChangesQuery)
+			.orderBy('counter', 'asc')
+			.limit(pagination.limit) as any[];
+
+		// Maps dest_file_id to item_id and then the rest of the code can just
+		// work without having to check if it's a shared file or not.
+		const changes = changesWithDestFile.map(c => {
+			if (c.dest_file_id) {
+				return { ...c, item_id: c.dest_file_id };
+			} else {
+				return c;
+			}
+		});
+
 		const compressedChanges = this.compressChanges(changes);
+
 		const changeWithItems = await this.loadChangeItems(compressedChanges);
 
 		return {
@@ -122,8 +181,12 @@ export default class ChangeModel extends BaseModel<Change> {
 
 	private async loadChangeItems(changes: Change[]): Promise<ChangeWithItem[]> {
 		const itemIds = changes.map(c => c.item_id);
-		const fileModel = this.models().file({ userId: this.userId });
-		const items: File[] = await fileModel.loadByIds(itemIds);
+
+		// We skip permission check here because, when a file is shared, we need
+		// to fetch files that don't belong to the current user. This check
+		// would not be needed anyway because the change items are generated in
+		// a context where permissions have already been checked.
+		const items: File[] = await this.models().file().loadByIds(itemIds, { skipPermissionCheck: true });
 
 		const output: ChangeWithItem[] = [];
 
@@ -161,7 +224,8 @@ export default class ChangeModel extends BaseModel<Change> {
 		const itemChanges: Record<Uuid, Change> = {};
 
 		for (const change of changes) {
-			const previous = itemChanges[change.item_id];
+			const itemId = change.item_id;
+			const previous = itemChanges[itemId];
 
 			if (previous) {
 				// create - update => create
@@ -174,22 +238,22 @@ export default class ChangeModel extends BaseModel<Change> {
 				}
 
 				if (previous.type === ChangeType.Create && change.type === ChangeType.Delete) {
-					delete itemChanges[change.item_id];
+					delete itemChanges[itemId];
 				}
 
 				if (previous.type === ChangeType.Update && change.type === ChangeType.Update) {
-					itemChanges[change.item_id] = change;
+					itemChanges[itemId] = change;
 				}
 
 				if (previous.type === ChangeType.Update && change.type === ChangeType.Delete) {
-					itemChanges[change.item_id] = change;
+					itemChanges[itemId] = change;
 				}
 			} else {
-				itemChanges[change.item_id] = change;
+				itemChanges[itemId] = change;
 			}
 		}
 
-		const output = [];
+		const output: Change[] = [];
 
 		for (const itemId in itemChanges) {
 			output.push(itemChanges[itemId]);

packages/server/src/models/FileModel.test.ts

@@ -40,8 +40,13 @@ describe('FileModel', function() {
 			.concat(Object.keys(tree.folder2))
 			.concat(Object.keys(tree.folder3));
 
+		const allFiles = await fileModel.all();
+		const loadByName = (name: string) => {
+			return allFiles.find(f => f.name === name);
+		};
+
 		for (const t of testCases) {
-			const file: File = await fileModel.loadByName(t);
+			const file: File = await loadByName(t);
 			const path = await fileModel.itemFullPath(file);
 			const fileBackId: string = await fileModel.pathToFileId(path);
 			expect(file.id).toBe(fileBackId);

packages/server/src/models/FileModel.ts

@@ -1,10 +1,11 @@
 import BaseModel, { ValidateOptions, SaveOptions, DeleteOptions } from './BaseModel';
-import { File, ItemType, databaseSchema, Uuid } from '../db';
+import { File, ItemType, databaseSchema, Uuid, FileContentType } from '../db';
 import { ErrorForbidden, ErrorUnprocessableEntity, ErrorNotFound, ErrorBadRequest, ErrorConflict } from '../utils/errors';
 import uuidgen from '../utils/uuidgen';
 import { splitItemPath, filePathInfo } from '../utils/routeUtils';
 import { paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination';
 import { setQueryParameters } from '../utils/urlUtils';
+import { Models } from './factory';
 
 const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
 
@@ -12,6 +13,33 @@ const nodeEnv = process.env.NODE_ENV || 'development';
 
 const removeTrailingColonsRegex = /^(:|)(.*?)(:|)$/;
 
+export type FileContent = any;
+
+export interface LoadContentHandlerEvent {
+	models: Models;
+	file: File;
+	content: FileContent;
+	options: LoadOptions;
+}
+
+export interface SaveContentHandlerEvent {
+	// We pass the models to the handler so that any db call is executed within
+	// the same context. It matters in particular for transactions, which needs
+	// to be executed with the same `db` connection.
+	models: Models;
+	file: File;
+	content: FileContent;
+	options: SaveOptions;
+}
+
+export interface FileWithContent {
+	file: File;
+	content: FileContent;
+}
+
+export type LoadContentHandler = (event: LoadContentHandlerEvent)=> Promise<any> | null;
+export type SaveContentHandler = (event: SaveContentHandlerEvent)=> Promise<File> | null;
+
 export interface PaginatedFiles extends PaginatedResults {
 	items: File[];
 }
@@ -23,11 +51,15 @@ export interface PathToFileOptions {
 
 export interface LoadOptions {
 	skipPermissionCheck?: boolean;
+	fields?: string[];
+	skipFollowLinks?: boolean;
 }
 
 export default class FileModel extends BaseModel<File> {
 
 	private readonly reservedCharacters = ['/', '\\', '*', '<', '>', '?', ':', '|', '#', '%'];
+	private static loadContentHandlers_: LoadContentHandler[] = [];
+	private static saveContentHandlers_: SaveContentHandler[] = [];
 
 	protected get tableName(): string {
 		return 'files';
@@ -212,16 +244,19 @@ export default class FileModel extends BaseModel<File> {
 	}
 
 	public async fileByName(parentId: string, name: string, options: LoadOptions = {}): Promise<File> {
-		const file = await this.db<File>(this.tableName).select(...this.defaultFields).where({
-			parent_id: parentId,
-			name: name,
-		}).first();
+		const file = await this.db<File>(this.tableName)
+			.select(...this.defaultFields)
+			.where({
+				parent_id: parentId,
+				name: name,
+			})
+			.first();
 
 		if (!file) return null;
 
 		if (!options.skipPermissionCheck) await this.checkCanReadPermissions(file);
 
-		return file;
+		return this.processFileLink(file);
 	}
 
 	protected async validate(object: File, options: ValidateOptions = {}): Promise<File> {
@@ -269,8 +304,8 @@ export default class FileModel extends BaseModel<File> {
 
 		if ('name' in file && !file.is_root) {
 			const existingFile = await this.fileByName(parentId, file.name);
-			if (existingFile && options.isNew) throw new ErrorConflict(`Already a file with name "${file.name}"`);
-			if (existingFile && file.id === existingFile.id) throw new ErrorConflict(`Already a file with name "${file.name}"`);
+			if (existingFile && options.isNew) throw new ErrorConflict(`Already a file with name "${file.name}" (1)`);
+			if (existingFile && file.id !== existingFile.id) throw new ErrorConflict(`Already a file with name "${file.name}" (2)`);
 		}
 
 		if ('name' in file) {
@@ -324,7 +359,10 @@ export default class FileModel extends BaseModel<File> {
 
 		if (!files.length || !files[0]) throw new ErrorNotFound();
 
-		const fileIds = files.map(f => f.id);
+		const fileIds = files.map(f => {
+			if (!f.id) throw new Error('One of the file is missing an ID');
+			return f.id;
+		});
 
 		const permissionModel = this.models().permission();
 		const permissionGrantedMap = await permissionModel[methodName](fileIds, this.userId);
@@ -334,7 +372,7 @@ export default class FileModel extends BaseModel<File> {
 		}
 
 		for (const fileId in permissionGrantedMap) {
-			if (!permissionGrantedMap[fileId]) throw new ErrorForbidden(`No read access to: ${fileId}`);
+			if (!permissionGrantedMap[fileId]) throw new ErrorForbidden(`No "${methodName}" access to: ${fileId}`);
 		}
 	}
 
@@ -384,46 +422,162 @@ export default class FileModel extends BaseModel<File> {
 		return output;
 	}
 
-	// Mostly makes sense for testing/debugging because the filename would
-	// have to globally unique, which is not a requirement.
-	public async loadByName(name: string, options: LoadOptions = {}): Promise<File> {
-		const file: File = await this.db(this.tableName)
-			.select(this.defaultFields)
-			.where({ name: name })
-			.andWhere({ owner_id: this.userId })
-			.first();
-		if (!file) throw new ErrorNotFound(`No such file: ${name}`);
-		if (!options.skipPermissionCheck) await this.checkCanReadPermissions(file);
-		return file;
+	private async processFileLink(file: File): Promise<File> {
+		const files = await this.processFileLinks([file]);
+		return files[0];
 	}
 
-	public async loadWithContent(id: string, options: LoadOptions = {}): Promise<any> {
+	// If the file is a link to another file, the content of the source if
+	// assigned to the content of the destination. The updated_time property is
+	// also set to the most recent one among the source and the dest.
+	private async processFileLinks(files: File[]): Promise<File[]> {
+		const sourceFileIds: Uuid[] = [];
+
+		for (const file of files) {
+			if (!('source_file_id' in file)) throw new Error('Cannot process file links without a "source_file_id" property');
+			if (!file.source_file_id) continue;
+			sourceFileIds.push(file.source_file_id);
+		}
+
+		if (!sourceFileIds.length) return files;
+
+		const fields = Object.keys(files[0]);
+		const sourceFiles = await this.loadByIds(sourceFileIds, {
+			fields,
+			skipPermissionCheck: true,
+			// Current we don't follow links more than one level deep. In
+			// practice it means that if a user tries to share a note that has
+			// been shared with them, it will not work. Instead they'll have to
+			// make a copy of that note and share that. Anything else would
+			// probably be too complex to make any sense in terms of UI.
+			skipFollowLinks: true,
+		});
+
+		const modFiles = files.slice();
+		for (let i = 0; i < modFiles.length; i++) {
+			const file = modFiles[i];
+			if (!file.source_file_id) continue;
+			const sourceFile = sourceFiles.find(f => f.id === file.source_file_id);
+			if (!sourceFile) {
+				throw new Error(`File is linked to a file that no longer exists: ${file.id} => ${file.source_file_id}`);
+			}
+
+			const modFile = { ...file };
+
+			if ('updated_time' in modFile) modFile.updated_time = Math.max(sourceFile.updated_time, file.updated_time);
+			if ('content' in modFile) modFile.content = sourceFile.content;
+
+			modFiles[i] = modFile;
+		}
+
+		return modFiles;
+	}
+
+	public async content(file: string | File, serialized: boolean = true): Promise<FileContent> {
+		if (typeof file === 'string') {
+			file = (await this.db<File>(this.tableName).select(['content_id', 'content_type', 'content']).where({ id: file }).first()) as File;
+		}
+
+		if (file.content_type === FileContentType.Any) {
+			return file.content;
+		} else if (file.content_type === FileContentType.JoplinItem) {
+			const content = await this.models().joplinFileContent().load(file.content_id);
+
+			if (serialized) {
+				const unserialized = await FileModel.processLoadContentHandlers({
+					models: this.models(),
+					file: file,
+					content: content,
+					options: {},
+				});
+
+				if (unserialized === null) throw new Error(`No handler to unserialize content for file ${file.id}`);
+
+				return unserialized;
+			} else {
+				return content;
+			}
+		} else {
+			throw new Error(`Unsupported content type: ${file.content_type}`);
+		}
+	}
+
+	public async loadWithContent(id: string, options: LoadOptions = {}): Promise<FileWithContent | null> {
 		const file: File = await this.db<File>(this.tableName).select('*').where({ id: id }).first();
 		if (!file) return null;
 		if (!options.skipPermissionCheck) await this.checkCanReadPermissions(file);
-		return file;
+
+		return {
+			file,
+			content: await this.content(file),
+		};
 	}
 
 	public async loadByIds(ids: string[], options: LoadOptions = {}): Promise<File[]> {
-		const files: File[] = await super.loadByIds(ids);
+		const files: File[] = await super.loadByIds(ids, options);
 		if (!files.length) return [];
 		if (!options.skipPermissionCheck) await this.checkCanReadPermissions(files);
-		return files;
+		return options.skipFollowLinks ? files : this.processFileLinks(files);
 	}
 
 	public async load(id: string, options: LoadOptions = {}): Promise<File> {
-		const file: File = await super.load(id);
-		if (!file) return null;
-		if (!options.skipPermissionCheck) await this.checkCanReadPermissions(file);
-		return file;
+		const files = await this.loadByIds([id], options);
+		return files.length ? files[0] : null;
+	}
+
+	public static registerSaveContentHandler(handler: SaveContentHandler) {
+		this.saveContentHandlers_.push(handler);
+	}
+
+	public static registerLoadContentHandler(handler: LoadContentHandler) {
+		this.loadContentHandlers_.push(handler);
+	}
+
+	private static async processSaveContentHandlers(event: SaveContentHandlerEvent): Promise<File> | null {
+		for (const handler of this.saveContentHandlers_) {
+			const result = await handler(event);
+			if (result) return result;
+		}
+		return null;
+	}
+
+	private static async processLoadContentHandlers(event: LoadContentHandlerEvent): Promise<any> | null {
+		for (const handler of this.loadContentHandlers_) {
+			const result = await handler(event);
+			if (result) return result;
+		}
+		return null;
 	}
 
 	public async save(object: File, options: SaveOptions = {}): Promise<File> {
 		const isNew = await this.isNew(object, options);
 
 		const file: File = { ... object };
+		let sourceFile: File = null;
 
-		if ('content' in file) file.size = file.content ? file.content.byteLength : 0;
+		if ('content' in file) {
+			let sourceFileId: string = null;
+
+			if (file.id) {
+				if (!('source_file_id' in file)) throw new Error('source_file_id is required when setting the content');
+				sourceFileId = file.source_file_id;
+			}
+
+			const fileSize = file.content ? file.content.byteLength : 0;
+
+			if (sourceFileId) {
+				sourceFile = {
+					id: sourceFileId,
+					content: file.content,
+					size: fileSize,
+					source_file_id: '',
+				};
+
+				delete file.content;
+			} else {
+				file.size = file.content ? file.content.byteLength : 0;
+			}
+		}
 
 		if (isNew) {
 			if (!file.parent_id && !file.is_root) file.parent_id = await this.userRootFileId();
@@ -437,7 +591,20 @@ export default class FileModel extends BaseModel<File> {
 			file.owner_id = this.userId;
 		}
 
-		return super.save(file, options);
+		return this.withTransaction<File>(async () => {
+			if ('content' in sourceFile) {
+				/* const processedFile = */ await FileModel.processSaveContentHandlers({
+					models: this.models(),
+					file: sourceFile,
+					content: sourceFile.content,
+					options,
+				});
+
+				// if (processedFile) return processedFile;
+			}
+
+			return super.save(file, options);
+		});
 	}
 
 	public async childrenCount(id: string): Promise<number> {
@@ -450,7 +617,9 @@ export default class FileModel extends BaseModel<File> {
 	public async childrens(id: string, pagination: Pagination): Promise<PaginatedFiles> {
 		const parent = await this.load(id);
 		await this.checkCanReadPermissions(parent);
-		return paginateDbQuery(this.db(this.tableName).select(...this.defaultFields).where('parent_id', id), pagination);
+		const page = await paginateDbQuery(this.db(this.tableName).select(...this.defaultFields).where('parent_id', id), pagination);
+		page.items = await this.processFileLinks(page.items);
+		return page;
 	}
 
 	private async childrenIds(id: string): Promise<string[]> {

packages/server/src/models/ShareModel.test.ts

@@ -1,6 +1,6 @@
 import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, checkThrowAsync } from '../utils/testing/testUtils';
-import { ShareType } from '../db';
 import { ErrorBadRequest, ErrorNotFound } from '../utils/errors';
+import { ShareType } from '../db';
 
 describe('ShareModel', function() {
 
@@ -26,10 +26,10 @@ describe('ShareModel', function() {
 
 		let error = null;
 
-		error = await checkThrowAsync(async () => await models().share({ userId: user.id }).add(20 as ShareType, file.id));
+		error = await checkThrowAsync(async () => await models().share({ userId: user.id }).createShare(20 as ShareType, file.id));
 		expect(error instanceof ErrorBadRequest).toBe(true);
 
-		error = await checkThrowAsync(async () => await models().share({ userId: user.id }).add(ShareType.Link, 'doesntexist'));
+		error = await checkThrowAsync(async () => await models().share({ userId: user.id }).createShare(ShareType.Link, 'doesntexist'));
 		expect(error instanceof ErrorNotFound).toBe(true);
 	});
 

packages/server/src/models/ShareModel.ts

@@ -1,4 +1,4 @@
-import { Share, ShareType, Uuid } from '../db';
+import { File, Share, ShareType, Uuid } from '../db';
 import { ErrorBadRequest } from '../utils/errors';
 import { setQueryParameters } from '../utils/urlUtils';
 import BaseModel, { ValidateOptions } from './BaseModel';
@@ -15,12 +15,14 @@ export default class ShareModel extends BaseModel<Share> {
 		return share;
 	}
 
-	public async add(type: ShareType, path: string): Promise<Share> {
-		const fileId: Uuid = await this.models().file({ userId: this.userId }).pathToFileId(path);
+	public async createShare(shareType: ShareType, path: string): Promise<Share> {
+		const file: File = await this.models().file({ userId: this.userId }).pathToFile(path);
+
+		if (file.source_file_id) throw new ErrorBadRequest('A shared file cannot be shared again');
 
 		const toSave: Share = {
-			type: type,
-			file_id: fileId,
+			type: shareType,
+			file_id: file.id,
 			owner_id: this.userId,
 		};
 

packages/server/src/models/ShareUserModel.test.ts

@@ -0,0 +1,39 @@
+import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createFile } from '../utils/testing/testUtils';
+import { shareWithUserAndAccept } from '../utils/testing/shareApiUtils';
+
+describe('ShareUserModel', function() {
+
+	beforeAll(async () => {
+		await beforeAllDb('ShareUserModel');
+	});
+
+	afterAll(async () => {
+		await afterAllTests();
+	});
+
+	beforeEach(async () => {
+		await beforeEachDb();
+	});
+
+	test('should get the list of linked user IDs', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		const { user: user3, session: session3 } = await createUserAndSession(3);
+
+		const file1a = await createFile(user1.id, 'root:/test1a.txt:', 'test1a');
+		const file1b = await createFile(user1.id, 'root:/test1b.txt:', 'test1b');
+		const file2 = await createFile(user2.id, 'root:/test2.txt:', 'test2');
+
+		await shareWithUserAndAccept(session1.id, user1, session3.id, user3, file1a);
+		await shareWithUserAndAccept(session1.id, user1, session3.id, user3, file1b);
+		await shareWithUserAndAccept(session2.id, user1, session3.id, user3, file2);
+
+		const userIds = await models().shareUser({ userId: user3.id }).linkedUserIds();
+		userIds.sort();
+		const expectedUserIds = [user1.id, user2.id];
+		expectedUserIds.sort();
+
+		expect(userIds).toEqual(expectedUserIds);
+	});
+
+});

packages/server/src/models/ShareUserModel.ts

@@ -0,0 +1,77 @@
+import { File, ShareUser, Uuid } from '../db';
+import { ErrorNotFound } from '../utils/errors';
+import BaseModel from './BaseModel';
+
+export default class ShareUserModel extends BaseModel<ShareUser> {
+
+	public get tableName(): string {
+		return 'share_users';
+	}
+
+	public async loadByShareIdAndUser(shareId: Uuid, userId: Uuid): Promise<ShareUser> {
+		const link: ShareUser = {
+			share_id: shareId,
+			user_id: userId,
+		};
+
+		return this.db(this.tableName).where(link).first();
+	}
+
+	public async addByEmail(shareId: Uuid, userEmail: string): Promise<ShareUser> {
+		// TODO: check that user can access this share
+		const share = await this.models().share({ userId: this.userId }).load(shareId);
+		if (!share) throw new ErrorNotFound(`No such share: ${shareId}`);
+
+		const user = await this.models().user().loadByEmail(userEmail);
+		if (!user) throw new ErrorNotFound(`No such user: ${userEmail}`);
+
+		return this.save({
+			share_id: shareId,
+			user_id: user.id,
+		});
+	}
+
+	public async accept(shareId: Uuid, userId: Uuid, accept: boolean = true): Promise<File> {
+		const shareUser = await this.loadByShareIdAndUser(shareId, userId);
+		if (!shareUser) throw new ErrorNotFound(`File has not been shared with this user: ${shareId} / ${userId}`);
+
+		const share = await this.models().share().load(shareId);
+		if (!share) throw new ErrorNotFound(`No such share: ${shareId}`);
+
+		const sourceFile = await this.models().file({ userId: share.owner_id }).load(share.file_id);
+		const rootId = await this.models().file({ userId: this.userId }).userRootFileId();
+
+		return this.withTransaction<File>(async () => {
+			await this.save({ ...shareUser, is_accepted: accept ? 1 : 0 });
+
+			const file: File = {
+				owner_id: userId,
+				source_file_id: share.file_id,
+				parent_id: rootId,
+				name: sourceFile.name,
+			};
+
+			return this.models().file({ userId }).save(file);
+		});
+	}
+
+	public async reject(shareId: Uuid, userId: Uuid) {
+		await this.accept(shareId, userId, false);
+	}
+
+	// Returns the users who have shared files with the current user
+	public async linkedUserIds(): Promise<Uuid[]> {
+		const fileSubQuery = this
+			.db('files')
+			.select('source_file_id')
+			.where('source_file_id', '!=', '')
+			.andWhere('owner_id', '=', this.userId);
+
+		return this
+			.db('files')
+			.distinct('owner_id')
+			.whereIn('files.id', fileSubQuery)
+			.pluck('owner_id');
+	}
+
+}

packages/server/src/models/factory.ts

@@ -64,6 +64,8 @@ import SessionModel from './SessionModel';
 import ChangeModel from './ChangeModel';
 import NotificationModel from './NotificationModel';
 import ShareModel from './ShareModel';
+import ShareUserModel from './ShareUserModel';
+import JoplinFileContentModel from '../apps/joplin/JoplinFileContentModel';
 
 export class Models {
 
@@ -107,6 +109,14 @@ export class Models {
 		return new ShareModel(this.db_, newModelFactory, this.baseUrl_, options);
 	}
 
+	public shareUser(options: ModelOptions = null) {
+		return new ShareUserModel(this.db_, newModelFactory, this.baseUrl_, options);
+	}
+
+	public joplinFileContent(options: ModelOptions = null) {
+		return new JoplinFileContentModel(this.db_, newModelFactory, this.baseUrl_, options);
+	}
+
 }
 
 export default function newModelFactory(db: DbConnection, baseUrl: string): Models {

packages/server/src/routes/api/files.test.ts

@@ -6,6 +6,35 @@ import { Pagination, PaginationOrderDir } from '../../models/utils/pagination';
 import { ErrorUnprocessableEntity, ErrorForbidden, ErrorNotFound, ErrorConflict } from '../../utils/errors';
 import { msleep } from '../../utils/time';
 
+// const joplinSerializedNote = `Item title
+
+// Item body
+
+// id: d2a73b249d9e4e3da491ad991fbacd87
+// parent_id: c403e0672da5446e8e50927d9ab085a6
+// created_time: 2021-03-07T15:59:35.691Z
+// updated_time: 2021-03-11T19:28:20.929Z
+// is_conflict: 0
+// latitude: 0.00000000
+// longitude: 0.00000000
+// altitude: 0.0000
+// author:
+// source_url:
+// is_todo: 0
+// todo_due: 0
+// todo_completed: 0
+// source: joplin-desktop
+// source_application: net.cozic.joplin-desktop
+// application_data:
+// order: 0
+// user_created_time: 2021-03-07T15:59:35.691Z
+// user_updated_time: 2021-03-11T19:28:20.929Z
+// encryption_cipher_text:
+// encryption_applied: 0
+// markup_language: 1
+// is_shared: 0
+// type_: 1`;
+
 async function makeTempFileWithContent(content: string): Promise<string> {
 	const d = await tempDir();
 	const filePath = `${d}/${randomHash()}`;
@@ -416,4 +445,13 @@ describe('api_files', function() {
 		expect(context.response.status).toBe(ErrorForbidden.httpCode);
 	});
 
+	// TODO: test that MD file format is the same after having been unserialized and serialized again
+
+	// test('should save Joplin item to its own special table', async function() {
+	// 	const { user, session } = await createUserAndSession(1);
+
+	// 	const file = await createFile(user.id, 'root:/d2a73b249d9e4e3da491ad991fbacd87.md:', joplinSerializedNote);
+	// 	// const content = await
+	// });
+
 });

packages/server/src/routes/api/files.ts

@@ -44,9 +44,9 @@ router.del('api/files/:id', async (path: SubPath, ctx: AppContext) => {
 router.get('api/files/:id/content', async (path: SubPath, ctx: AppContext) => {
 	const fileModel = ctx.models.file({ userId: ctx.owner.id });
 	const fileId: Uuid = await fileModel.pathToFileId(path.id);
-	const file = await fileModel.loadWithContent(fileId);
-	if (!file) throw new ErrorNotFound();
-	return respondWithFileContent(ctx.response, file);
+	const fileWithContent = await fileModel.loadWithContent(fileId);
+	if (!fileWithContent) throw new ErrorNotFound();
+	return respondWithFileContent(ctx.response, fileWithContent.file, fileWithContent.content);
 });
 
 router.put('api/files/:id/content', async (path: SubPath, ctx: AppContext) => {
@@ -60,8 +60,23 @@ router.put('api/files/:id/content', async (path: SubPath, ctx: AppContext) => {
 	// https://github.com/laurent22/joplin/issues/4402
 	const buffer = result?.files?.file ? await fs.readFile(result.files.file.path) : Buffer.alloc(0);
 
-	const file: File = await fileModel.pathToFile(fileId, { mustExist: false, returnFullEntity: false });
-	file.content = buffer;
+	const parsedFile: File = await fileModel.pathToFile(fileId, { mustExist: false });
+
+	const isNewFile = !parsedFile.id;
+
+	const file: File = {
+		name: parsedFile.name,
+		content: buffer,
+		source_file_id: 'source_file_id' in parsedFile ? parsedFile.source_file_id : '',
+	};
+
+	if (!isNewFile) {
+		file.id = parsedFile.id;
+	} else {
+		file.name = parsedFile.name;
+		if ('parent_id' in parsedFile) file.parent_id = parsedFile.parent_id;
+	}
+
 	return fileModel.toApiOutput(await fileModel.save(file, { validationRules: { mustBeFile: true } }));
 });
 
@@ -70,8 +85,12 @@ router.del('api/files/:id/content', async (path: SubPath, ctx: AppContext) => {
 	const fileId = path.id;
 	const file: File = await fileModel.pathToFile(fileId, { mustExist: false, returnFullEntity: false });
 	if (!file) return;
-	file.content = Buffer.alloc(0);
-	await fileModel.save(file, { validationRules: { mustBeFile: true } });
+
+	await fileModel.save({
+		id: file.id,
+		content: Buffer.alloc(0),
+		source_file_id: file.source_file_id,
+	}, { validationRules: { mustBeFile: true } });
 });
 
 router.get('api/files/:id/delta', async (path: SubPath, ctx: AppContext) => {

packages/server/src/routes/api/share_users.ts

@@ -0,0 +1,24 @@
+import { ErrorBadRequest, ErrorNotFound } from '../../utils/errors';
+import { bodyFields } from '../../utils/requestUtils';
+import { SubPath } from '../../utils/routeUtils';
+import Router from '../../utils/Router';
+import { AppContext } from '../../utils/types';
+
+const router = new Router();
+
+router.patch('api/share_users/:id', async (path: SubPath, ctx: AppContext) => {
+	// TODO: check permissions
+	const shareUserModel = ctx.models.shareUser({ userId: ctx.owner.id });
+	const shareUser = await shareUserModel.load(path.id);
+	if (!shareUser) throw new ErrorNotFound();
+
+	const body = await bodyFields(ctx.req);
+
+	if ('is_accepted' in body) {
+		return shareUserModel.accept(shareUser.share_id, shareUser.user_id, !!body.is_accepted);
+	} else {
+		throw new ErrorBadRequest('Only setting is_accepted is supported');
+	}
+});
+
+export default router;

packages/server/src/routes/api/shares.test.ts

@@ -1,7 +1,12 @@
-import { ShareType } from '../../db';
+import { ChangeType, File, Share, ShareType, ShareUser } from '../../db';
 import { putFileContent, testFilePath } from '../../utils/testing/fileApiUtils';
-import { getShareContext, postShareContext } from '../../utils/testing/shareApiUtils';
-import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession } from '../../utils/testing/testUtils';
+import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models, createFile, updateFile, checkThrowAsync } from '../../utils/testing/testUtils';
+import { postApiC, postApi, getApiC, patchApi, getApi } from '../../utils/testing/apiUtils';
+import { PaginatedFiles } from '../../models/FileModel';
+import { PaginatedChanges } from '../../models/ChangeModel';
+import { shareWithUserAndAccept } from '../../utils/testing/shareApiUtils';
+import { msleep } from '../../utils/time';
+import { ErrorBadRequest } from '../../utils/errors';
 
 describe('api_shares', function() {
 
@@ -17,20 +22,222 @@ describe('api_shares', function() {
 		await beforeEachDb();
 	});
 
-	test('should share a file', async function() {
-		const { session } = await createUserAndSession(1, false);
+	test('should share a file by link', async function() {
+		const { session } = await createUserAndSession(1);
 		const file = await putFileContent(session.id, 'root:/photo.jpg:', testFilePath());
 
-		const context = await postShareContext(session.id, 'root:/photo.jpg:');
+		const context = await postApiC(session.id, 'shares', {
+			type: ShareType.Link,
+			file_id: 'root:/photo.jpg:',
+		});
+
 		expect(context.response.status).toBe(200);
 		const shareId = context.response.body.id;
 
 		{
-			const context = await getShareContext(shareId);
+			const context = await getApiC(session.id, `shares/${shareId}`);
 			expect(context.response.body.id).toBe(shareId);
 			expect(context.response.body.file_id).toBe(file.id);
 			expect(context.response.body.type).toBe(ShareType.Link);
 		}
 	});
 
+	test('should share a file with another user', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		await createFile(user1.id, 'root:/test.txt:', 'created by sharer');
+
+		// ----------------------------------------------------------------
+		// Create the file share object
+		// ----------------------------------------------------------------
+		const share = await postApi<Share>(session1.id, 'shares', {
+			type: ShareType.App,
+			file_id: 'root:/test.txt:',
+		});
+
+		// ----------------------------------------------------------------
+		// Send the share to a user
+		// ----------------------------------------------------------------
+		let shareUser = await postApi(session1.id, `shares/${share.id}/users`, {
+			email: user2.email,
+		}) as ShareUser;
+
+		shareUser = await models().shareUser().load(shareUser.id);
+		expect(shareUser.share_id).toBe(share.id);
+		expect(shareUser.user_id).toBe(user2.id);
+		expect(shareUser.is_accepted).toBe(0);
+
+		// ----------------------------------------------------------------
+		// On the sharee side, accept the share
+		// ----------------------------------------------------------------
+		await patchApi<ShareUser>(session2.id, `share_users/${shareUser.id}`, { is_accepted: 1 });
+
+		{
+			shareUser = await models().shareUser().load(shareUser.id);
+			expect(shareUser.is_accepted).toBe(1);
+		}
+
+		// ----------------------------------------------------------------
+		// On the sharee side, check that the file is present
+		// and with the right content.
+		// ----------------------------------------------------------------
+		const results = await getApi<PaginatedFiles>(session2.id, 'files/root/children');
+		expect(results.items.length).toBe(1);
+		expect(results.items[0].name).toBe('test.txt');
+
+		const fileContent = await getApi<Buffer>(session2.id, 'files/root:/test.txt:/content');
+		expect(fileContent.toString()).toBe('created by sharer');
+
+		// ----------------------------------------------------------------
+		// If file is changed by sharee, sharer should see the change too
+		// ----------------------------------------------------------------
+		{
+			await updateFile(user2.id, 'root:/test.txt:', 'modified by sharee');
+			const fileContent = await getApi<Buffer>(session1.id, 'files/root:/test.txt:/content');
+			expect(fileContent.toString()).toBe('modified by sharee');
+		}
+	});
+
+	test('should get updated time of shared file', async function() {
+		// If sharer changes the file, sharee should see the updated_time of the sharer file.
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+
+		let { sharerFile, shareeFile } = await shareWithUserAndAccept(session1.id, user1, session2.id, user2);
+
+		await msleep(1);
+
+		await updateFile(user1.id, sharerFile.id, 'content modified');
+
+		sharerFile = await models().file({ userId: user1.id }).load(sharerFile.id);
+
+		// Check files/:id
+
+		shareeFile = await getApi<File>(session2.id, `files/${shareeFile.id}`);
+		expect(shareeFile.updated_time).toBe(sharerFile.updated_time);
+
+		// Check files/:id/children
+
+		const rootFileId2 = await models().file({ userId: user2.id }).userRootFileId();
+		const page = await getApi<PaginatedFiles>(session2.id, `files/${rootFileId2}/children`);
+		expect(page.items[0].updated_time).toBe(sharerFile.updated_time);
+	});
+
+	test('should not share an already shared file', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		const { user: user3, session: session3 } = await createUserAndSession(3);
+
+		const { shareeFile } = await shareWithUserAndAccept(session1.id, user1, session2.id, user2);
+		const error = await checkThrowAsync(async () => shareWithUserAndAccept(session2.id, user2, session3.id, user3, shareeFile));
+		expect(error.httpCode).toBe(ErrorBadRequest.httpCode);
+	});
+
+	test('should see delta changes for linked files', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		const rootDirId1 = await models().file({ userId: user1.id }).userRootFileId();
+		const rootDirId2 = await models().file({ userId: user2.id }).userRootFileId();
+
+		const { sharerFile, shareeFile } = await shareWithUserAndAccept(session1.id, user1, session2.id, user2);
+
+		let cursor1: string = null;
+		let cursor2: string = null;
+
+		{
+			const page1 = await getApi<PaginatedChanges>(session1.id, `files/${rootDirId1}/delta`);
+			expect(page1.items.length).toBe(1);
+			expect(page1.items[0].item.id).toBe(sharerFile.id);
+			expect(page1.items[0].type).toBe(ChangeType.Create);
+			cursor1 = page1.cursor;
+
+			const page2 = await getApi<PaginatedChanges>(session2.id, `files/${rootDirId2}/delta`);
+			expect(page2.items.length).toBe(1);
+			expect(page2.items[0].item.id).toBe(shareeFile.id);
+			expect(page2.items[0].type).toBe(ChangeType.Create);
+			cursor2 = page2.cursor;
+		}
+
+		// --------------------------------------------------------------------
+		// If file is changed on sharer side, sharee should see the changes
+		// --------------------------------------------------------------------
+
+		await msleep(1);
+		await updateFile(user1.id, sharerFile.id, 'from sharer');
+
+		{
+			const page1 = await getApi<PaginatedChanges>(session1.id, `files/${rootDirId1}/delta`, { query: { cursor: cursor1 } });
+			expect(page1.items.length).toBe(1);
+			expect(page1.items[0].item.id).toBe(sharerFile.id);
+			expect(page1.items[0].type).toBe(ChangeType.Update);
+			cursor1 = page1.cursor;
+
+			const page2 = await getApi<PaginatedChanges>(session2.id, `files/${rootDirId2}/delta`, { query: { cursor: cursor2 } });
+			expect(page2.items.length).toBe(1);
+			expect(page2.items[0].item.id).toBe(shareeFile.id);
+			expect(page2.items[0].type).toBe(ChangeType.Update);
+			expect(page2.items[0].item.updated_time).toBe(page1.items[0].item.updated_time);
+			cursor2 = page2.cursor;
+		}
+
+		// --------------------------------------------------------------------
+		// If file is changed on sharee side, sharer should see the changes
+		// --------------------------------------------------------------------
+
+		await msleep(1);
+		await updateFile(user2.id, shareeFile.id, 'from sharee');
+
+		{
+			const page1 = await getApi<PaginatedChanges>(session1.id, `files/${rootDirId1}/delta`, { query: { cursor: cursor1 } });
+			expect(page1.items.length).toBe(1);
+			expect(page1.items[0].item.id).toBe(sharerFile.id);
+			expect(page1.items[0].type).toBe(ChangeType.Update);
+			cursor1 = page1.cursor;
+
+			const page2 = await getApi<PaginatedChanges>(session2.id, `files/${rootDirId2}/delta`, { query: { cursor: cursor2 } });
+			expect(page2.items.length).toBe(1);
+			expect(page2.items[0].item.id).toBe(shareeFile.id);
+			expect(page2.items[0].type).toBe(ChangeType.Update);
+			cursor2 = page2.cursor;
+
+			// The updated_time properties don't necessarily match because first
+			// the sharer's file content is updated, and then the sharee's file
+			// metadata may be updated too.
+
+			// expect(page1.items[0].item.updated_time).toBe(page2.items[0].item.updated_time);
+		}
+	});
+
+	test('should see delta changes when a third user joins in', async function() {
+		// - User 1 shares a file with User 2
+		// - User 2 syncs and get a new delta cursor C2
+		// - User 3 shares a file with User 2
+		// - User 2 syncs **starting from C2**
+		// => The new changes from User 3 share should be visible
+
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		const { user: user3, session: session3 } = await createUserAndSession(3);
+		const rootDirId2 = await models().file({ userId: user2.id }).userRootFileId();
+
+		await shareWithUserAndAccept(session1.id, user1, session2.id, user2);
+		let cursor = null;
+
+		{
+			const page = await getApi<PaginatedChanges>(session2.id, `files/${rootDirId2}/delta`);
+			cursor = page.cursor;
+		}
+
+		const file3 = await createFile(user3.id, 'root:/test3.txt:', 'from user 3');
+		const { shareeFile } = await shareWithUserAndAccept(session3.id, user3, session2.id, user2, file3);
+
+		{
+			const page = await getApi<PaginatedChanges>(session2.id, `files/${rootDirId2}/delta`, { query: { cursor } });
+			cursor = page.cursor;
+			expect(page.items.length).toBe(1);
+			expect(page.items[0].type).toBe(ChangeType.Create);
+			expect(page.items[0].item.id).toBe(shareeFile.id);
+		}
+	});
+
 });

packages/server/src/routes/api/shares.ts

@@ -1,5 +1,5 @@
 import { ErrorNotFound } from '../../utils/errors';
-import { Share } from '../../db';
+import { Share, User } from '../../db';
 import { bodyFields, ownerRequired } from '../../utils/requestUtils';
 import { SubPath } from '../../utils/routeUtils';
 import Router from '../../utils/Router';
@@ -14,7 +14,14 @@ router.post('api/shares', async (_path: SubPath, ctx: AppContext) => {
 
 	const shareModel = ctx.models.share({ userId: ctx.owner.id });
 	const share: Share = shareModel.fromApiInput(await bodyFields(ctx.req)) as Share;
-	return shareModel.add(share.type, share.file_id);
+	return shareModel.createShare(share.type, share.file_id);
+});
+
+router.post('api/shares/:id/users', async (path: SubPath, ctx: AppContext) => {
+	ownerRequired(ctx);
+
+	const user: User = await bodyFields(ctx.req) as User;
+	return ctx.models.shareUser({ userId: ctx.owner.id }).addByEmail(path.id, user.email);
 });
 
 router.get('api/shares/:id', async (path: SubPath, ctx: AppContext) => {

packages/server/src/routes/index/files.ts

@@ -93,10 +93,10 @@ router.get('files/:id', async (path: SubPath, ctx: AppContext) => {
 
 router.get('files/:id/content', async (path: SubPath, ctx: AppContext) => {
 	const fileModel = ctx.models.file({ userId: ctx.owner.id });
-	let file: File = await fileModel.pathToFile(path.id, { returnFullEntity: false });
-	file = await fileModel.loadWithContent(file.id);
-	if (!file) throw new ErrorNotFound();
-	return respondWithFileContent(ctx.response, file);
+	const file: File = await fileModel.pathToFile(path.id, { returnFullEntity: false });
+	const fileWithContent = await fileModel.loadWithContent(file.id);
+	if (!fileWithContent) throw new ErrorNotFound();
+	return respondWithFileContent(ctx.response, fileWithContent.file, fileWithContent.content);
 });
 
 router.post('files', async (_path: SubPath, ctx: AppContext) => {

packages/server/src/routes/index/shares.joplin.test.ts

@@ -1,4 +1,5 @@
 import { NoteEntity } from '@joplin/lib/services/database/types';
+import { ShareType } from '../../db';
 import routeHandler from '../../middleware/routeHandler';
 import { putFileContent, testFilePath, postDirectory } from '../../utils/testing/fileApiUtils';
 import { postShare } from '../../utils/testing/shareApiUtils';
@@ -92,7 +93,7 @@ describe('shares.joplin', function() {
 			body: 'Testing body',
 		}));
 
-		const share = await postShare(session.id, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
+		const share = await postShare(session.id, ShareType.Link, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
 
 		const bodyHtml = await getShareContent(share.id);
 
@@ -109,7 +110,7 @@ describe('shares.joplin', function() {
 			body: '$\\sqrt{3x-1}+(1+x)^2$',
 		}));
 
-		const share = await postShare(session.id, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
+		const share = await postShare(session.id, ShareType.Link, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
 
 		const bodyHtml = await getShareContent(share.id);
 
@@ -126,7 +127,7 @@ describe('shares.joplin', function() {
 		await putFileContent(session.id, 'root:/.resource/96765a68655f4446b3dbad7d41b6566e:', testFilePath());
 		await createFile(user.id, 'root:/96765a68655f4446b3dbad7d41b6566e.md:', resourceContents.image);
 
-		const share = await postShare(session.id, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
+		const share = await postShare(session.id, ShareType.Link, 'root:/b39dadd7a63742bebf3125fd2a9286d4.md:');
 
 		const bodyHtml = await getShareContent(share.id) as string;
 
@@ -156,7 +157,7 @@ describe('shares.joplin', function() {
 			await createFile(user.id, `root:/${noteId}.md:`, makeNoteSerializedBody({
 				body: '![missing](:/531a2a839a2c493a88c45e39c6cb9ed4)',
 			}));
-			const share = await postShare(session.id, `root:/${noteId}.md:`);
+			const share = await postShare(session.id, ShareType.Link, `root:/${noteId}.md:`);
 			await expectNotThrow(async () => getShareContent(share.id));
 		}
 
@@ -165,7 +166,9 @@ describe('shares.joplin', function() {
 			await createFile(user.id, `root:/${noteId}.md:`, makeNoteSerializedBody({
 				body: '[missing too](:/531a2a839a2c493a88c45e39c6cb9ed4)',
 			}));
-			const share = await postShare(session.id, `root:/${noteId}.md:`);
+
+			// Was commented out:
+			const share = await postShare(session.id, ShareType.Link, `root:/${noteId}.md:`);
 			await expectNotThrow(async () => getShareContent(share.id));
 		}
 	});

packages/server/src/routes/index/shares.ts

@@ -2,18 +2,21 @@ import { SubPath, ResponseType, Response } from '../../utils/routeUtils';
 import Router from '../../utils/Router';
 import { AppContext } from '../../utils/types';
 import { ErrorNotFound } from '../../utils/errors';
-import { File, Share } from '../../db';
+import { Share } from '../../db';
 import { FileViewerResponse } from '../../apps/joplin/Application';
+import { FileWithContent } from '../../models/FileModel';
 
-async function renderFile(context: AppContext, file: File, share: Share): Promise<FileViewerResponse> {
+async function renderFile(context: AppContext, fileWithContent: FileWithContent, share: Share): Promise<FileViewerResponse> {
 	const joplinApp = await context.apps.joplin();
 
-	if (await joplinApp.isItemFile(file)) {
-		return joplinApp.renderFile(file, share, context.query);
+	if (await joplinApp.fileToJoplinItem(fileWithContent)) {
+		return joplinApp.renderFile(fileWithContent, share, context.query);
 	}
 
+	const { file, content } = fileWithContent;
+
 	return {
-		body: file.content,
+		body: content,
 		mime: file.mime_type,
 		size: file.size,
 	};
@@ -30,11 +33,10 @@ router.get('shares/:id', async (path: SubPath, ctx: AppContext) => {
 	const share = await shareModel.load(path.id);
 	if (!share) throw new ErrorNotFound();
 
-	const file = await fileModel.loadWithContent(share.file_id, { skipPermissionCheck: true });
-	if (!file) throw new ErrorNotFound();
+	const fileWithContent = await fileModel.loadWithContent(share.file_id, { skipPermissionCheck: true });
+	if (!fileWithContent) throw new ErrorNotFound();
 
-
-	const result = await renderFile(ctx, file, share);
+	const result = await renderFile(ctx, fileWithContent, share);
 
 	ctx.response.body = result.body;
 	ctx.response.set('Content-Type', result.mime);

packages/server/src/routes/routes.ts

@@ -4,6 +4,7 @@ import apiSessions from './api/sessions';
 import apiPing from './api/ping';
 import apiFiles from './api/files';
 import apiShares from './api/shares';
+import apiShareUsers from './api/share_users';
 
 import indexLogin from './index/login';
 import indexLogout from './index/logout';
@@ -21,6 +22,7 @@ const routes: Routers = {
 	'api/sessions': apiSessions,
 	'api/files': apiFiles,
 	'api/shares': apiShares,
+	'api/share_users': apiShareUsers,
 
 	'login': indexLogin,
 	'logout': indexLogout,

packages/server/src/tools/generateTypes.ts

@@ -30,15 +30,19 @@ const config = {
 		'main.changes': 'WithDates, WithUuid',
 		'main.notifications': 'WithDates, WithUuid',
 		'main.shares': 'WithDates, WithUuid',
+		'main.share_users': 'WithDates, WithUuid',
 	},
 };
 
 const propertyTypes: Record<string, string> = {
 	'*.item_type': 'ItemType',
 	'files.content': 'Buffer',
+	'files.content_type': 'FileContentType',
 	'changes.type': 'ChangeType',
 	'notifications.level': 'NotificationLevel',
 	'shares.type': 'ShareType',
+	'joplin_items.id': 'string',
+	'joplin_items.parent_id': 'string',
 };
 
 function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void {
@@ -73,9 +77,9 @@ function createTypeString(table: any) {
 			if (['id'].includes(name)) continue;
 		}
 
+		if ((name === 'id' || name.endsWith('_id') || name === 'uuid') && type === 'string') type = 'Uuid';
 		if (propertyTypes[`*.${name}`]) type = propertyTypes[`*.${name}`];
 		if (propertyTypes[`${table.name}.${name}`]) type = propertyTypes[`${table.name}.${name}`];
-		if ((name === 'id' || name.endsWith('_id') || name === 'uuid') && type === 'string') type = 'Uuid';
 
 		colStrings.push(`\t${name}?: ${type};`);
 	}

packages/server/src/utils/errors.ts

@@ -1,7 +1,7 @@
 // For explanation of the setPrototypeOf call, see:
 // https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
 
-class ApiError extends Error {
+export class ApiError extends Error {
 	public static httpCode: number = 400;
 
 	public httpCode: number;

packages/server/src/utils/requestUtils.ts

@@ -35,7 +35,7 @@ export async function formParse(req: any): Promise<FormParseResult> {
 	});
 }
 
-export async function bodyFields(req: any): Promise<BodyFields> {
+export async function bodyFields(req: any/* , filter:string[] = null*/): Promise<BodyFields> {
 	// Formidable needs the content-type to be 'application/json' so on our side
 	// we explicitely set it to that. However save the previous value so that it
 	// can be restored.
@@ -47,7 +47,18 @@ export async function bodyFields(req: any): Promise<BodyFields> {
 
 	const form = await formParse(req);
 	if (previousContentType) req.headers['content-type'] = previousContentType;
+
 	return form.fields;
+
+	// if (filter) {
+	// 	const output:BodyFields = {};
+	// 	Object.keys(form.fields).forEach(f => {
+	// 		if (filter.includes(f)) output[f] = form.fields[f];
+	// 	});
+	// 	return output;
+	// } else {
+	// 	return form.fields;
+	// }
 }
 
 export function ownerRequired(ctx: AppContext) {

packages/server/src/utils/routeUtils.ts

@@ -1,7 +1,8 @@
 import { File, ItemAddressingType } from '../db';
-import { ErrorBadRequest } from './errors';
+import { ErrorBadRequest, ErrorForbidden, ErrorNotFound } from './errors';
+import { FileContent } from '../models/FileModel';
 import Router from './Router';
-import { AppContext } from './types';
+import { AppContext, HttpMethod } from './types';
 
 const { ltrimSlashes, rtrimSlashes } = require('@joplin/lib/path-utils');
 
@@ -151,16 +152,31 @@ export function parseSubPath(basePath: string, p: string): SubPath {
 	return output;
 }
 
-export function routeResponseFormat(match: MatchedRoute, context: AppContext): RouteResponseFormat {
-	const rawPath = context.path;
-	if (match && match.route.responseFormat) return match.route.responseFormat;
+export function routeResponseFormat(context: AppContext): RouteResponseFormat {
+	// const rawPath = context.path;
+	// if (match && match.route.responseFormat) return match.route.responseFormat;
 
-	let path = rawPath;
-	if (match) path = match.basePath ? match.basePath : match.subPath.raw;
+	// let path = rawPath;
+	// if (match) path = match.basePath ? match.basePath : match.subPath.raw;
 
+	const path = context.path;
 	return path.indexOf('api') === 0 || path.indexOf('/api') === 0 ? RouteResponseFormat.Json : RouteResponseFormat.Html;
 }
 
+export async function execRequest(routes: Routers, ctx: AppContext) {
+	const match = findMatchingRoute(ctx.path, routes);
+	if (!match) throw new ErrorNotFound();
+
+	const routeHandler = match.route.findEndPoint(ctx.request.method as HttpMethod, match.subPath.schema);
+
+	// This is a generic catch-all for all private end points - if we
+	// couldn't get a valid session, we exit now. Individual end points
+	// might have additional permission checks depending on the action.
+	if (!match.route.public && !ctx.owner) throw new ErrorForbidden();
+
+	return routeHandler(match.subPath, ctx);
+}
+
 // In a path such as "/api/files/SOME_ID/content" we want to find:
 // - The base path: "api/files"
 // - The ID: "SOME_ID"
@@ -172,10 +188,16 @@ export function findMatchingRoute(path: string, routes: Routers): MatchedRoute {
 	// an empty string. So for example we now have ['api', 'files', 'SOME_ID', 'content'].
 	splittedPath.splice(0, 1);
 
+	let namespace = '';
+	if (splittedPath[0] === 'apps') {
+		namespace = splittedPath.splice(0, 2).join('/');
+	}
+
+	// Paths such as "/api/files/:id" will be processed here
 	if (splittedPath.length >= 2) {
 		// Create the base path, eg. "api/files", to match it to one of the
-		// routes.s
-		const basePath = `${splittedPath[0]}/${splittedPath[1]}`;
+		// routes.
+		const basePath = `${namespace ? `${namespace}/` : ''}${splittedPath[0]}/${splittedPath[1]}`;
 		if (routes[basePath]) {
 			// Remove the base path from the array so that parseSubPath() can
 			// extract the ID and link from the URL. So the array will contain
@@ -189,16 +211,19 @@ export function findMatchingRoute(path: string, routes: Routers): MatchedRoute {
 		}
 	}
 
+	// Paths such as "/users/:id" or "/apps/joplin/notes/:id" will get here
 	const basePath = splittedPath[0];
-	if (routes[basePath]) {
+	const basePathNS = (namespace ? `${namespace}/` : '') + basePath;
+	if (routes[basePathNS]) {
 		splittedPath.splice(0, 1);
 		return {
-			route: routes[basePath],
+			route: routes[basePathNS],
 			basePath: basePath,
 			subPath: parseSubPath(basePath, `/${splittedPath.join('/')}`),
 		};
 	}
 
+	// Default routes - to process CSS or JS files for example
 	if (routes['']) {
 		return {
 			route: routes[''],
@@ -210,8 +235,8 @@ export function findMatchingRoute(path: string, routes: Routers): MatchedRoute {
 	throw new Error('Unreachable');
 }
 
-export function respondWithFileContent(koaResponse: any, file: File): Response {
-	koaResponse.body = file.content;
+export function respondWithFileContent(koaResponse: any, file: File, content: FileContent): Response {
+	koaResponse.body = content;
 	koaResponse.set('Content-Type', file.mime_type);
 	koaResponse.set('Content-Length', file.size.toString());
 	return new Response(ResponseType.KoaResponse, koaResponse);

packages/server/src/utils/setupAppContext.ts

@@ -4,11 +4,13 @@ import { DbConnection } from '../db';
 import newModelFactory from '../models/factory';
 import Applications from '../services/Applications';
 import { AppContext, Env } from './types';
+import routes from '../routes/routes';
 
 export default async function(appContext: AppContext, env: Env, dbConnection: DbConnection, appLogger: ()=> LoggerWrapper) {
 	appContext.env = env;
 	appContext.db = dbConnection;
-	appContext.models = newModelFactory(appContext.db, config().baseUrl);
 	appContext.apps = new Applications(appContext.models);
+	appContext.models = newModelFactory(appContext.db, config().baseUrl);
 	appContext.appLogger = appLogger;
+	appContext.routes = routes;
 }

packages/server/src/utils/testing/apiUtils.ts

@@ -0,0 +1,67 @@
+import { AppContext } from '../types';
+import routeHandler from '../../middleware/routeHandler';
+import { AppContextTestOptions, checkContextError, koaAppContext } from './testUtils';
+
+interface ExecRequestOptions {
+	filePath?: string;
+	query?: Record<string, any>;
+}
+
+export async function putApiC(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<AppContext> {
+	return execApiC(sessionId, 'PUT', path, body, options);
+}
+
+export async function putApi<T>(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<T> {
+	return execApi<T>(sessionId, 'PUT', path, body, options);
+}
+
+export async function patchApiC(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<AppContext> {
+	return execApiC(sessionId, 'PATCH', path, body, options);
+}
+
+export async function patchApi<T>(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<T> {
+	return execApi<T>(sessionId, 'PATCH', path, body, options);
+}
+
+export async function getApiC(sessionId: string, path: string, options: ExecRequestOptions = null): Promise<AppContext> {
+	return execApiC(sessionId, 'GET', path, null, options);
+}
+
+export async function getApi<T>(sessionId: string, path: string, options: ExecRequestOptions = null): Promise<T> {
+	return execApi<T>(sessionId, 'GET', path, null, options);
+}
+
+export async function postApiC(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<AppContext> {
+	return execApiC(sessionId, 'POST', path, body, options);
+}
+
+export async function postApi<T>(sessionId: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<T> {
+	return execApi<T>(sessionId, 'POST', path, body, options);
+}
+
+export async function execApiC(sessionId: string, method: string, path: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<AppContext> {
+	options = options || {};
+
+	const appContextOptions: AppContextTestOptions = {
+		sessionId,
+		request: {
+			method,
+			url: `/api/${path}`,
+		},
+	};
+
+	if (body) appContextOptions.request.body = body;
+
+	if (options.filePath) appContextOptions.request.files = { file: { path: options.filePath } };
+	if (options.query) appContextOptions.request.query = options.query;
+
+	const context = await koaAppContext(appContextOptions);
+	await routeHandler(context);
+	return context;
+}
+
+export async function execApi<T>(sessionId: string, method: string, url: string, body: Record<string, any> = null, options: ExecRequestOptions = null): Promise<T> {
+	const context = await execApiC(sessionId, method, url, body, options);
+	await checkContextError(context);
+	return context.response.body as T;
+}

packages/server/src/utils/testing/shareApiUtils.ts

@@ -1,9 +1,40 @@
-import { Share, ShareType, Uuid } from '../../db';
+import { File, Share, ShareType, ShareUser, User, Uuid } from '../../db';
 import routeHandler from '../../middleware/routeHandler';
 import { AppContext } from '../types';
-import { checkContextError, koaAppContext } from './testUtils';
+import { patchApi, postApi } from './apiUtils';
+import { checkContextError, createFile, koaAppContext, models } from './testUtils';
 
-export async function postShareContext(sessionId: string, itemId: Uuid): Promise<AppContext> {
+// Handles the whole process of:
+//
+// - User 1 creates a file (optionally)
+// - User 1 creates a file share for it
+// - User 1 shares this with user 2
+// - User 2 accepts the share
+//
+// The result is that user 2 will have a file linked to user 1's file.
+export async function shareWithUserAndAccept(sharerSessionId:string, sharer:User, shareeSessionId:string, sharee:User, file:File = null) {
+	file = file || await createFile(sharer.id, 'root:/test.txt:', 'testing share');
+
+	const share = await postApi<Share>(sharerSessionId, 'shares', {
+		type: ShareType.App,
+		file_id: file.id,
+	});
+
+	let shareUser = await postApi(sharerSessionId, `shares/${share.id}/users`, {
+		email: sharee.email,
+	}) as ShareUser;
+
+	shareUser = await models().shareUser().load(shareUser.id);
+
+	const shareeFile:File = await patchApi(shareeSessionId, `share_users/${shareUser.id}`, { is_accepted: 1 });
+
+	return {
+		sharerFile: file,
+		shareeFile: shareeFile,
+	};
+}
+
+export async function postShareContext(sessionId: string, shareType: ShareType, itemId: Uuid): Promise<AppContext> {
 	const context = await koaAppContext({
 		sessionId: sessionId,
 		request: {
@@ -11,7 +42,7 @@ export async function postShareContext(sessionId: string, itemId: Uuid): Promise
 			url: '/api/shares',
 			body: {
 				file_id: itemId,
-				type: ShareType.Link,
+				type: shareType,
 			},
 		},
 	});
@@ -19,8 +50,47 @@ export async function postShareContext(sessionId: string, itemId: Uuid): Promise
 	return context;
 }
 
-export async function postShare(sessionId: string, itemId: Uuid): Promise<Share> {
-	const context = await postShareContext(sessionId, itemId);
+export async function postShare(sessionId: string, shareType: ShareType, itemId: Uuid): Promise<Share> {
+	const context = await postShareContext(sessionId, shareType, itemId);
+	checkContextError(context);
+	return context.response.body;
+}
+
+export async function postShareUserContext(sessionId: string, shareId: Uuid, userEmail: string): Promise<AppContext> {
+	const context = await koaAppContext({
+		sessionId: sessionId,
+		request: {
+			method: 'POST',
+			url: `/api/shares/${shareId}/users`,
+			body: {
+				email: userEmail,
+			},
+		},
+	});
+	await routeHandler(context);
+	return context;
+}
+
+export async function patchShareUserContext(sessionId: string, shareUserId: Uuid, body: ShareUser): Promise<AppContext> {
+	const context = await koaAppContext({
+		sessionId: sessionId,
+		request: {
+			method: 'PATCH',
+			url: `/api/share_users/${shareUserId}`,
+			body: body,
+		},
+	});
+	await routeHandler(context);
+	return context;
+}
+
+export async function patchShareUser(sessionId: string, shareUserId: Uuid, body: ShareUser): Promise<void> {
+	const context = await patchShareUserContext(sessionId, shareUserId, body);
+	checkContextError(context);
+}
+
+export async function postShareUser(sessionId: string, shareId: Uuid, userEmail: string): Promise<ShareUser> {
+	const context = await postShareUserContext(sessionId, shareId, userEmail);
 	checkContextError(context);
 	return context.response.body;
 }

packages/server/src/utils/testing/testUtils.ts

@@ -13,6 +13,7 @@ import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as jsdom from 'jsdom';
 import setupAppContext from '../setupAppContext';
+import { ApiError } from '../errors';
 
 // Takes into account the fact that this file will be inside the /dist directory
 // when it runs.
@@ -68,7 +69,7 @@ export async function beforeEachDb() {
 	await truncateTables(db_);
 }
 
-interface AppContextTestOptions {
+export interface AppContextTestOptions {
 	// owner?: User;
 	sessionId?: string;
 	request?: any;
@@ -79,6 +80,33 @@ function initGlobalLogger() {
 	Logger.initializeGlobalLogger(globalLogger);
 }
 
+export function msleep(ms: number) {
+	// It seems setTimeout can sometimes last less time than the provided
+	// interval:
+	//
+	// https://stackoverflow.com/a/50912029/561309
+	//
+	// This can cause issues in tests where we expect the actual duration to be
+	// the same as the provided interval or more, but not less. So the code
+	// below check that the elapsed time is no less than the provided interval,
+	// and if it is, it waits a bit longer.
+	const startTime = Date.now();
+	return new Promise((resolve) => {
+		setTimeout(() => {
+			if (Date.now() - startTime < ms) {
+				const iid = setInterval(() => {
+					if (Date.now() - startTime >= ms) {
+						clearInterval(iid);
+						resolve(null);
+					}
+				}, 2);
+			} else {
+				resolve(null);
+			}
+		}, ms);
+	});
+}
+
 export async function koaAppContext(options: AppContextTestOptions = null): Promise<AppContext> {
 	if (!db_) throw new Error('Database must be initialized first');
 
@@ -210,10 +238,21 @@ export async function createFile(userId: string, path: string, content: string):
 	return fileModel.load(savedFile.id);
 }
 
+export async function updateFile(userId: string, path: string, content: string): Promise<File> {
+	const fileModel = models().file({ userId });
+	const file: File = await fileModel.pathToFile(path, { returnFullEntity: true });
+	await fileModel.save({
+		id: file.id,
+		content: Buffer.from(content),
+		source_file_id: file.source_file_id,
+	});
+	return fileModel.load(file.id);
+}
+
 export function checkContextError(context: AppContext) {
 	if (context.response.status >= 400) {
 		// console.info(context.response.body);
-		throw new Error(`${context.method} ${context.path} ${JSON.stringify(context.response)}`);
+		throw new ApiError(`${context.method} ${context.path} ${JSON.stringify(context.response)}`, context.response.status);
 	}
 }
 

packages/server/src/utils/types.ts

@@ -3,6 +3,7 @@ import * as Koa from 'koa';
 import { DbConnection, User, Uuid } from '../db';
 import { Models } from '../models/factory';
 import Applications from '../services/Applications';
+import { Routers } from './routeUtils';
 
 export enum Env {
 	Dev = 'dev',
@@ -25,6 +26,7 @@ export interface AppContext extends Koa.Context {
 	notifications: NotificationView[];
 	owner: User;
 	apps: Applications;
+	routes: Routers;
 }
 
 export enum DatabaseConfigClient {