Server: Add support for uploading multiple items in one request

packages/server/src/models/ItemModel.ts

@@ -12,6 +12,18 @@ const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
 // Converts "root:/myfile.txt:" to "myfile.txt"
 const extractNameRegex = /^root:\/(.*):$/;
 
+export interface SaveFromRawContentItem {
+	name: string;
+	body: Buffer;
+}
+
+export interface SaveFromRawContentResultItem {
+	item: Item;
+	error: any;
+}
+
+export type SaveFromRawContentResult = Record<string, SaveFromRawContentResultItem>;
+
 export interface PaginatedItems extends PaginatedResults {
 	items: Item[];
 }
@@ -282,16 +294,32 @@ export default class ItemModel extends BaseModel<Item> {
 		return this.itemToJoplinItem(raw);
 	}
 
-	public async saveFromRawContent(user: User, name: string, buffer: Buffer, options: ItemSaveOption = null): Promise<Item> {
+	public async saveFromRawContent(user: User, rawContentItems: SaveFromRawContentItem[], options: ItemSaveOption = null): Promise<SaveFromRawContentResult> {
 		options = options || {};
 
-		const existingItem = await this.loadByName(user.id, name);
+		// In this function, first we process the input items, which may be
+		// serialized Joplin items or actual buffers (for resources) and convert
+		// them to database items. Once it's done those db items are saved in
+		// batch at the end.
 
-		const isJoplinItem = isJoplinItemName(name);
+		interface ItemToProcess {
+			item: Item;
+			error: Error;
+			resourceIds?: string[];
+			isNote?: boolean;
+			joplinItem?: any;
+		}
+
+		const existingItems = await this.loadByNames(user.id, rawContentItems.map(i => i.name));
+		const itemsToProcess: Record<string, ItemToProcess> = {};
+
+		for (const rawItem of rawContentItems) {
+			try {
+				const isJoplinItem = isJoplinItemName(rawItem.name);
 				let isNote = false;
 
 				const item: Item = {
-			name,
+					name: rawItem.name,
 				};
 
 				let joplinItem: any = null;
@@ -299,7 +327,7 @@ export default class ItemModel extends BaseModel<Item> {
 				let resourceIds: string[] = [];
 
 				if (isJoplinItem) {
-			joplinItem = await unserializeJoplinItem(buffer.toString());
+					joplinItem = await unserializeJoplinItem(rawItem.body.toString());
 					isNote = joplinItem.type_ === ModelType.Note;
 					resourceIds = isNote ? linkedResourceIds(joplinItem.body) : [];
 
@@ -319,25 +347,69 @@ export default class ItemModel extends BaseModel<Item> {
 
 					item.content = Buffer.from(JSON.stringify(joplinItemToSave));
 				} else {
-			item.content = buffer;
+					item.content = rawItem.body;
 				}
 
+				const existingItem = existingItems.find(i => i.name === rawItem.name);
 				if (existingItem) item.id = existingItem.id;
 
 				if (options.shareId) item.jop_share_id = options.shareId;
 
-		await this.models().user().checkMaxItemSizeLimit(user, buffer, item, joplinItem);
+				await this.models().user().checkMaxItemSizeLimit(user, rawItem.body, item, joplinItem);
 
-		return this.withTransaction<Item>(async () => {
-			const savedItem = await this.saveForUser(user.id, item);
-
-			if (isNote) {
-				await this.models().itemResource().deleteByItemId(savedItem.id);
-				await this.models().itemResource().addResourceIds(savedItem.id, resourceIds);
+				itemsToProcess[rawItem.name] = {
+					item: item,
+					error: null,
+					resourceIds,
+					isNote,
+					joplinItem,
+				};
+			} catch (error) {
+				itemsToProcess[rawItem.name] = {
+					item: null,
+					error: error,
+				};
+			}
 		}
 
-			return savedItem;
+		const output: SaveFromRawContentResult = {};
+
+		await this.withTransaction(async () => {
+			for (const name of Object.keys(itemsToProcess)) {
+				const o = itemsToProcess[name];
+
+				if (o.error) {
+					output[name] = {
+						item: null,
+						error: o.error,
+					};
+					continue;
+				}
+
+				const itemToSave = o.item;
+
+				try {
+					const savedItem = await this.saveForUser(user.id, itemToSave);
+
+					if (o.isNote) {
+						await this.models().itemResource().deleteByItemId(savedItem.id);
+						await this.models().itemResource().addResourceIds(savedItem.id, o.resourceIds);
+					}
+
+					output[name] = {
+						item: savedItem,
+						error: null,
+					};
+				} catch (error) {
+					output[name] = {
+						item: null,
+						error: error,
+					};
+				}
+			}
 		});
+
+		return output;
 	}
 
 	protected async validate(item: Item, options: ValidateOptions = {}): Promise<Item> {

packages/server/src/routes/api/batch_items.ts

@@ -0,0 +1,19 @@
+import { SubPath } from '../../utils/routeUtils';
+import Router from '../../utils/Router';
+import { RouteType } from '../../utils/types';
+import { AppContext } from '../../utils/types';
+import { putItemContents } from './items';
+import { PaginatedResults } from '../../models/utils/pagination';
+
+const router = new Router(RouteType.Api);
+
+router.put('api/batch_items', async (path: SubPath, ctx: AppContext) => {
+	const output: PaginatedResults = {
+		items: await putItemContents(path, ctx, true) as any,
+		has_more: false,
+	};
+
+	return output;
+});
+
+export default router;

packages/server/src/routes/api/items.test.ts

@@ -3,10 +3,11 @@ import { NoteEntity } from '@joplin/lib/services/database/types';
 import { ModelType } from '@joplin/lib/BaseModel';
 import { deleteApi, getApi, putApi } from '../../utils/testing/apiUtils';
 import { Item } from '../../db';
-import { PaginatedItems } from '../../models/ItemModel';
+import { PaginatedItems, SaveFromRawContentResult } from '../../models/ItemModel';
 import { shareFolderWithUser } from '../../utils/testing/shareApiUtils';
 import { resourceBlobPath } from '../../utils/joplinUtils';
 import { ErrorForbidden, ErrorPayloadTooLarge } from '../../utils/errors';
+import { PaginatedResults } from '../../models/utils/pagination';
 
 describe('api_items', function() {
 
@@ -149,6 +150,56 @@ describe('api_items', function() {
 		expect(result.name).toBe(`${noteId}.md`);
 	});
 
+	test('should batch upload items', async function() {
+		const { session: session1 } = await createUserAndSession(1, false);
+
+		const result: PaginatedResults = await putApi(session1.id, 'batch_items', {
+			items: [
+				{
+					name: '00000000000000000000000000000001.md',
+					body: makeNoteSerializedBody({ id: '00000000000000000000000000000001' }),
+				},
+				{
+					name: '00000000000000000000000000000002.md',
+					body: makeNoteSerializedBody({ id: '00000000000000000000000000000002' }),
+				},
+			],
+		});
+
+		expect(Object.keys(result.items).length).toBe(2);
+		expect(Object.keys(result.items).sort()).toEqual(['00000000000000000000000000000001.md', '00000000000000000000000000000002.md']);
+	});
+
+	test('should report errors when batch uploading', async function() {
+		const { user: user1,session: session1 } = await createUserAndSession(1, false);
+
+		const note1 = makeNoteSerializedBody({ id: '00000000000000000000000000000001' });
+		await models().user().save({ id: user1.id, max_item_size: note1.length });
+
+		const result: PaginatedResults = await putApi(session1.id, 'batch_items', {
+			items: [
+				{
+					name: '00000000000000000000000000000001.md',
+					body: note1,
+				},
+				{
+					name: '00000000000000000000000000000002.md',
+					body: makeNoteSerializedBody({ id: '00000000000000000000000000000002', body: 'too large' }),
+				},
+			],
+		});
+
+		const items: SaveFromRawContentResult = result.items as any;
+
+		expect(Object.keys(items).length).toBe(2);
+		expect(Object.keys(items).sort()).toEqual(['00000000000000000000000000000001.md', '00000000000000000000000000000002.md']);
+
+		expect(items['00000000000000000000000000000001.md'].item).toBeTruthy();
+		expect(items['00000000000000000000000000000001.md'].error).toBeFalsy();
+		expect(items['00000000000000000000000000000002.md'].item).toBeFalsy();
+		expect(items['00000000000000000000000000000002.md'].error.httpCode).toBe(ErrorPayloadTooLarge.httpCode);
+	});
+
 	test('should list children', async function() {
 		const { session } = await createUserAndSession(1, true);
 

packages/server/src/routes/api/items.ts

@@ -6,13 +6,63 @@ import { RouteType } from '../../utils/types';
 import { AppContext } from '../../utils/types';
 import * as fs from 'fs-extra';
 import { ErrorForbidden, ErrorMethodNotAllowed, ErrorNotFound } from '../../utils/errors';
-import ItemModel, { ItemSaveOption } from '../../models/ItemModel';
+import ItemModel, { ItemSaveOption, SaveFromRawContentItem } from '../../models/ItemModel';
 import { requestDeltaPagination, requestPagination } from '../../models/utils/pagination';
 import { AclAction } from '../../models/BaseModel';
 import { safeRemove } from '../../utils/fileUtils';
 
 const router = new Router(RouteType.Api);
 
+export async function putItemContents(path: SubPath, ctx: AppContext, isBatch: boolean) {
+	if (!ctx.owner.can_upload) throw new ErrorForbidden('Uploading content is disabled');
+
+	const parsedBody = await formParse(ctx.req);
+	const bodyFields = parsedBody.fields;
+	const saveOptions: ItemSaveOption = {};
+
+	let items: SaveFromRawContentItem[] = [];
+
+	if (isBatch) {
+		items = bodyFields.items.map((item: any) => {
+			return {
+				name: item.name,
+				body: item.body ? Buffer.from(item.body, 'utf8') : Buffer.alloc(0),
+			};
+		});
+	} else {
+		const filePath = parsedBody?.files?.file ? parsedBody.files.file.path : null;
+
+		try {
+			const buffer = filePath ? await fs.readFile(filePath) : Buffer.alloc(0);
+
+			// This end point can optionally set the associated jop_share_id field. It
+			// is only useful when uploading resource blob (under .resource folder)
+			// since they can't have metadata. Note, Folder and Resource items all
+			// include the "share_id" field property so it doesn't need to be set via
+			// query parameter.
+			if (ctx.query['share_id']) {
+				saveOptions.shareId = ctx.query['share_id'];
+				await ctx.models.item().checkIfAllowed(ctx.owner, AclAction.Create, { jop_share_id: saveOptions.shareId });
+			}
+
+			items = [
+				{
+					name: ctx.models.item().pathToName(path.id),
+					body: buffer,
+				},
+			];
+		} finally {
+			if (filePath) await safeRemove(filePath);
+		}
+	}
+
+	const output = await ctx.models.item().saveFromRawContent(ctx.owner, items, saveOptions);
+	for (const [name] of Object.entries(output)) {
+		if (output[name].item) output[name].item = ctx.models.item().toApiOutput(output[name].item) as Item;
+	}
+	return output;
+}
+
 // Note about access control:
 //
 // - All these calls are scoped to a user, which is derived from the session
@@ -66,36 +116,10 @@ router.get('api/items/:id/content', async (path: SubPath, ctx: AppContext) => {
 });
 
 router.put('api/items/:id/content', async (path: SubPath, ctx: AppContext) => {
-	if (!ctx.owner.can_upload) throw new ErrorForbidden('Uploading content is disabled');
-
-	const itemModel = ctx.models.item();
-	const name = itemModel.pathToName(path.id);
-	const parsedBody = await formParse(ctx.req);
-	const filePath = parsedBody?.files?.file ? parsedBody.files.file.path : null;
-
-	let outputItem: Item = null;
-
-	try {
-		const buffer = filePath ? await fs.readFile(filePath) : Buffer.alloc(0);
-		const saveOptions: ItemSaveOption = {};
-
-		// This end point can optionally set the associated jop_share_id field. It
-		// is only useful when uploading resource blob (under .resource folder)
-		// since they can't have metadata. Note, Folder and Resource items all
-		// include the "share_id" field property so it doesn't need to be set via
-		// query parameter.
-		if (ctx.query['share_id']) {
-			saveOptions.shareId = ctx.query['share_id'];
-			await itemModel.checkIfAllowed(ctx.owner, AclAction.Create, { jop_share_id: saveOptions.shareId });
-		}
-
-		const item = await itemModel.saveFromRawContent(ctx.owner, name, buffer, saveOptions);
-		outputItem = itemModel.toApiOutput(item) as Item;
-	} finally {
-		if (filePath) await safeRemove(filePath);
-	}
-
-	return outputItem;
+	const results = await putItemContents(path, ctx, false);
+	const result = results[Object.keys(results)[0]];
+	if (result.error) throw result.error;
+	return result.item;
 });
 
 router.get('api/items/:id/delta', async (_path: SubPath, ctx: AppContext) => {

packages/server/src/routes/routes.ts

@@ -3,6 +3,7 @@ import { Routers } from '../utils/routeUtils';
 import apiBatch from './api/batch';
 import apiDebug from './api/debug';
 import apiEvents from './api/events';
+import apiBatchItems from './api/batch_items';
 import apiItems from './api/items';
 import apiPing from './api/ping';
 import apiSessions from './api/sessions';
@@ -27,6 +28,7 @@ import defaultRoute from './default';
 
 const routes: Routers = {
 	'api/batch': apiBatch,
+	'api/batch_items': apiBatchItems,
 	'api/debug': apiDebug,
 	'api/events': apiEvents,
 	'api/items': apiItems,

packages/server/src/utils/testing/shareApiUtils.ts

@@ -60,7 +60,8 @@ async function createItemTree3(sessionId: Uuid, userId: Uuid, parentFolderId: st
 			}
 		}
 
-		const newItem = await models().item().saveFromRawContent(user, `${jopItem.id}.md`, Buffer.from(serializedBody));
+		const result = await models().item().saveFromRawContent(user, [{ name: `${jopItem.id}.md`, body: Buffer.from(serializedBody) }]);
+		const newItem = result[`${jopItem.id}.md`].item;
 		if (isFolder && jopItem.children.length) await createItemTree3(sessionId, userId, newItem.jop_id, shareId, jopItem.children);
 	}
 }

packages/server/src/utils/testing/testUtils.ts

@@ -275,19 +275,20 @@ export async function createItemTree(userId: Uuid, parentFolderId: string, tree:
 	}
 }
 
-export async function createItemTree2(userId: Uuid, parentFolderId: string, tree: any[]): Promise<void> {
-	const itemModel = models().item();
-	const user = await models().user().load(userId);
+// export async function createItemTree2(userId: Uuid, parentFolderId: string, tree: any[]): Promise<void> {
+// 	const itemModel = models().item();
+// 	const user = await models().user().load(userId);
 
-	for (const jopItem of tree) {
-		const isFolder = !!jopItem.children;
-		const serializedBody = isFolder ?
-			makeFolderSerializedBody({ ...jopItem, parent_id: parentFolderId }) :
-			makeNoteSerializedBody({ ...jopItem, parent_id: parentFolderId });
-		const newItem = await itemModel.saveFromRawContent(user, `${jopItem.id}.md`, Buffer.from(serializedBody));
-		if (isFolder && jopItem.children.length) await createItemTree2(userId, newItem.jop_id, jopItem.children);
-	}
-}
+// 	for (const jopItem of tree) {
+// 		const isFolder = !!jopItem.children;
+// 		const serializedBody = isFolder ?
+// 			makeFolderSerializedBody({ ...jopItem, parent_id: parentFolderId }) :
+// 			makeNoteSerializedBody({ ...jopItem, parent_id: parentFolderId });
+// 		const result = await itemModel.saveFromRawContent(user, [{ name: `${jopItem.id}.md`, body: Buffer.from(serializedBody) }]);
+// 		const newItem = result[`${jopItem.id}.md`].item;
+// 		if (isFolder && jopItem.children.length) await createItemTree2(userId, newItem.jop_id, jopItem.children);
+// 	}
+// }
 
 export async function createItemTree3(userId: Uuid, parentFolderId: string, shareId: Uuid, tree: any[]): Promise<void> {
 	const itemModel = models().item();
@@ -298,7 +299,8 @@ export async function createItemTree3(userId: Uuid, parentFolderId: string, shar
 		const serializedBody = isFolder ?
 			makeFolderSerializedBody({ ...jopItem, parent_id: parentFolderId, share_id: shareId }) :
 			makeNoteSerializedBody({ ...jopItem, parent_id: parentFolderId, share_id: shareId });
-		const newItem = await itemModel.saveFromRawContent(user, `${jopItem.id}.md`, Buffer.from(serializedBody));
+		const result = await itemModel.saveFromRawContent(user, [{ name: `${jopItem.id}.md`, body: Buffer.from(serializedBody) }]);
+		const newItem = result[`${jopItem.id}.md`].item;
 		if (isFolder && jopItem.children.length) await createItemTree3(userId, newItem.jop_id, shareId, jopItem.children);
 	}
 }