Chore: Server: Added test tools to automatically populate the database (#9085)

packages/app-mobile/ios/Podfile.lock

@@ -288,7 +288,7 @@ PODS:
     - React-Core
   - react-native-get-random-values (1.9.0):
     - React-Core
-  - react-native-image-picker (5.6.1):
+  - react-native-image-picker (5.7.0):
     - React-Core
   - react-native-image-resizer (3.0.7):
     - React-Core
@@ -418,11 +418,11 @@ PODS:
     - React-Core
   - RNVectorIcons (10.0.0):
     - React-Core
-  - RNZipArchive (6.0.9):
+  - RNZipArchive (6.1.0):
     - React-Core
-    - RNZipArchive/Core (= 6.0.9)
+    - RNZipArchive/Core (= 6.1.0)
     - SSZipArchive (~> 2.2)
-  - RNZipArchive/Core (6.0.9):
+  - RNZipArchive/Core (6.1.0):
     - React-Core
     - SSZipArchive (~> 2.2)
   - SSZipArchive (2.4.3)
@@ -669,7 +669,7 @@ SPEC CHECKSUMS:
   react-native-fingerprint-scanner: ac6656f18c8e45a7459302b84da41a44ad96dbbe
   react-native-geolocation: 0f7fe8a4c2de477e278b0365cce27d089a8c5903
   react-native-get-random-values: dee677497c6a740b71e5612e8dbd83e7539ed5bb
-  react-native-image-picker: 5fcac5a5ffcb3737837f0617d43fd767249290de
+  react-native-image-picker: 3269f75c251cdcd61ab51b911dd30d6fff8c6169
   react-native-image-resizer: 681f7607418b97c084ba2d0999b153b103040d8a
   react-native-netinfo: fefd4e98d75cbdd6e85fc530f7111a8afdf2b0c5
   react-native-rsa-native: 12132eb627797529fdb1f0d22fd0f8f9678df64a
@@ -705,7 +705,7 @@ SPEC CHECKSUMS:
   RNSecureRandom: 07efbdf2cd99efe13497433668e54acd7df49fef
   RNShare: 32e97adc8d8c97d4a26bcdd3c45516882184f8b6
   RNVectorIcons: 8b5bb0fa61d54cd2020af4f24a51841ce365c7e9
-  RNZipArchive: 68a0c6db4b1c103f846f1559622050df254a3ade
+  RNZipArchive: ef9451b849c45a29509bf44e65b788829ab07801
   SSZipArchive: fe6a26b2a54d5a0890f2567b5cc6de5caa600aef
   Yoga: e7ea9e590e27460d28911403b894722354d73479
 

packages/server/package.json

@@ -17,6 +17,7 @@
     "test-ci": "yarn test",
     "test-debug": "node --inspect node_modules/.bin/jest -- --verbose=false",
     "clean": "gulp clean",
+    "populateDatabase": "JOPLIN_TESTS_SERVER_DB=pg node dist/utils/testing/populateDatabase",
     "stripeListen": "stripe listen --forward-to http://joplincloud.local:22300/stripe/webhook",
     "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json"
   },
@@ -60,6 +61,7 @@
   "devDependencies": {
     "@joplin/tools": "~2.13",
     "@rmp135/sql-ts": "1.18.0",
+    "@types/bcryptjs": "2.4.5",
     "@types/formidable": "3.4.3",
     "@types/fs-extra": "11.0.2",
     "@types/jest": "29.5.4",

packages/server/src/migrations/20190913171451_create.ts

@@ -98,7 +98,7 @@ export const up = async (db: DbConnection) => {
 	await db('users').insert({
 		id: adminId,
 		email: defaultAdminEmail,
-		password: hashPassword(defaultAdminPassword),
+		password: await hashPassword(defaultAdminPassword),
 		full_name: 'Admin',
 		is_admin: 1,
 		updated_time: now,

packages/server/src/models/BaseModel.ts

@@ -197,7 +197,7 @@ export default abstract class BaseModel<T> {
 	// The `name` argument is only for debugging, so that any stuck transaction
 	// can be more easily identified.
 	// eslint-disable-next-line @typescript-eslint/ban-types -- Old code before rule was applied
-	protected async withTransaction<T>(fn: Function, name: string): Promise<T> {
+	protected async withTransaction<T>(fn: Function, name = ''): Promise<T> {
 		const debugSteps = false;
 		const debugTimeout = true;
 		const timeoutMs = 10000;

packages/server/src/models/TaskStateModel.ts

@@ -21,13 +21,15 @@ export default class TaskStateModel extends BaseModel<TaskState> {
 	}
 
 	public async init(taskId: TaskId) {
-		const taskState: TaskState = await this.loadByTaskId(taskId);
-		if (taskState) return taskState;
+		return this.withTransaction(async () => {
+			const taskState: TaskState = await this.loadByTaskId(taskId);
+			if (taskState) return taskState;
 
-		return this.save({
-			task_id: taskId,
-			enabled: 1,
-			running: 0,
+			return this.save({
+				task_id: taskId,
+				enabled: 1,
+				running: 0,
+			});
 		});
 	}
 

packages/server/src/models/UserItemModel.ts

@@ -186,6 +186,9 @@ export default class UserItemModel extends BaseModel<UserItem> {
 			for (const userItem of userItems) {
 				const item = items.find(i => i.id === userItem.item_id);
 
+				// The item may have been deleted between the async calls above
+				if (!item) continue;
+
 				if (options.recordChanges && this.models().item().shouldRecordChange(item.name)) {
 					await this.models().change().save({
 						item_type: ItemType.UserItem,

packages/server/src/models/UserModel.ts

@@ -125,7 +125,7 @@ export default class UserModel extends BaseModel<User> {
 	public async login(email: string, password: string): Promise<User> {
 		const user = await this.loadByEmail(email);
 		if (!user) return null;
-		if (!checkPassword(password, user.password)) return null;
+		if (!(await checkPassword(password, user.password))) return null;
 		return user;
 	}
 
@@ -635,16 +635,23 @@ export default class UserModel extends BaseModel<User> {
 	public async save(object: User, options: SaveOptions = {}): Promise<User> {
 		const user = this.formatValues(object);
 
+		const isNew = await this.isNew(object, options);
+
 		if (user.password) {
 			if (isHashedPassword(user.password)) {
-				throw new ErrorBadRequest(`Unable to save user because password already seems to be hashed. User id: ${user.id}`);
+				if (!isNew) {
+					throw new ErrorBadRequest(`Unable to save user because password already seems to be hashed. User id: ${user.id}`);
+				} else {
+					// OK - We allow supplying an already hashed password for
+					// new users. This is mostly used for testing, because
+					// generating a bcrypt hash for each user is slow.
+				}
+			} else {
+				if (!options.skipValidation) this.validatePassword(user.password);
+				user.password = await hashPassword(user.password);
 			}
-			if (!options.skipValidation) this.validatePassword(user.password);
-			user.password = hashPassword(user.password);
 		}
 
-		const isNew = await this.isNew(object, options);
-
 		return this.withTransaction(async () => {
 			const savedUser = await super.save(user, options);
 

packages/server/src/utils/array.ts

@@ -1,7 +1,16 @@
-/* eslint-disable import/prefer-default-export */
-
 export function unique(array: any[]): any[] {
 	return array.filter((elem, index, self) => {
 		return index === self.indexOf(elem);
 	});
 }
+
+export const randomElement = <T>(array: T[]): T => {
+	if (!array || !array.length) return null;
+	return array[Math.floor(Math.random() * array.length)];
+};
+
+export const removeElement = (array: any[], element: any) => {
+	const index = array.indexOf(element);
+	if (index < 0) return;
+	array.splice(index, 1);
+};

packages/server/src/utils/auth.test.ts

@@ -12,7 +12,7 @@ describe('hashPassword', () => {
 			'$2a$10$LMKVPiNOWDZhtw9NizNIEuNGLsjOxQAcrwQJ0lnKuiaOtyFgZEnwO',
 		],
 	)('should return a string that starts with $2a$10 for the password: %', async (plainText) => {
-		expect(hashPassword(plainText).startsWith('$2a$10')).toBe(true);
+		expect((await hashPassword(plainText)).startsWith('$2a$10')).toBe(true);
 	});
 
 });

packages/server/src/utils/auth.ts

@@ -1,12 +1,12 @@
-const bcrypt = require('bcryptjs');
+import * as bcrypt from 'bcryptjs';
 
-export function hashPassword(password: string): string {
-	const salt = bcrypt.genSaltSync(10);
-	return bcrypt.hashSync(password, salt);
+export async function hashPassword(password: string): Promise<string> {
+	const salt = await bcrypt.genSalt(10);
+	return bcrypt.hash(password, salt);
 }
 
-export function checkPassword(password: string, hash: string): boolean {
-	return bcrypt.compareSync(password, hash);
+export async function checkPassword(password: string, hash: string): Promise<boolean> {
+	return bcrypt.compare(password, hash);
 }
 
 export const isHashedPassword = (password: string) => {

packages/server/src/utils/testing/populateDatabase.ts

@@ -0,0 +1,371 @@
+import { FolderEntity, NoteEntity } from '@joplin/lib/services/database/types';
+import Logger, { LogLevel, TargetType } from '@joplin/utils/Logger';
+import { User } from '../../services/database/types';
+import { randomElement } from '../array';
+import { CustomErrorCode } from '../errors';
+import { randomWords } from './randomWords';
+import { afterAllTests, beforeAllDb, createdDbPath, makeFolderSerializedBody, makeNoteSerializedBody, makeResourceSerializedBody, models, randomHash } from './testUtils';
+const { shimInit } = require('@joplin/lib/shim-init-node.js');
+const nodeSqlite = require('sqlite3');
+
+let logger_: Logger = null;
+
+const logger = () => {
+	if (!logger_) {
+		logger_ = new Logger();
+		logger_.addTarget(TargetType.Console);
+		logger_.setLevel(LogLevel.Debug);
+	}
+	return logger_;
+};
+
+export interface Options {
+	userCount?: number;
+	minNoteCountPerUser?: number;
+	maxNoteCountPerUser?: number;
+	minFolderCountPerUser?: number;
+	maxFolderCountPerUser?: number;
+}
+
+interface Context {
+	createdFolderIds: Record<string, string[]>;
+	createdNoteIds: Record<string, string[]>;
+	createdResourceIds: Record<string, string[]>;
+}
+
+enum Action {
+	CreateNote = 'createNote',
+	CreateFolder = 'createFolder',
+	CreateNoteAndResource = 'createNoteAndResource',
+	UpdateNote = 'updateNote',
+	UpdateFolder = 'updateFolder',
+	DeleteNote = 'deleteNote',
+	DeleteFolder = 'deleteFolder',
+}
+
+const createActions = [Action.CreateNote, Action.CreateFolder, Action.CreateNoteAndResource];
+const updateActions = [Action.UpdateNote, Action.UpdateFolder];
+const deleteActions = [Action.DeleteNote, Action.DeleteFolder];
+
+const isCreateAction = (action: Action) => {
+	return createActions.includes(action);
+};
+
+const isUpdateAction = (action: Action) => {
+	return updateActions.includes(action);
+};
+
+const isDeleteAction = (action: Action) => {
+	return deleteActions.includes(action);
+};
+
+type Reaction = (context: Context, user: User)=> Promise<boolean>;
+
+const randomInt = (min: number, max: number) => {
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+};
+
+const createRandomNote = async (user: User, note: NoteEntity = null) => {
+	const id = randomHash();
+	const itemName = `${id}.md`;
+
+	const serializedBody = makeNoteSerializedBody({
+		id,
+		title: randomWords(randomInt(1, 10)),
+		...note,
+	});
+
+	const result = await models().item().saveFromRawContent(user, {
+		name: itemName,
+		body: Buffer.from(serializedBody),
+	});
+
+	if (result[itemName].error) throw result[itemName].error;
+
+	return result[itemName].item;
+};
+
+const createRandomFolder = async (user: User, folder: FolderEntity = null) => {
+	const id = randomHash();
+	const itemName = `${id}.md`;
+
+	const serializedBody = makeFolderSerializedBody({
+		id,
+		title: randomWords(randomInt(1, 5)),
+		...folder,
+	});
+
+	const result = await models().item().saveFromRawContent(user, {
+		name: itemName,
+		body: Buffer.from(serializedBody),
+	});
+
+	if (result[itemName].error) throw result[itemName].error;
+
+	return result[itemName].item;
+};
+
+const reactions: Record<Action, Reaction> = {
+	[Action.CreateNote]: async (context, user) => {
+		const item = await createRandomNote(user);
+		if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
+		context.createdNoteIds[user.id].push(item.jop_id);
+		return true;
+	},
+
+	[Action.CreateFolder]: async (context, user) => {
+		const item = await createRandomFolder(user);
+		if (!context.createdFolderIds[user.id]) context.createdFolderIds[user.id] = [];
+		context.createdFolderIds[user.id].push(item.jop_id);
+		return true;
+	},
+
+	[Action.CreateNoteAndResource]: async (context, user) => {
+		const resourceContent = randomWords(20);
+		const resourceId = randomHash();
+
+		const metadataBody = makeResourceSerializedBody({
+			id: resourceId,
+			title: randomWords(5),
+			size: resourceContent.length,
+		});
+
+		await models().item().saveFromRawContent(user, {
+			name: `${resourceId}.md`,
+			body: Buffer.from(metadataBody),
+		});
+
+		await models().item().saveFromRawContent(user, {
+			name: `.resource/${resourceId}`,
+			body: Buffer.from(resourceContent),
+		});
+
+		if (!context.createdResourceIds[user.id]) context.createdResourceIds[user.id] = [];
+		context.createdResourceIds[user.id].push(resourceId);
+
+		const noteItem = await createRandomNote(user, {
+			body: `[](:/${resourceId})`,
+		});
+
+		if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
+		context.createdNoteIds[user.id].push(noteItem.jop_id);
+
+		return true;
+	},
+
+	[Action.UpdateNote]: async (context, user) => {
+		const noteId = randomElement(context.createdNoteIds[user.id]);
+		if (!noteId) return false;
+
+		try {
+			const noteItem = await models().item().loadByJopId(user.id, noteId);
+			const note = await models().item().loadAsJoplinItem(noteItem.id);
+			const serialized = makeNoteSerializedBody({
+				title: randomWords(10),
+				...note,
+			});
+
+			await models().item().saveFromRawContent(user, {
+				name: `${note.id}.md`,
+				body: Buffer.from(serialized),
+			});
+		} catch (error) {
+			if (error.code === CustomErrorCode.NotFound) return false;
+			throw error;
+		}
+
+		return true;
+	},
+
+	[Action.UpdateFolder]: async (context, user) => {
+		const folderId = randomElement(context.createdFolderIds[user.id]);
+		if (!folderId) return false;
+
+		try {
+			const folderItem = await models().item().loadByJopId(user.id, folderId);
+			const folder = await models().item().loadAsJoplinItem(folderItem.id);
+			const serialized = makeFolderSerializedBody({
+				title: randomWords(5),
+				...folder,
+			});
+
+			await models().item().saveFromRawContent(user, {
+				name: `${folder.id}.md`,
+				body: Buffer.from(serialized),
+			});
+		} catch (error) {
+			if (error.code === CustomErrorCode.NotFound) return false;
+			throw error;
+		}
+
+		return true;
+	},
+
+	[Action.DeleteNote]: async (context, user) => {
+		const noteId = randomElement(context.createdNoteIds[user.id]);
+		if (!noteId) return false;
+		const item = await models().item().loadByJopId(user.id, noteId, { fields: ['id'] });
+		await models().item().delete(item.id, { allowNoOp: true });
+		return true;
+	},
+
+	[Action.DeleteFolder]: async (context, user) => {
+		const folderId = randomElement(context.createdFolderIds[user.id]);
+		if (!folderId) return false;
+		const item = await models().item().loadByJopId(user.id, folderId, { fields: ['id'] });
+		await models().item().delete(item.id, { allowNoOp: true });
+		return true;
+	},
+};
+
+const randomActionKey = () => {
+	const r = Math.random();
+	if (r <= .5) {
+		return randomElement(createActions);
+	} else if (r <= .8) {
+		return randomElement(updateActions);
+	} else {
+		return randomElement(deleteActions);
+	}
+};
+
+const main = async (_options?: Options) => {
+	// options = {
+	// 	userCount: 10,
+	// 	minNoteCountPerUser: 0,
+	// 	maxNoteCountPerUser: 1000,
+	// 	minFolderCountPerUser: 0,
+	// 	maxFolderCountPerUser: 50,
+	// 	...options,
+	// };
+
+	shimInit({ nodeSqlite });
+	await beforeAllDb('populateDatabase');
+
+	logger().info(`Populating database: ${createdDbPath()}`);
+
+	const context: Context = {
+		createdNoteIds: {},
+		createdFolderIds: {},
+		createdResourceIds: {},
+	};
+
+	const report = {
+		created: 0,
+		updated: 0,
+		deleted: 0,
+	};
+
+	const updateReport = (action: Action) => {
+		if (isCreateAction(action)) report.created++;
+		if (isUpdateAction(action)) report.updated++;
+		if (isDeleteAction(action)) report.deleted++;
+	};
+
+	let users: User[] = [];
+
+	// -------------------------------------------------------------
+	// CREATE USERS
+	// -------------------------------------------------------------
+
+	{
+		const promises = [];
+
+		for (let i = 0; i < 20; i++) {
+			promises.push((async () => {
+				const user = await models().user().save({
+					full_name: `Toto ${i}`,
+					email: `toto${i}@example.com`,
+					password: '$2a$10$/2DMDnrx0PAspJ2DDnW/PO5x5M9H1abfSPsqxlPMhYiXgDi25751u', // Password = 111111
+				});
+
+				users.push(user);
+
+				logger().info(`Created user ${i}`);
+			})());
+		}
+
+		await Promise.all(promises);
+	}
+
+	users = await models().user().loadByIds(users.map(u => u.id));
+
+	// -------------------------------------------------------------
+	// CREATE NOTES, FOLDERS AND RESOURCES
+	// -------------------------------------------------------------
+
+	{
+		const promises = [];
+
+		for (let i = 0; i < 1000; i++) {
+			promises.push((async () => {
+				const user = randomElement(users);
+				const action = randomElement(createActions);
+				await reactions[action](context, user);
+				updateReport(action);
+				logger().info(`Done action ${i}: ${action}. User: ${user.email}`);
+			})());
+		}
+
+		await Promise.all(promises);
+	}
+
+	// -------------------------------------------------------------
+	// CREATE/UPDATE/DELETE NOTES, FOLDERS AND RESOURCES
+	// -------------------------------------------------------------
+
+	{
+		const promises = [];
+
+		for (let i = 0; i < 20000; i++) {
+			promises.push((async () => {
+				const user = randomElement(users);
+				const action = randomActionKey();
+				try {
+					const done = await reactions[action](context, user);
+					if (done) updateReport(action);
+					logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`);
+				} catch (error) {
+					error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`;
+					throw error;
+				}
+			})());
+		}
+
+		await Promise.all(promises);
+	}
+
+	// const changeIds = (await models().change().all()).map(c => c.id);
+
+	// const serverDir = (await getRootDir()) + '/packages/server';
+
+	// for (let i = 0; i < 100000; i++) {
+	// 	const user = randomElement(users);
+	// 	const cursor = Math.random() < .3 ? '' : randomElement(changeIds);
+
+	// 	try {
+	// 		const result1 = await models().change().delta(user.id, { cursor, limit: 1000 }, 1);
+	// 		const result2 = await models().change().delta(user.id, { cursor, limit: 1000 }, 2);
+
+	// 		logger().info('Test ' + i + ': Found ' + result1.items.length + ' and ' + result2.items.length + ' items');
+
+	// 		if (JSON.stringify(result1) !== JSON.stringify(result2)) {
+	// 			await writeFile(serverDir + '/result1.json', JSON.stringify(result1.items, null, '\t'));
+	// 			await writeFile(serverDir + '/result2.json', JSON.stringify(result2.items, null, '\t'));
+	// 			throw new Error('Found different results');
+	// 		}
+	// 	} catch (error) {
+	// 		error.message = 'User ' + user.id + ', Cursor ' + cursor + ': ' + error.message;
+	// 		throw error;
+	// 	}
+	// }
+
+	await afterAllTests();
+
+	logger().info(report);
+};
+
+main().catch((error) => {
+	logger().error('Fatal error', error);
+	process.exit(1);
+});

packages/server/src/utils/testing/testUtils.ts

@@ -15,7 +15,7 @@ import * as fs from 'fs-extra';
 import * as jsdom from 'jsdom';
 import setupAppContext from '../setupAppContext';
 import { ApiError } from '../errors';
-import { getApi, putApi } from './apiUtils';
+import { deleteApi, getApi, putApi } from './apiUtils';
 import { FolderEntity, NoteEntity, ResourceEntity } from '@joplin/lib/services/database/types';
 import { ModelType } from '@joplin/lib/BaseModel';
 import { initializeJoplinUtils } from '../joplinUtils';
@@ -73,6 +73,7 @@ export async function beforeAllDb(unitName: string, createDbOptions: CreateDbOpt
 	unitName = unitName.replace(/\//g, '_');
 
 	createdDbPath_ = `${packageRootDir}/db-test-${unitName}.sqlite`;
+	await fs.remove(createdDbPath_);
 
 	const tempDir = `${packageRootDir}/temp/test-${unitName}`;
 	await fs.mkdirp(tempDir);
@@ -111,6 +112,10 @@ export async function beforeAllDb(unitName: string, createDbOptions: CreateDbOpt
 	await initializeJoplinUtils(config(), models(), mustache);
 }
 
+export const createdDbPath = () => {
+	return createdDbPath_;
+};
+
 export async function afterAllTests() {
 	if (db_) {
 		await disconnectDb(db_);
@@ -237,7 +242,7 @@ export function koaNext(): Promise<void> {
 
 export const testAssetDir = `${packageRootDir}/assets/tests`;
 
-interface UserAndSession {
+export interface UserAndSession {
 	user: User;
 	session: Session;
 	password: string;
@@ -352,6 +357,10 @@ export async function updateItem(sessionId: string, path: string, content: strin
 	return models().item().load(item.id);
 }
 
+export async function deleteItem(sessionId: string, jopId: string): Promise<void> {
+	await deleteApi(sessionId, `items/root:/${jopId}.md:`);
+}
+
 export async function createNote(sessionId: string, note: NoteEntity): Promise<Item> {
 	note = {
 		id: '00000000000000000000000000000001',
@@ -561,7 +570,16 @@ type_: 2`;
 }
 
 export function makeResourceSerializedBody(resource: ResourceEntity = {}): string {
-	return `Test Resource
+	resource = {
+		id: randomHash(),
+		mime: 'plain/text',
+		file_extension: 'txt',
+		size: 0,
+		title: 'Test Resource',
+		...resource,
+	};
+
+	return `${resource.title}
 
 id: ${resource.id}
 mime: ${resource.mime}

packages/utils/Logger.ts

@@ -85,6 +85,14 @@ class Logger {
 		this.enabled_ = v;
 	}
 
+	public status(): string {
+		const output: string[] = [];
+		output.push(`Enabled: ${this.enabled}`);
+		output.push(`Level: ${this.level()}`);
+		output.push(`Targets: ${this.targets().map(t => t.type).join(', ')}`);
+		return output.join('\n');
+	}
+
 	public static initializeGlobalLogger(logger: Logger) {
 		this.globalLogger_ = logger;
 	}

yarn.lock

@@ -5136,6 +5136,7 @@ __metadata:
     "@joplin/utils": ~2.13
     "@koa/cors": 3.4.3
     "@rmp135/sql-ts": 1.18.0
+    "@types/bcryptjs": 2.4.5
     "@types/formidable": 3.4.3
     "@types/fs-extra": 11.0.2
     "@types/jest": 29.5.4
@@ -7721,6 +7722,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/bcryptjs@npm:2.4.5":
+  version: 2.4.5
+  resolution: "@types/bcryptjs@npm:2.4.5"
+  checksum: f721d72d8e1374ee2a342ce90cc902e2308cd059317af6e663d752537e704ea73bb119a2d34a6a68475f80abc1342635f48570119e0381f83a202724974f1e9f
+  languageName: node
+  linkType: hard
+
 "@types/body-parser@npm:*":
   version: 1.19.2
   resolution: "@types/body-parser@npm:1.19.2"