self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2026-06-18 20:16:34 +02:00
Code Issues Releases Activity

Chore: Server: Debug: Add debug populateDatabase API (#13800)

Browse Source
This commit is contained in:
Henry Heino
2025-11-29 09:54:05 +00:00
committed by GitHub
parent 599f7a24ce
commit 2240cf77b5
12 changed files with 545 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/models/items/storage/testUtils.ts
+2 -1
View File
@@ -3,7 +3,8 @@
import config from '../../../config';
import { Item } from '../../../services/database/types';
import { CustomErrorCode } from '../../../utils/errors';
import { createUserAndSession, db, dbSlave, makeNoteSerializedBody, models } from '../../../utils/testing/testUtils';
import { makeNoteSerializedBody } from '../../../utils/testing/serializedItems';
import { createUserAndSession, db, dbSlave, models } from '../../../utils/testing/testUtils';
import { Config, StorageDriverConfig, StorageDriverMode } from '../../../utils/types';
import newModelFactory from '../../factory';
import loadStorageDriver from './loadStorageDriver';
packages/server/src/routes/api/debug.ts
+20 -1
View File
@@ -1,13 +1,17 @@
/* eslint-disable no-console */
import config from '../../config';
import { clearDatabase, createTestUsers, CreateTestUsersOptions, createUserDeletions } from '../../tools/debugTools';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { Env, RouteType } from '../../utils/types';
import { SubPath } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { ErrorForbidden } from '../../utils/errors';
import createTestUsers, { CreateTestUsersOptions } from '../../tools/debug/createTestUsers';
import createUserDeletions from '../../tools/debug/createUserDeletions';
import clearDatabase from '../../tools/debug/clearDatabase';
import populateDatabase from '../../tools/debug/populateDatabase';
import uuid from '@joplin/lib/uuid';
const router = new Router(RouteType.Api);
@@ -47,6 +51,21 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
if (query.action === 'clearKeyValues') {
await models.keyValue().deleteAll();
}
if (query.action === 'populateDatabase') {
const size = 'size' in query ? Number(query.size) : 1;
const actionCount = (() => {
if (size === 1) return 1024;
if (size === 2) return 4096;
return 1024 * 1024;
})();
await populateDatabase(ctx.joplin.models, {
actionCount,
userCount: 10,
userPrefix: uuid.create(),
});
}
});
export default router;
packages/server/src/routes/api/items.test.ts
+2 -1
View File
@@ -1,4 +1,4 @@
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models, createItem, makeTempFileWithContent, makeNoteSerializedBody, createItemTree, expectHttpError, createNote, expectNoHttpError, getItem, deleteItem, createBaseAppContext } from '../../utils/testing/testUtils';
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models, createItem, makeTempFileWithContent, createItemTree, expectHttpError, createNote, expectNoHttpError, getItem, deleteItem, createBaseAppContext } from '../../utils/testing/testUtils';
import { NoteEntity } from '@joplin/lib/services/database/types';
import { ModelType } from '@joplin/lib/BaseModel';
import { deleteApi, getApi, putApi } from '../../utils/testing/apiUtils';
@@ -8,6 +8,7 @@ import { shareFolderWithUser } from '../../utils/testing/shareApiUtils';
import { resourceBlobPath } from '../../utils/joplinUtils';
import { ErrorForbidden, ErrorPayloadTooLarge } from '../../utils/errors';
import { PaginatedResults } from '../../models/utils/pagination';
import { makeNoteSerializedBody } from '../../utils/testing/serializedItems';
describe('api/items', () => {
packages/server/src/routes/api/shares.folder.test.ts
+2 -1
View File
@@ -1,5 +1,5 @@
import { ChangeType, Session, Share, ShareType, ShareUser, ShareUserStatus } from '../../services/database/types';
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models, createNote, createFolder, updateItem, createItemTree, makeNoteSerializedBody, updateNote, expectHttpError, createResource, expectNotThrow } from '../../utils/testing/testUtils';
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models, createNote, createFolder, updateItem, createItemTree, updateNote, expectHttpError, createResource, expectNotThrow } from '../../utils/testing/testUtils';
import { postApi, patchApi, getApi, deleteApi } from '../../utils/testing/apiUtils';
import { PaginatedDeltaChanges } from '../../models/ChangeModel';
import { inviteUserToShare, shareFolderWithUser } from '../../utils/testing/shareApiUtils';
@@ -8,6 +8,7 @@ import { ErrorForbidden } from '../../utils/errors';
import { resourceBlobPath, serializeJoplinItem, unserializeJoplinItem } from '../../utils/joplinUtils';
import { PaginatedItems } from '../../models/ItemModel';
import { NoteEntity } from '@joplin/lib/services/database/types';
import { makeNoteSerializedBody } from '../../utils/testing/serializedItems';
const createSingleFolderShare = async (session1: Session, session2: Session) => {
const folderItem = await createFolder(session1.id, { title: 'created by sharer' });
packages/server/src/tools/debug/clearDatabase.ts
+6
View File
@@ -0,0 +1,6 @@
import { DbConnection, dropTables, migrateLatest } from '../../db';
export default async function clearDatabase(db: DbConnection) {
await dropTables(db);
await migrateLatest(db);
}
packages/server/src/tools/debugTools.ts → packages/server/src/tools/debug/createTestUsers.ts
+6 -36
View File
@@ -1,32 +1,15 @@
import time from '@joplin/lib/time';
import { DbConnection, dropTables, migrateLatest, truncateTables } from '../db';
import newModelFactory from '../models/factory';
import { AccountType } from '../models/UserModel';
import { User, UserFlagType } from '../services/database/types';
import { Minute, Second } from '../utils/time';
import { Config } from '../utils/types';
import { DbConnection, truncateTables } from '../../db';
import newModelFactory from '../../models/factory';
import { AccountType } from '../../models/UserModel';
import { User, UserFlagType } from '../../services/database/types';
import { Config } from '../../utils/types';
export interface CreateTestUsersOptions {
count?: number;
fromNum?: number;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
export async function handleDebugCommands(argv: any, db: DbConnection, config: Config): Promise<boolean> {
if (argv.debugCreateTestUsers) {
await createTestUsers(db, config);
} else {
return false;
}
return true;
}
export async function clearDatabase(db: DbConnection) {
await dropTables(db);
await migrateLatest(db);
}
const includedTables = [
'changes',
'emails',
@@ -46,7 +29,7 @@ const includedTables = [
'users',
];
export async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
export default async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
options = {
count: 0,
fromNum: 1,
@@ -125,16 +108,3 @@ export async function createTestUsers(db: DbConnection, config: Config, options:
}
}
}
export async function createUserDeletions(db: DbConnection, config: Config) {
const models = newModelFactory(db, db, config);
const users = await models.user().all();
for (let i = 0; i < 3; i++) {
if (i >= users.length) break;
if (users[i].is_admin) continue;
await models.userDeletion().add(users[i].id, Date.now() + 60 * Second + (i * 10 * Minute));
}
}
packages/server/src/tools/debug/createUserDeletions.ts
+17
View File
@@ -0,0 +1,17 @@
import { Minute, Second } from '@joplin/utils/time';
import { DbConnection } from '../../db';
import newModelFactory from '../../models/factory';
import { Config } from '../../utils/types';
export default async function createUserDeletions(db: DbConnection, config: Config) {
const models = newModelFactory(db, db, config);
const users = await models.user().all();
for (let i = 0; i < 3; i++) {
if (i >= users.length) break;
if (users[i].is_admin) continue;
await models.userDeletion().add(users[i].id, Date.now() + 60 * Second + (i * 10 * Minute));
}
}
packages/server/src/tools/debug/populateDatabase.ts
+394
View File
@@ -0,0 +1,394 @@
import { FolderEntity, NoteEntity } from '@joplin/lib/services/database/types';
import Logger, { LogLevel, TargetType } from '@joplin/utils/Logger';
import { User } from '../../services/database/types';
import { Models } from '../../models/factory';
import { randomWords } from '../../utils/testing/randomWords';
import { makeFolderSerializedBody, makeNoteSerializedBody, makeResourceSerializedBody } from '../../utils/testing/serializedItems';
import { randomElement } from '../../utils/array';
import { CustomErrorCode } from '../../utils/errors';
import uuid from '@joplin/lib/uuid';
let logger_: Logger = null;
const logger = () => {
if (!logger_) {
logger_ = new Logger();
logger_.addTarget(TargetType.Console);
logger_.setLevel(LogLevel.Debug);
}
return logger_;
};
export interface Options {
userPrefix: string;
userCount: number;
actionCount: number;
}
interface Context {
models: Models;
userEmails: string[];
createdFolderIds: Record<string, string[]>;
createdNoteIds: Record<string, string[]>;
createdResourceIds: Record<string, string[]>;
}
enum Action {
CreateNote = 'createNote',
CreateFolder = 'createFolder',
CreateAndShareFolder = 'createAndShareFolder',
CreateNoteAndResource = 'createNoteAndResource',
UpdateNote = 'updateNote',
UpdateFolder = 'updateFolder',
DeleteNote = 'deleteNote',
DeleteFolder = 'deleteFolder',
}
const createActions = [Action.CreateNote, Action.CreateFolder, Action.CreateAndShareFolder, Action.CreateNoteAndResource];
const updateActions = [Action.UpdateNote, Action.UpdateFolder];
const deleteActions = [Action.DeleteNote, Action.DeleteFolder];
const isCreateAction = (action: Action) => {
return createActions.includes(action);
};
const isUpdateAction = (action: Action) => {
return updateActions.includes(action);
};
const isDeleteAction = (action: Action) => {
return deleteActions.includes(action);
};
type Reaction = (context: Context, user: User)=> Promise<boolean>;
const randomInt = (min: number, max: number) => {
return Math.floor(Math.random() * (max - min + 1)) + min;
};
const joplinIdToItem = (context: Context, user: User, parentId: string) => {
return context.models.item().loadByName(user.id, `${parentId}.md`);
};
const createRandomNote = async (context: Context, user: User, note: NoteEntity = null) => {
const id = uuid.create();
const itemName = `${id}.md`;
const parentId = randomElement(context.createdFolderIds[user.id] ?? []) ?? '';
const parentItem = parentId ? await joplinIdToItem(context, user, parentId) : null;
const serializedBody = makeNoteSerializedBody({
id,
title: randomWords(randomInt(1, 10)),
parent_id: parentId,
share_id: parentItem?.jop_share_id,
...note,
});
const result = await context.models.item().saveFromRawContent(user, {
name: itemName,
body: Buffer.from(serializedBody),
});
if (result[itemName].error) throw result[itemName].error;
return result[itemName].item;
};
const createRandomFolder = async (context: Context, user: User, folder: FolderEntity = null) => {
const id = uuid.create();
const itemName = `${id}.md`;
const serializedBody = makeFolderSerializedBody({
id,
title: randomWords(randomInt(1, 5)),
...folder,
});
const result = await context.models.item().saveFromRawContent(user, {
name: itemName,
body: Buffer.from(serializedBody),
});
if (result[itemName].error) throw result[itemName].error;
const item = result[itemName].item;
if (!context.createdFolderIds[user.id]) context.createdFolderIds[user.id] = [];
context.createdFolderIds[user.id].push(item.jop_id);
return item;
};
const reactions: Record<Action, Reaction> = {
[Action.CreateNote]: async (context, user) => {
const item = await createRandomNote(context, user);
if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
context.createdNoteIds[user.id].push(item.jop_id);
return true;
},
[Action.CreateFolder]: async (context, user) => {
const item = await createRandomFolder(context, user);
if (!context.createdFolderIds[user.id]) context.createdFolderIds[user.id] = [];
context.createdFolderIds[user.id].push(item.jop_id);
return true;
},
[Action.CreateAndShareFolder]: async (context, user) => {
const item = await createRandomFolder(context, user);
const share = await context.models.share().shareFolder(user, item.jop_id, '');
const recipientEmail = randomElement(context.userEmails.filter(email => email !== user.email));
await context.models.shareUser().addByEmail(share.id, recipientEmail, '');
return true;
},
[Action.CreateNoteAndResource]: async (context, user) => {
const resourceContent = randomWords(20);
const resourceId = uuid.create();
const metadataBody = makeResourceSerializedBody({
id: resourceId,
title: randomWords(5),
size: resourceContent.length,
});
await context.models.item().saveFromRawContent(user, {
name: `${resourceId}.md`,
body: Buffer.from(metadataBody),
});
await context.models.item().saveFromRawContent(user, {
name: `.resource/${resourceId}`,
body: Buffer.from(resourceContent),
});
if (!context.createdResourceIds[user.id]) context.createdResourceIds[user.id] = [];
context.createdResourceIds[user.id].push(resourceId);
const noteItem = await createRandomNote(context, user, {
body: `[](:/${resourceId})`,
});
if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
context.createdNoteIds[user.id].push(noteItem.jop_id);
return true;
},
[Action.UpdateNote]: async (context, user) => {
const noteId = randomElement(context.createdNoteIds[user.id]);
if (!noteId) return false;
try {
const noteItem = await context.models.item().loadByJopId(user.id, noteId);
const note = await context.models.item().loadAsJoplinItem(noteItem.id);
const serialized = makeNoteSerializedBody({
title: randomWords(10),
...note,
});
await context.models.item().saveFromRawContent(user, {
name: `${note.id}.md`,
body: Buffer.from(serialized),
});
} catch (error) {
if (error.code === CustomErrorCode.NotFound) return false;
throw error;
}
return true;
},
[Action.UpdateFolder]: async (context, user) => {
const folderId = randomElement(context.createdFolderIds[user.id]);
if (!folderId) return false;
try {
const folderItem = await context.models.item().loadByJopId(user.id, folderId);
const folder = await context.models.item().loadAsJoplinItem(folderItem.id);
const serialized = makeFolderSerializedBody({
title: randomWords(5),
...folder,
});
await context.models.item().saveFromRawContent(user, {
name: `${folder.id}.md`,
body: Buffer.from(serialized),
});
} catch (error) {
if (error.code === CustomErrorCode.NotFound) return false;
throw error;
}
return true;
},
[Action.DeleteNote]: async (context, user) => {
const noteId = randomElement(context.createdNoteIds[user.id]);
if (!noteId) return false;
const item = await context.models.item().loadByJopId(user.id, noteId, { fields: ['id'] });
await context.models.item().delete(item.id, { allowNoOp: true });
return true;
},
[Action.DeleteFolder]: async (context, user) => {
const folderId = randomElement(context.createdFolderIds[user.id]);
if (!folderId) return false;
const item = await context.models.item().loadByJopId(user.id, folderId, { fields: ['id'] });
await context.models.item().delete(item.id, { allowNoOp: true });
return true;
},
};
const randomActionKey = () => {
const r = Math.random();
if (r <= .35) {
return randomElement(createActions);
} else if (r <= .8) {
return randomElement(updateActions);
} else {
return randomElement(deleteActions);
}
};
const populateDatabase = async (models: Models, options: Options) => {
logger().info('Populating database...');
const createActionCount = Math.ceil(options.actionCount / 16);
const postCreateActionCount = options.actionCount - createActionCount;
const context: Context = {
models,
userEmails: [],
createdNoteIds: {},
createdFolderIds: {},
createdResourceIds: {},
};
const report = {
created: 0,
updated: 0,
deleted: 0,
};
const updateReport = (action: Action) => {
if (isCreateAction(action)) report.created++;
if (isUpdateAction(action)) report.updated++;
if (isDeleteAction(action)) report.deleted++;
};
let users: User[] = [];
// -------------------------------------------------------------
// CREATE USERS
// -------------------------------------------------------------
{
const promises = [];
// cSpell:disable
for (let i = 0; i < options.userCount; i++) {
promises.push((async () => {
const email = `${options.userPrefix}-toto${i}@example.com`;
const user = await models.user().save({
full_name: `Toto ${i}`,
email,
password: '$2a$10$/2DMDnrx0PAspJ2DDnW/PO5x5M9H1abfSPsqxlPMhYiXgDi25751u', // Password = 111111
});
users.push(user);
context.userEmails.push(email);
logger().info(`Created user ${i}`);
})());
}
// cSpell:enable
await Promise.all(promises);
}
users = await models.user().loadByIds(users.map(u => u.id));
// -------------------------------------------------------------
// CREATE NOTES, FOLDERS AND RESOURCES
// -------------------------------------------------------------
{
// Keep the batch size small. Doing so allows future actions to use a larger number
// of parent items.
const batchSize = 10;
for (let i = 0; i < createActionCount; i += batchSize) {
const promises = [];
for (let j = 0; j < batchSize; j++) {
promises.push((async () => {
const user = randomElement(users);
const action = randomElement(createActions);
await reactions[action](context, user);
updateReport(action);
logger().info(`Done action ${i}: ${action}. User: ${user.email}`);
})());
}
await Promise.all(promises);
}
}
// -------------------------------------------------------------
// CREATE/UPDATE/DELETE NOTES, FOLDERS AND RESOURCES
// -------------------------------------------------------------
{
const promises = [];
const batchSize = 200; // Don't change this - it will fail with higher numbers
const loopCount = Math.ceil(postCreateActionCount / batchSize);
for (let loopIndex = 0; loopIndex < loopCount; loopIndex++) {
for (let i = 0; i < batchSize; i++) {
promises.push((async () => {
const user = randomElement(users);
const action = randomActionKey();
try {
const done = await reactions[action](context, user);
if (done) updateReport(action);
logger().info(`Done action ${loopIndex}.${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`);
} catch (error) {
error.message = `Could not do action ${loopIndex}.${i}: ${action}. User: ${user.email}: ${error.message}`;
logger().warn(error.message);
}
})());
}
await Promise.all(promises);
}
}
// const changeIds = (await models.change().all()).map(c => c.id);
// const serverDir = (await getRootDir()) + '/packages/server';
// for (let i = 0; i < 100000; i++) {
// const user = randomElement(users);
// const cursor = Math.random() < .3 ? '' : randomElement(changeIds);
// try {
// const result1 = await models.change().delta(user.id, { cursor, limit: 1000 }, 1);
// const result2 = await models.change().delta(user.id, { cursor, limit: 1000 }, 2);
// logger().info('Test ' + i + ': Found ' + result1.items.length + ' and ' + result2.items.length + ' items');
// if (JSON.stringify(result1) !== JSON.stringify(result2)) {
// await writeFile(serverDir + '/result1.json', JSON.stringify(result1.items, null, '\t'));
// await writeFile(serverDir + '/result2.json', JSON.stringify(result2.items, null, '\t'));
// throw new Error('Found different results');
// }
// } catch (error) {
// error.message = 'User ' + user.id + ', Cursor ' + cursor + ': ' + error.message;
// throw error;
// }
// }
logger().info(report);
};
export default populateDatabase;
packages/server/src/utils/testing/populateDatabase.ts
+8 -340
View File
@@ -1,10 +1,6 @@
import { FolderEntity, NoteEntity } from '@joplin/lib/services/database/types';
import { afterAllTests, beforeAllDb, createdDbPath, models } from './testUtils';
import populateDatabase from '../../tools/debug/populateDatabase';
import Logger, { LogLevel, TargetType } from '@joplin/utils/Logger';
import { User } from '../../services/database/types';
import { randomElement } from '../array';
import { CustomErrorCode } from '../errors';
import { randomWords } from './randomWords';
import { afterAllTests, beforeAllDb, createdDbPath, makeFolderSerializedBody, makeNoteSerializedBody, makeResourceSerializedBody, models, randomHash } from './testUtils';
const { shimInit } = require('@joplin/lib/shim-init-node.js');
const nodeSqlite = require('sqlite3');
@@ -19,217 +15,7 @@ const logger = () => {
return logger_;
};
export interface Options {
userCount?: number;
minNoteCountPerUser?: number;
maxNoteCountPerUser?: number;
minFolderCountPerUser?: number;
maxFolderCountPerUser?: number;
}
interface Context {
createdFolderIds: Record<string, string[]>;
createdNoteIds: Record<string, string[]>;
createdResourceIds: Record<string, string[]>;
}
enum Action {
CreateNote = 'createNote',
CreateFolder = 'createFolder',
CreateNoteAndResource = 'createNoteAndResource',
UpdateNote = 'updateNote',
UpdateFolder = 'updateFolder',
DeleteNote = 'deleteNote',
DeleteFolder = 'deleteFolder',
}
const createActions = [Action.CreateNote, Action.CreateFolder, Action.CreateNoteAndResource];
const updateActions = [Action.UpdateNote, Action.UpdateFolder];
const deleteActions = [Action.DeleteNote, Action.DeleteFolder];
const isCreateAction = (action: Action) => {
return createActions.includes(action);
};
const isUpdateAction = (action: Action) => {
return updateActions.includes(action);
};
const isDeleteAction = (action: Action) => {
return deleteActions.includes(action);
};
type Reaction = (context: Context, user: User)=> Promise<boolean>;
const randomInt = (min: number, max: number) => {
return Math.floor(Math.random() * (max - min + 1)) + min;
};
const createRandomNote = async (user: User, note: NoteEntity = null) => {
const id = randomHash();
const itemName = `${id}.md`;
const serializedBody = makeNoteSerializedBody({
id,
title: randomWords(randomInt(1, 10)),
...note,
});
const result = await models().item().saveFromRawContent(user, {
name: itemName,
body: Buffer.from(serializedBody),
});
if (result[itemName].error) throw result[itemName].error;
return result[itemName].item;
};
const createRandomFolder = async (user: User, folder: FolderEntity = null) => {
const id = randomHash();
const itemName = `${id}.md`;
const serializedBody = makeFolderSerializedBody({
id,
title: randomWords(randomInt(1, 5)),
...folder,
});
const result = await models().item().saveFromRawContent(user, {
name: itemName,
body: Buffer.from(serializedBody),
});
if (result[itemName].error) throw result[itemName].error;
return result[itemName].item;
};
const reactions: Record<Action, Reaction> = {
[Action.CreateNote]: async (context, user) => {
const item = await createRandomNote(user);
if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
context.createdNoteIds[user.id].push(item.jop_id);
return true;
},
[Action.CreateFolder]: async (context, user) => {
const item = await createRandomFolder(user);
if (!context.createdFolderIds[user.id]) context.createdFolderIds[user.id] = [];
context.createdFolderIds[user.id].push(item.jop_id);
return true;
},
[Action.CreateNoteAndResource]: async (context, user) => {
const resourceContent = randomWords(20);
const resourceId = randomHash();
const metadataBody = makeResourceSerializedBody({
id: resourceId,
title: randomWords(5),
size: resourceContent.length,
});
await models().item().saveFromRawContent(user, {
name: `${resourceId}.md`,
body: Buffer.from(metadataBody),
});
await models().item().saveFromRawContent(user, {
name: `.resource/${resourceId}`,
body: Buffer.from(resourceContent),
});
if (!context.createdResourceIds[user.id]) context.createdResourceIds[user.id] = [];
context.createdResourceIds[user.id].push(resourceId);
const noteItem = await createRandomNote(user, {
body: `[](:/${resourceId})`,
});
if (!context.createdNoteIds[user.id]) context.createdNoteIds[user.id] = [];
context.createdNoteIds[user.id].push(noteItem.jop_id);
return true;
},
[Action.UpdateNote]: async (context, user) => {
const noteId = randomElement(context.createdNoteIds[user.id]);
if (!noteId) return false;
try {
const noteItem = await models().item().loadByJopId(user.id, noteId);
const note = await models().item().loadAsJoplinItem(noteItem.id);
const serialized = makeNoteSerializedBody({
title: randomWords(10),
...note,
});
await models().item().saveFromRawContent(user, {
name: `${note.id}.md`,
body: Buffer.from(serialized),
});
} catch (error) {
if (error.code === CustomErrorCode.NotFound) return false;
throw error;
}
return true;
},
[Action.UpdateFolder]: async (context, user) => {
const folderId = randomElement(context.createdFolderIds[user.id]);
if (!folderId) return false;
try {
const folderItem = await models().item().loadByJopId(user.id, folderId);
const folder = await models().item().loadAsJoplinItem(folderItem.id);
const serialized = makeFolderSerializedBody({
title: randomWords(5),
...folder,
});
await models().item().saveFromRawContent(user, {
name: `${folder.id}.md`,
body: Buffer.from(serialized),
});
} catch (error) {
if (error.code === CustomErrorCode.NotFound) return false;
throw error;
}
return true;
},
[Action.DeleteNote]: async (context, user) => {
const noteId = randomElement(context.createdNoteIds[user.id]);
if (!noteId) return false;
const item = await models().item().loadByJopId(user.id, noteId, { fields: ['id'] });
await models().item().delete(item.id, { allowNoOp: true });
return true;
},
[Action.DeleteFolder]: async (context, user) => {
const folderId = randomElement(context.createdFolderIds[user.id]);
if (!folderId) return false;
const item = await models().item().loadByJopId(user.id, folderId, { fields: ['id'] });
await models().item().delete(item.id, { allowNoOp: true });
return true;
},
};
const randomActionKey = () => {
const r = Math.random();
if (r <= .5) {
return randomElement(createActions);
} else if (r <= .8) {
return randomElement(updateActions);
} else {
return randomElement(deleteActions);
}
};
const main = async (_options?: Options) => {
const main = async () => {
// options = {
// userCount: 10,
// minNoteCountPerUser: 0,
@@ -244,131 +30,13 @@ const main = async (_options?: Options) => {
logger().info(`Populating database: ${createdDbPath()}`);
const context: Context = {
createdNoteIds: {},
createdFolderIds: {},
createdResourceIds: {},
};
const report = {
created: 0,
updated: 0,
deleted: 0,
};
const updateReport = (action: Action) => {
if (isCreateAction(action)) report.created++;
if (isUpdateAction(action)) report.updated++;
if (isDeleteAction(action)) report.deleted++;
};
let users: User[] = [];
// -------------------------------------------------------------
// CREATE USERS
// -------------------------------------------------------------
{
const promises = [];
// cSpell:disable
for (let i = 0; i < 20; i++) {
promises.push((async () => {
const user = await models().user().save({
full_name: `Toto ${i}`,
email: `toto${i}@example.com`,
password: '$2a$10$/2DMDnrx0PAspJ2DDnW/PO5x5M9H1abfSPsqxlPMhYiXgDi25751u', // Password = 111111
});
users.push(user);
logger().info(`Created user ${i}`);
})());
}
// cSpell:enable
await Promise.all(promises);
}
users = await models().user().loadByIds(users.map(u => u.id));
// -------------------------------------------------------------
// CREATE NOTES, FOLDERS AND RESOURCES
// -------------------------------------------------------------
{
const promises = [];
for (let i = 0; i < 1000; i++) {
promises.push((async () => {
const user = randomElement(users);
const action = randomElement(createActions);
await reactions[action](context, user);
updateReport(action);
logger().info(`Done action ${i}: ${action}. User: ${user.email}`);
})());
}
await Promise.all(promises);
}
// -------------------------------------------------------------
// CREATE/UPDATE/DELETE NOTES, FOLDERS AND RESOURCES
// -------------------------------------------------------------
{
const promises = [];
const totalActions = 5000;
const batchSize = 1000; // Don't change this - it will fail with higher numbers
const loopCount = Math.ceil(totalActions / batchSize);
for (let loopIndex = 0; loopIndex < loopCount; loopIndex++) {
for (let i = 0; i < batchSize; i++) {
promises.push((async () => {
const user = randomElement(users);
const action = randomActionKey();
try {
const done = await reactions[action](context, user);
if (done) updateReport(action);
logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`);
} catch (error) {
error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`;
logger().warn(error.message);
}
})());
}
await Promise.all(promises);
}
}
// const changeIds = (await models().change().all()).map(c => c.id);
// const serverDir = (await getRootDir()) + '/packages/server';
// for (let i = 0; i < 100000; i++) {
// const user = randomElement(users);
// const cursor = Math.random() < .3 ? '' : randomElement(changeIds);
// try {
// const result1 = await models().change().delta(user.id, { cursor, limit: 1000 }, 1);
// const result2 = await models().change().delta(user.id, { cursor, limit: 1000 }, 2);
// logger().info('Test ' + i + ': Found ' + result1.items.length + ' and ' + result2.items.length + ' items');
// if (JSON.stringify(result1) !== JSON.stringify(result2)) {
// await writeFile(serverDir + '/result1.json', JSON.stringify(result1.items, null, '\t'));
// await writeFile(serverDir + '/result2.json', JSON.stringify(result2.items, null, '\t'));
// throw new Error('Found different results');
// }
// } catch (error) {
// error.message = 'User ' + user.id + ', Cursor ' + cursor + ': ' + error.message;
// throw error;
// }
// }
await populateDatabase(models(), {
userPrefix: 'test',
actionCount: 6000,
userCount: 10,
});
await afterAllTests();
logger().info(report);
};
main().catch((error) => {
packages/server/src/utils/testing/serializedItems.ts
+85
View File
@@ -0,0 +1,85 @@
import { FolderEntity, NoteEntity, ResourceEntity } from '@joplin/lib/services/database/types';
import uuid from '@joplin/lib/uuid';
export function makeNoteSerializedBody(note: NoteEntity = {}): string {
return `${'title' in note ? note.title : 'Title'}
${'body' in note ? note.body : 'Body'}
id: ${'id' in note ? note.id : 'b39dadd7a63742bebf3125fd2a9286d4'}
parent_id: ${'parent_id' in note ? note.parent_id : '000000000000000000000000000000F1'}
created_time: 2020-10-15T10:34:16.044Z
updated_time: 2021-01-28T23:10:30.054Z
is_conflict: 0
latitude: 0.00000000
longitude: 0.00000000
altitude: 0.0000
author:
source_url:
is_todo: 1
todo_due: 1602760405000
todo_completed: 0
source: joplindev-desktop
source_application: net.cozic.joplindev-desktop
application_data:
order: 0
user_created_time: 2020-10-15T10:34:16.044Z
user_updated_time: 2020-10-19T17:21:03.394Z
encryption_cipher_text:
encryption_applied: 0
markup_language: 1
is_shared: 1
share_id: ${note.share_id || ''}
conflict_original_id:
master_key_id:
user_data:
deleted_time: 0
type_: 1`;
}
export function makeFolderSerializedBody(folder: FolderEntity = {}): string {
return `${'title' in folder ? folder.title : 'Title'}
id: ${folder.id || '000000000000000000000000000000F1'}
created_time: 2020-11-11T18:44:14.534Z
updated_time: 2020-11-11T18:44:14.534Z
user_created_time: 2020-11-11T18:44:14.534Z
user_updated_time: 2020-11-11T18:44:14.534Z
encryption_cipher_text:
encryption_applied: 0
parent_id: ${folder.parent_id || ''}
is_shared: 0
share_id: ${folder.share_id || ''}
user_data:
type_: 2`;
}
export function makeResourceSerializedBody(resource: ResourceEntity = {}): string {
resource = {
id: uuid.create(),
mime: 'plain/text',
file_extension: 'txt',
size: 0,
title: 'Test Resource',
...resource,
};
return `${resource.title}
id: ${resource.id}
mime: ${resource.mime}
filename:
created_time: 2020-10-15T10:37:58.090Z
updated_time: 2020-10-15T10:37:58.090Z
user_created_time: 2020-10-15T10:37:58.090Z
user_updated_time: 2020-10-15T10:37:58.090Z
file_extension: ${resource.file_extension}
encryption_cipher_text:
encryption_applied: 0
encryption_blob_encrypted: 0
size: ${resource.size}
share_id: ${resource.share_id || ''}
is_shared: 0
type_: 4`;
}
packages/server/src/utils/testing/shareApiUtils.ts
+2 -1
View File
@@ -4,7 +4,8 @@ import { Item, Share, ShareType, ShareUser, ShareUserStatus, User, Uuid } from '
import routeHandler from '../../middleware/routeHandler';
import { AppContext } from '../types';
import { patchApi, postApi } from './apiUtils';
import { checkContextError, createFolder, createItem, koaAppContext, models, makeFolderSerializedBody, makeNoteSerializedBody, updateFolder, createResource } from './testUtils';
import { checkContextError, createFolder, createItem, koaAppContext, models, updateFolder, createResource } from './testUtils';
import { makeFolderSerializedBody, makeNoteSerializedBody } from './serializedItems';
interface ShareResult {
share: Share;
packages/server/src/utils/testing/testUtils.ts
+1 -82
View File
@@ -26,6 +26,7 @@ import { cookieSet } from '../cookies';
import { parseEnv } from '../../env';
import { URL } from 'url';
import initLib from '@joplin/lib/initLib';
import { makeFolderSerializedBody, makeNoteSerializedBody, makeResourceSerializedBody } from './serializedItems';
// Takes into account the fact that this file will be inside the /dist directory
// when it runs.
@@ -582,88 +583,6 @@ export async function expectNoHttpError(asyncFn: Function): Promise<void> {
}
}
export function makeNoteSerializedBody(note: NoteEntity = {}): string {
return `${'title' in note ? note.title : 'Title'}
${'body' in note ? note.body : 'Body'}
id: ${'id' in note ? note.id : 'b39dadd7a63742bebf3125fd2a9286d4'}
parent_id: ${'parent_id' in note ? note.parent_id : '000000000000000000000000000000F1'}
created_time: 2020-10-15T10:34:16.044Z
updated_time: 2021-01-28T23:10:30.054Z
is_conflict: 0
latitude: 0.00000000
longitude: 0.00000000
altitude: 0.0000
author:
source_url:
is_todo: 1
todo_due: 1602760405000
todo_completed: 0
source: joplindev-desktop
source_application: net.cozic.joplindev-desktop
application_data:
order: 0
user_created_time: 2020-10-15T10:34:16.044Z
user_updated_time: 2020-10-19T17:21:03.394Z
encryption_cipher_text:
encryption_applied: 0
markup_language: 1
is_shared: 1
share_id: ${note.share_id || ''}
conflict_original_id:
master_key_id:
user_data:
deleted_time: 0
type_: 1`;
}
export function makeFolderSerializedBody(folder: FolderEntity = {}): string {
return `${'title' in folder ? folder.title : 'Title'}
id: ${folder.id || '000000000000000000000000000000F1'}
created_time: 2020-11-11T18:44:14.534Z
updated_time: 2020-11-11T18:44:14.534Z
user_created_time: 2020-11-11T18:44:14.534Z
user_updated_time: 2020-11-11T18:44:14.534Z
encryption_cipher_text:
encryption_applied: 0
parent_id: ${folder.parent_id || ''}
is_shared: 0
share_id: ${folder.share_id || ''}
user_data:
type_: 2`;
}
export function makeResourceSerializedBody(resource: ResourceEntity = {}): string {
resource = {
id: randomHash(),
mime: 'plain/text',
file_extension: 'txt',
size: 0,
title: 'Test Resource',
...resource,
};
return `${resource.title}
id: ${resource.id}
mime: ${resource.mime}
filename:
created_time: 2020-10-15T10:37:58.090Z
updated_time: 2020-10-15T10:37:58.090Z
user_created_time: 2020-10-15T10:37:58.090Z
user_updated_time: 2020-10-15T10:37:58.090Z
file_extension: ${resource.file_extension}
encryption_cipher_text:
encryption_applied: 0
encryption_blob_encrypted: 0
size: ${resource.size}
share_id: ${resource.share_id || ''}
is_shared: 0
type_: 4`;
}
// eslint-disable-next-line @typescript-eslint/ban-types -- Old code before rule was applied
export async function expectNotThrow(asyncFn: Function) {
let thrownError = null;