Server: Added report page

packages/server/src/routes/admin/reports.ts

@@ -0,0 +1,105 @@
+import { SubPath } from '../../utils/routeUtils';
+import Router from '../../utils/Router';
+import { RouteType } from '../../utils/types';
+import { AppContext } from '../../utils/types';
+import defaultView from '../../utils/defaultView';
+import { makeTableView, Row, Table } from '../../utils/views/table';
+import userActivity from '../../services/reports/userActivity';
+import { adminUserUrl, adminReportUrl } from '../../utils/urlUtils';
+import { Hour } from '../../utils/time';
+import { ErrorNotFound } from '../../utils/errors';
+import { ReportType } from '../../services/reports/types';
+
+const router: Router = new Router(RouteType.Web);
+
+interface Query {
+	intervalHours: number;
+}
+
+const parseQuery = (query: Record<string, string>) => {
+	const output: Query = {
+		intervalHours: 1,
+	};
+
+	if (query.intervalHours) output.intervalHours = Number(query.intervalHours);
+
+	return output;
+};
+
+router.get('admin/reports/:id', async (path: SubPath, ctx: AppContext) => {
+	const reportType = path.id;
+
+	if (reportType === ReportType.UserActivity) {
+		const query = parseQuery(ctx.query as Record<string, string>);
+
+		const changes = await userActivity(ctx.joplin.db, { interval: query.intervalHours * Hour });
+
+		const models = ctx.joplin.models;
+		const users = await models.user().loadByIds(changes.map(c => c.user_id), { fields: ['id', 'email'] });
+
+		const changeRows: Row[] = [];
+		for (const change of changes) {
+			const user = users.find(u => u.id === change.user_id);
+
+			changeRows.push({
+				items: [
+					{
+						value: user ? user.email : change.user_id,
+						url: adminUserUrl(change.user_id),
+					},
+					{
+						value: change.total_count.toString(),
+					},
+					{
+						value: change.create_count.toString(),
+					},
+					{
+						value: change.update_count.toString(),
+					},
+					{
+						value: change.delete_count.toString(),
+					},
+				],
+			});
+		}
+
+		const table: Table = {
+			headers: [
+				{
+					name: 'user_id',
+					label: 'User',
+				},
+				{
+					name: 'total_count',
+					label: 'Total',
+				},
+				{
+					name: 'created_count',
+					label: 'Created',
+				},
+				{
+					name: 'updated_count',
+					label: 'Updated',
+				},
+				{
+					name: 'deleted_count',
+					label: 'Deleted',
+				},
+			],
+			rows: changeRows,
+		};
+
+		return {
+			...defaultView(`admin/reports/${reportType}`, 'Report'),
+			content: {
+				itemTable: makeTableView(table),
+				getUrl: adminReportUrl(reportType),
+				intervalHours: query.intervalHours,
+			},
+		};
+	}
+
+	throw new ErrorNotFound(`No such report: ${path.id}`);
+});
+
+export default router;

packages/server/src/routes/routes.ts

@@ -17,6 +17,7 @@ import adminEmails from './admin/emails';
 import adminTasks from './admin/tasks';
 import adminUserDeletions from './admin/user_deletions';
 import adminUsers from './admin/users';
+import adminReports from './admin/reports';
 
 import indexChanges from './index/changes';
 import indexHelp from './index/help';
@@ -54,6 +55,7 @@ const routes: Routers = {
 	'admin/tasks': adminTasks,
 	'admin/user_deletions': adminUserDeletions,
 	'admin/users': adminUsers,
+	'admin/reports': adminReports,
 
 	'changes': indexChanges,
 	'help': indexHelp,

packages/server/src/services/MustacheService.ts

@@ -9,8 +9,9 @@ import { makeUrl, SubPath, UrlType } from '../utils/routeUtils';
 import MarkdownIt = require('markdown-it');
 import { headerAnchor } from '@joplin/renderer';
 import { _ } from '@joplin/lib/locale';
-import { adminDashboardUrl, adminEmailsUrl, adminTasksUrl, adminUserDeletionsUrl, adminUsersUrl, homeUrl, itemsUrl } from '../utils/urlUtils';
+import { adminDashboardUrl, adminEmailsUrl, adminTasksUrl, adminUserDeletionsUrl, adminUsersUrl, homeUrl, itemsUrl, adminReportUrl } from '../utils/urlUtils';
 import { MenuItem, setSelectedMenu } from '../utils/views/menu';
+import { ReportType } from './reports/types';
 
 export interface RenderOptions {
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
@@ -132,6 +133,10 @@ export default class MustacheService {
 						title: _('Emails'),
 						url: adminEmailsUrl(),
 					},
+					{
+						title: _('Reports'),
+						url: adminReportUrl(ReportType.UserActivity),
+					},
 				],
 			},
 		];

packages/server/src/services/reports/types.ts

@@ -0,0 +1,5 @@
+/* eslint-disable import/prefer-default-export */
+
+export enum ReportType {
+	UserActivity = 'user_activity',
+}

packages/server/src/services/reports/userActivity.test.ts

@@ -0,0 +1,67 @@
+import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, createFolder, updateFolder, dbSlave, deleteFolder } from '../../utils/testing/testUtils';
+import { Hour } from '../../utils/time';
+import userActivity from './userActivity';
+
+describe('reports/userActivity', () => {
+
+	beforeAll(async () => {
+		await beforeAllDb('reports/userActivity');
+	});
+
+	afterAll(async () => {
+		await afterAllTests();
+	});
+
+	beforeEach(async () => {
+		await beforeEachDb();
+	});
+
+	test('should create a report on user activity', async () => {
+		const { session: session1, user: user1 } = await createUserAndSession(1, false);
+		const { session: session2, user: user2 } = await createUserAndSession(2, false);
+
+		expect(await userActivity(dbSlave())).toEqual([]);
+
+		jest.useFakeTimers();
+
+		const t0 = new Date('2022-01-01 00:00:00').getTime();
+		jest.setSystemTime(t0);
+
+		await createFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1a' });
+		await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1b' });
+
+		const t1 = new Date('2022-01-01 02:00:00').getTime();
+		jest.setSystemTime(t1);
+
+		await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1c' });
+		await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1d' });
+		await deleteFolder(user1.id, '000000000000000000000000000000F1');
+
+		await createFolder(session2.id, { id: '000000000000000000000000000000F2', title: 'folder 2a' });
+		await updateFolder(session2.id, { id: '000000000000000000000000000000F2', title: 'folder 2b' });
+
+		const results = await userActivity(dbSlave(), { batchSize: 2, interval: 1 * Hour });
+
+		expect(results).toEqual(
+			[
+				{
+					user_id: user1.id,
+					total_count: 3,
+					create_count: 0,
+					update_count: 2,
+					delete_count: 1,
+				},
+				{
+					user_id: user2.id,
+					total_count: 2,
+					create_count: 1,
+					update_count: 1,
+					delete_count: 0,
+				},
+			],
+		);
+
+		jest.useRealTimers();
+	});
+
+});

packages/server/src/services/reports/userActivity.ts

@@ -0,0 +1,84 @@
+import { DbConnection } from '../../db';
+import { ChangeType, Uuid } from '../database/types';
+
+export interface Options {
+	interval: number;
+	batchSize?: number;
+}
+
+export default async (db: DbConnection, options: Options = null) => {
+	options = {
+		batchSize: 10000,
+		...options,
+	};
+
+	const cutOffTime = Date.now() - options.interval;
+
+	interface ChangeSlice {
+		user_id: Uuid;
+		updated_time: number;
+		counter: number;
+		type: ChangeType;
+	}
+
+	interface GroupedChange {
+		user_id: Uuid;
+		total_count: number;
+		create_count: number;
+		update_count: number;
+		delete_count: number;
+	}
+
+	let changes: ChangeSlice[] = [];
+	let counter = 0;
+
+	while (true) {
+		const query = db('changes')
+			.select('user_id', 'updated_time', 'counter', 'type')
+			.orderBy('counter', 'desc')
+			.limit(options.batchSize);
+
+		if (counter > 0) void query.where('counter', '<', counter);
+
+		const results: ChangeSlice[] = await query;
+
+		if (!results.length) break;
+
+		const filteredResults = results.filter(row => row.updated_time >= cutOffTime);
+
+		changes = changes.concat(filteredResults);
+
+		if (filteredResults.length !== results.length) break;
+
+		counter = filteredResults[filteredResults.length - 1].counter;
+	}
+
+	const groupedChanges: GroupedChange[] = [];
+	for (const c of changes) {
+		let grouped = groupedChanges.find(g => g.user_id === c.user_id);
+		if (!grouped) {
+			grouped = {
+				user_id: c.user_id,
+				total_count: 0,
+				create_count: 0,
+				update_count: 0,
+				delete_count: 0,
+			};
+
+			groupedChanges.push(grouped);
+		}
+
+		if (c.type === ChangeType.Create) grouped.create_count++;
+		if (c.type === ChangeType.Update) grouped.update_count++;
+		if (c.type === ChangeType.Delete) grouped.delete_count++;
+		grouped.total_count++;
+	}
+
+	groupedChanges.sort((a, b) => {
+		if (a.total_count > b.total_count) return -1;
+		if (a.total_count < b.total_count) return +1;
+		return 0;
+	});
+
+	return groupedChanges;
+};

packages/server/src/utils/routeUtils.ts

@@ -118,7 +118,7 @@ export function isPathBasedAddressing(fileId: string): boolean {
 
 export const urlMatchesSchema = (url: string, schema: string): boolean => {
 	url = stripOffQueryParameters(url);
-	const regex = new RegExp(`${schema.replace(/:id/, '[a-zA-Z0-9]+')}$`);
+	const regex = new RegExp(`${schema.replace(/:id/, '[a-zA-Z0-9_]+')}$`);
 	return !!url.match(regex);
 };
 
@@ -314,6 +314,7 @@ export enum UrlType {
 	Privacy = 'privacy',
 	Tasks = 'admin/tasks',
 	UserDeletions = 'admin/user_deletions',
+	Reports = 'admin/reports',
 }
 
 export function makeUrl(urlType: UrlType): string {

packages/server/src/utils/testing/populateDatabase.ts

@@ -319,22 +319,26 @@ const main = async (_options?: Options) => {
 	{
 		const promises = [];
 
-		for (let i = 0; i < 20000; i++) {
-			promises.push((async () => {
-				const user = randomElement(users);
-				const action = randomActionKey();
-				try {
-					const done = await reactions[action](context, user);
-					if (done) updateReport(action);
-					logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`);
-				} catch (error) {
-					error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`;
-					throw error;
-				}
-			})());
+		const totalActions = 5000;
+		const batchSize = 1000; // Don't change this - it will fail with higher numbers
+		const loopCount = Math.ceil(totalActions / batchSize);
+		for (let loopIndex = 0; loopIndex < loopCount; loopIndex++) {
+			for (let i = 0; i < batchSize; i++) {
+				promises.push((async () => {
+					const user = randomElement(users);
+					const action = randomActionKey();
+					try {
+						const done = await reactions[action](context, user);
+						if (done) updateReport(action);
+						logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`);
+					} catch (error) {
+						error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`;
+						logger().warn(error.message);
+					}
+				})());
+			}
+			await Promise.all(promises);
 		}
-
-		await Promise.all(promises);
 	}
 
 	// const changeIds = (await models().change().all()).map(c => c.id);

packages/server/src/utils/testing/testUtils.ts

@@ -434,6 +434,11 @@ export async function updateFolder(sessionId: string, folder: FolderEntity): Pro
 	return updateItem(sessionId, `root:/${folder.id}.md:`, makeFolderSerializedBody(folder));
 }
 
+export async function deleteFolder(userId: string, folderJopId: string): Promise<void> {
+	const item = await models().item().loadByJopId(userId, folderJopId, { fields: ['id'] });
+	await models().item().delete(item.id);
+}
+
 export async function createFolder(sessionId: string, folder: FolderEntity): Promise<Item> {
 	folder = {
 		id: '000000000000000000000000000000F1',

packages/server/src/utils/urlUtils.ts

@@ -1,6 +1,7 @@
 import { URL } from 'url';
 import config from '../config';
 import { Uuid } from '../services/database/types';
+import { ReportType } from '../services/reports/types';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 export function setQueryParameters(url: string, query: any): string {
@@ -94,3 +95,7 @@ export function adminEmailsUrl() {
 export function adminEmailUrl(id: number) {
 	return `${config().adminBaseUrl}/emails/${id}`;
 }
+
+export function adminReportUrl(type: ReportType) {
+	return `${config().adminBaseUrl}/reports/${type}`;
+}

packages/server/src/views/admin/reports/user_activity.mustache

@@ -0,0 +1,22 @@
+<div class="block">
+	<form method='GET' action="{{getUrl}}">
+		<div class="field">
+			<label class="label">Interval (hours)</label>
+			<div class="control">
+				<input name="intervalHours" id="intervalHours" class="input" type="text" placeholder="hours" value="{{intervalHours}}">
+			</div>
+		</div>
+
+		<div class="field is-grouped">
+			<div class="control">
+				<button class="button is-link">Update</button>
+			</div>
+		</div>
+	</form>
+</div>
+
+<div class="block">
+	{{#itemTable}}
+		{{>table}}
+	{{/itemTable}}
+</div>