From 7ad3b34ec3b98fd654a3dd84da2d1d7ab2b63bf6 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 13 May 2024 11:32:53 +0100 Subject: [PATCH] Server: Added report page --- packages/server/src/routes/admin/reports.ts | 105 ++++++++++++++++++ packages/server/src/routes/routes.ts | 2 + .../server/src/services/MustacheService.ts | 7 +- packages/server/src/services/reports/types.ts | 5 + .../src/services/reports/userActivity.test.ts | 67 +++++++++++ .../src/services/reports/userActivity.ts | 84 ++++++++++++++ packages/server/src/utils/routeUtils.ts | 3 +- .../src/utils/testing/populateDatabase.ts | 34 +++--- .../server/src/utils/testing/testUtils.ts | 5 + packages/server/src/utils/urlUtils.ts | 5 + .../admin/reports/user_activity.mustache | 22 ++++ 11 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 packages/server/src/routes/admin/reports.ts create mode 100644 packages/server/src/services/reports/types.ts create mode 100644 packages/server/src/services/reports/userActivity.test.ts create mode 100644 packages/server/src/services/reports/userActivity.ts create mode 100644 packages/server/src/views/admin/reports/user_activity.mustache diff --git a/packages/server/src/routes/admin/reports.ts b/packages/server/src/routes/admin/reports.ts new file mode 100644 index 000000000..f234bfc5c --- /dev/null +++ b/packages/server/src/routes/admin/reports.ts @@ -0,0 +1,105 @@ +import { SubPath } from '../../utils/routeUtils'; +import Router from '../../utils/Router'; +import { RouteType } from '../../utils/types'; +import { AppContext } from '../../utils/types'; +import defaultView from '../../utils/defaultView'; +import { makeTableView, Row, Table } from '../../utils/views/table'; +import userActivity from '../../services/reports/userActivity'; +import { adminUserUrl, adminReportUrl } from '../../utils/urlUtils'; +import { Hour } from '../../utils/time'; +import { ErrorNotFound } from '../../utils/errors'; +import { ReportType } from '../../services/reports/types'; + +const router: Router = new Router(RouteType.Web); + +interface Query { + intervalHours: number; +} + +const parseQuery = (query: Record) => { + const output: Query = { + intervalHours: 1, + }; + + if (query.intervalHours) output.intervalHours = Number(query.intervalHours); + + return output; +}; + +router.get('admin/reports/:id', async (path: SubPath, ctx: AppContext) => { + const reportType = path.id; + + if (reportType === ReportType.UserActivity) { + const query = parseQuery(ctx.query as Record); + + const changes = await userActivity(ctx.joplin.db, { interval: query.intervalHours * Hour }); + + const models = ctx.joplin.models; + const users = await models.user().loadByIds(changes.map(c => c.user_id), { fields: ['id', 'email'] }); + + const changeRows: Row[] = []; + for (const change of changes) { + const user = users.find(u => u.id === change.user_id); + + changeRows.push({ + items: [ + { + value: user ? user.email : change.user_id, + url: adminUserUrl(change.user_id), + }, + { + value: change.total_count.toString(), + }, + { + value: change.create_count.toString(), + }, + { + value: change.update_count.toString(), + }, + { + value: change.delete_count.toString(), + }, + ], + }); + } + + const table: Table = { + headers: [ + { + name: 'user_id', + label: 'User', + }, + { + name: 'total_count', + label: 'Total', + }, + { + name: 'created_count', + label: 'Created', + }, + { + name: 'updated_count', + label: 'Updated', + }, + { + name: 'deleted_count', + label: 'Deleted', + }, + ], + rows: changeRows, + }; + + return { + ...defaultView(`admin/reports/${reportType}`, 'Report'), + content: { + itemTable: makeTableView(table), + getUrl: adminReportUrl(reportType), + intervalHours: query.intervalHours, + }, + }; + } + + throw new ErrorNotFound(`No such report: ${path.id}`); +}); + +export default router; diff --git a/packages/server/src/routes/routes.ts b/packages/server/src/routes/routes.ts index f12bbbb15..7c12b9481 100644 --- a/packages/server/src/routes/routes.ts +++ b/packages/server/src/routes/routes.ts @@ -17,6 +17,7 @@ import adminEmails from './admin/emails'; import adminTasks from './admin/tasks'; import adminUserDeletions from './admin/user_deletions'; import adminUsers from './admin/users'; +import adminReports from './admin/reports'; import indexChanges from './index/changes'; import indexHelp from './index/help'; @@ -54,6 +55,7 @@ const routes: Routers = { 'admin/tasks': adminTasks, 'admin/user_deletions': adminUserDeletions, 'admin/users': adminUsers, + 'admin/reports': adminReports, 'changes': indexChanges, 'help': indexHelp, diff --git a/packages/server/src/services/MustacheService.ts b/packages/server/src/services/MustacheService.ts index 108beae65..6672217d3 100644 --- a/packages/server/src/services/MustacheService.ts +++ b/packages/server/src/services/MustacheService.ts @@ -9,8 +9,9 @@ import { makeUrl, SubPath, UrlType } from '../utils/routeUtils'; import MarkdownIt = require('markdown-it'); import { headerAnchor } from '@joplin/renderer'; import { _ } from '@joplin/lib/locale'; -import { adminDashboardUrl, adminEmailsUrl, adminTasksUrl, adminUserDeletionsUrl, adminUsersUrl, homeUrl, itemsUrl } from '../utils/urlUtils'; +import { adminDashboardUrl, adminEmailsUrl, adminTasksUrl, adminUserDeletionsUrl, adminUsersUrl, homeUrl, itemsUrl, adminReportUrl } from '../utils/urlUtils'; import { MenuItem, setSelectedMenu } from '../utils/views/menu'; +import { ReportType } from './reports/types'; export interface RenderOptions { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied @@ -132,6 +133,10 @@ export default class MustacheService { title: _('Emails'), url: adminEmailsUrl(), }, + { + title: _('Reports'), + url: adminReportUrl(ReportType.UserActivity), + }, ], }, ]; diff --git a/packages/server/src/services/reports/types.ts b/packages/server/src/services/reports/types.ts new file mode 100644 index 000000000..66e438426 --- /dev/null +++ b/packages/server/src/services/reports/types.ts @@ -0,0 +1,5 @@ +/* eslint-disable import/prefer-default-export */ + +export enum ReportType { + UserActivity = 'user_activity', +} diff --git a/packages/server/src/services/reports/userActivity.test.ts b/packages/server/src/services/reports/userActivity.test.ts new file mode 100644 index 000000000..3b6c849b8 --- /dev/null +++ b/packages/server/src/services/reports/userActivity.test.ts @@ -0,0 +1,67 @@ +import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, createFolder, updateFolder, dbSlave, deleteFolder } from '../../utils/testing/testUtils'; +import { Hour } from '../../utils/time'; +import userActivity from './userActivity'; + +describe('reports/userActivity', () => { + + beforeAll(async () => { + await beforeAllDb('reports/userActivity'); + }); + + afterAll(async () => { + await afterAllTests(); + }); + + beforeEach(async () => { + await beforeEachDb(); + }); + + test('should create a report on user activity', async () => { + const { session: session1, user: user1 } = await createUserAndSession(1, false); + const { session: session2, user: user2 } = await createUserAndSession(2, false); + + expect(await userActivity(dbSlave())).toEqual([]); + + jest.useFakeTimers(); + + const t0 = new Date('2022-01-01 00:00:00').getTime(); + jest.setSystemTime(t0); + + await createFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1a' }); + await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1b' }); + + const t1 = new Date('2022-01-01 02:00:00').getTime(); + jest.setSystemTime(t1); + + await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1c' }); + await updateFolder(session1.id, { id: '000000000000000000000000000000F1', title: 'folder 1d' }); + await deleteFolder(user1.id, '000000000000000000000000000000F1'); + + await createFolder(session2.id, { id: '000000000000000000000000000000F2', title: 'folder 2a' }); + await updateFolder(session2.id, { id: '000000000000000000000000000000F2', title: 'folder 2b' }); + + const results = await userActivity(dbSlave(), { batchSize: 2, interval: 1 * Hour }); + + expect(results).toEqual( + [ + { + user_id: user1.id, + total_count: 3, + create_count: 0, + update_count: 2, + delete_count: 1, + }, + { + user_id: user2.id, + total_count: 2, + create_count: 1, + update_count: 1, + delete_count: 0, + }, + ], + ); + + jest.useRealTimers(); + }); + +}); diff --git a/packages/server/src/services/reports/userActivity.ts b/packages/server/src/services/reports/userActivity.ts new file mode 100644 index 000000000..1cf788061 --- /dev/null +++ b/packages/server/src/services/reports/userActivity.ts @@ -0,0 +1,84 @@ +import { DbConnection } from '../../db'; +import { ChangeType, Uuid } from '../database/types'; + +export interface Options { + interval: number; + batchSize?: number; +} + +export default async (db: DbConnection, options: Options = null) => { + options = { + batchSize: 10000, + ...options, + }; + + const cutOffTime = Date.now() - options.interval; + + interface ChangeSlice { + user_id: Uuid; + updated_time: number; + counter: number; + type: ChangeType; + } + + interface GroupedChange { + user_id: Uuid; + total_count: number; + create_count: number; + update_count: number; + delete_count: number; + } + + let changes: ChangeSlice[] = []; + let counter = 0; + + while (true) { + const query = db('changes') + .select('user_id', 'updated_time', 'counter', 'type') + .orderBy('counter', 'desc') + .limit(options.batchSize); + + if (counter > 0) void query.where('counter', '<', counter); + + const results: ChangeSlice[] = await query; + + if (!results.length) break; + + const filteredResults = results.filter(row => row.updated_time >= cutOffTime); + + changes = changes.concat(filteredResults); + + if (filteredResults.length !== results.length) break; + + counter = filteredResults[filteredResults.length - 1].counter; + } + + const groupedChanges: GroupedChange[] = []; + for (const c of changes) { + let grouped = groupedChanges.find(g => g.user_id === c.user_id); + if (!grouped) { + grouped = { + user_id: c.user_id, + total_count: 0, + create_count: 0, + update_count: 0, + delete_count: 0, + }; + + groupedChanges.push(grouped); + } + + if (c.type === ChangeType.Create) grouped.create_count++; + if (c.type === ChangeType.Update) grouped.update_count++; + if (c.type === ChangeType.Delete) grouped.delete_count++; + grouped.total_count++; + } + + groupedChanges.sort((a, b) => { + if (a.total_count > b.total_count) return -1; + if (a.total_count < b.total_count) return +1; + return 0; + }); + + return groupedChanges; +}; diff --git a/packages/server/src/utils/routeUtils.ts b/packages/server/src/utils/routeUtils.ts index a4b0c801f..1025c5d7c 100644 --- a/packages/server/src/utils/routeUtils.ts +++ b/packages/server/src/utils/routeUtils.ts @@ -118,7 +118,7 @@ export function isPathBasedAddressing(fileId: string): boolean { export const urlMatchesSchema = (url: string, schema: string): boolean => { url = stripOffQueryParameters(url); - const regex = new RegExp(`${schema.replace(/:id/, '[a-zA-Z0-9]+')}$`); + const regex = new RegExp(`${schema.replace(/:id/, '[a-zA-Z0-9_]+')}$`); return !!url.match(regex); }; @@ -314,6 +314,7 @@ export enum UrlType { Privacy = 'privacy', Tasks = 'admin/tasks', UserDeletions = 'admin/user_deletions', + Reports = 'admin/reports', } export function makeUrl(urlType: UrlType): string { diff --git a/packages/server/src/utils/testing/populateDatabase.ts b/packages/server/src/utils/testing/populateDatabase.ts index b63a71343..84b611f59 100644 --- a/packages/server/src/utils/testing/populateDatabase.ts +++ b/packages/server/src/utils/testing/populateDatabase.ts @@ -319,22 +319,26 @@ const main = async (_options?: Options) => { { const promises = []; - for (let i = 0; i < 20000; i++) { - promises.push((async () => { - const user = randomElement(users); - const action = randomActionKey(); - try { - const done = await reactions[action](context, user); - if (done) updateReport(action); - logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`); - } catch (error) { - error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`; - throw error; - } - })()); + const totalActions = 5000; + const batchSize = 1000; // Don't change this - it will fail with higher numbers + const loopCount = Math.ceil(totalActions / batchSize); + for (let loopIndex = 0; loopIndex < loopCount; loopIndex++) { + for (let i = 0; i < batchSize; i++) { + promises.push((async () => { + const user = randomElement(users); + const action = randomActionKey(); + try { + const done = await reactions[action](context, user); + if (done) updateReport(action); + logger().info(`Done action ${i}: ${action}. User: ${user.email}${!done ? ' (Skipped)' : ''}`); + } catch (error) { + error.message = `Could not do action ${i}: ${action}. User: ${user.email}: ${error.message}`; + logger().warn(error.message); + } + })()); + } + await Promise.all(promises); } - - await Promise.all(promises); } // const changeIds = (await models().change().all()).map(c => c.id); diff --git a/packages/server/src/utils/testing/testUtils.ts b/packages/server/src/utils/testing/testUtils.ts index de628468e..9ab60f39c 100644 --- a/packages/server/src/utils/testing/testUtils.ts +++ b/packages/server/src/utils/testing/testUtils.ts @@ -434,6 +434,11 @@ export async function updateFolder(sessionId: string, folder: FolderEntity): Pro return updateItem(sessionId, `root:/${folder.id}.md:`, makeFolderSerializedBody(folder)); } +export async function deleteFolder(userId: string, folderJopId: string): Promise { + const item = await models().item().loadByJopId(userId, folderJopId, { fields: ['id'] }); + await models().item().delete(item.id); +} + export async function createFolder(sessionId: string, folder: FolderEntity): Promise { folder = { id: '000000000000000000000000000000F1', diff --git a/packages/server/src/utils/urlUtils.ts b/packages/server/src/utils/urlUtils.ts index 48d37f1bf..f44df679e 100644 --- a/packages/server/src/utils/urlUtils.ts +++ b/packages/server/src/utils/urlUtils.ts @@ -1,6 +1,7 @@ import { URL } from 'url'; import config from '../config'; import { Uuid } from '../services/database/types'; +import { ReportType } from '../services/reports/types'; // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied export function setQueryParameters(url: string, query: any): string { @@ -94,3 +95,7 @@ export function adminEmailsUrl() { export function adminEmailUrl(id: number) { return `${config().adminBaseUrl}/emails/${id}`; } + +export function adminReportUrl(type: ReportType) { + return `${config().adminBaseUrl}/reports/${type}`; +} diff --git a/packages/server/src/views/admin/reports/user_activity.mustache b/packages/server/src/views/admin/reports/user_activity.mustache new file mode 100644 index 000000000..9e9e1d4db --- /dev/null +++ b/packages/server/src/views/admin/reports/user_activity.mustache @@ -0,0 +1,22 @@ +
+
+
+ +
+ +
+
+ +
+
+ +
+
+
+
+ +
+ {{#itemTable}} + {{>table}} + {{/itemTable}} +
\ No newline at end of file