self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-11-23 22:36:32 +02:00
Code Issues Releases Activity

Server: Fix report service fails when there are a very large number of items to be processed (#13721)

Browse Source
This commit is contained in:
Henry Heino
2025-11-21 11:28:10 -08:00
committed by GitHub
parent 74fa2a6eb9
commit 5e2b79557c
1 changed files with 41 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
packages/server/src/services/reports/userActivity.ts
View File

@@ -7,14 +7,7 @@ export interface Options {
batchSize?: number; batchSize?: number;
} }
export default async (db: DbConnection, options: Options = null) => { const collectChanges = async (db: DbConnection, options: Options) => {
options = {
batchSize: 10000,
...options,
};
const cutOffTime = Date.now() - options.interval;
interface ChangeSlice { interface ChangeSlice {
user_id: Uuid; user_id: Uuid;
updated_time: number; updated_time: number;
@@ -23,20 +16,11 @@ export default async (db: DbConnection, options: Options = null) => {
item_id: Uuid; item_id: Uuid;
} }
interface GroupedChange {
user_id: Uuid;
total_count: number;
create_count: number;
update_count: number;
delete_count: number;
uploaded_size: number;
}
type ItemSlice = Pick<Item, 'content_size' | 'id'>;
let changes: ChangeSlice[] = []; let changes: ChangeSlice[] = [];
let counter = 0; let counter = 0;
const cutOffTime = Date.now() - options.interval;
while (true) { while (true) {
const query = db('changes') const query = db('changes')
.select('user_id', 'updated_time', 'counter', 'item_id', 'type') .select('user_id', 'updated_time', 'counter', 'item_id', 'type')
@@ -58,16 +42,44 @@ export default async (db: DbConnection, options: Options = null) => {
counter = filteredResults[filteredResults.length - 1].counter; counter = filteredResults[filteredResults.length - 1].counter;
} }
return changes;
};
export default async (db: DbConnection, options: Options = null) => {
options = {
batchSize: 10000,
...options,
};
interface GroupedChange {
user_id: Uuid;
total_count: number;
create_count: number;
update_count: number;
delete_count: number;
uploaded_size: number;
}
type ItemSlice = Pick<Item, 'content_size' | 'id'>;
const changes = await collectChanges(db, options);
const itemIds = unique(changes.map(c => c.item_id)); const itemIds = unique(changes.map(c => c.item_id));
const items: ItemSlice[] = await db('items') const batchSize = 1000;
const idToItem = new Map<string, ItemSlice>();
for (let i = 0; i < itemIds.length; i += batchSize) {
const itemBatch: ItemSlice[] = await db('items')
.select('id', 'content_size') .select('id', 'content_size')
.whereIn('id', itemIds); .whereIn('id', itemIds.slice(i, i + batchSize));
for (const item of itemBatch) {
idToItem.set(item.id, item);
}
}
const groupedChanges: GroupedChange[] = []; const groupedChanges = new Map<string, GroupedChange>();
for (const c of changes) { for (const c of changes) {
let grouped = groupedChanges.find(g => g.user_id === c.user_id); let grouped = groupedChanges.get(c.user_id);
if (!grouped) { if (!grouped) {
grouped = { grouped = {
user_id: c.user_id, user_id: c.user_id,
@@ -78,7 +90,7 @@ export default async (db: DbConnection, options: Options = null) => {
uploaded_size: 0, uploaded_size: 0,
}; };
groupedChanges.push(grouped); groupedChanges.set(c.user_id, grouped);
} }
if (c.type === ChangeType.Create) grouped.create_count++; if (c.type === ChangeType.Create) grouped.create_count++;
@@ -86,21 +98,19 @@ export default async (db: DbConnection, options: Options = null) => {
if (c.type === ChangeType.Delete) grouped.delete_count++; if (c.type === ChangeType.Delete) grouped.delete_count++;
grouped.total_count++; grouped.total_count++;
const item = items.find(it => it.id === c.item_id); const item = idToItem.get(c.item_id);
if (item) { if (item) {
if ([ChangeType.Create, ChangeType.Update].includes(c.type)) { if ([ChangeType.Create, ChangeType.Update].includes(c.type)) {
grouped.uploaded_size += item.content_size; grouped.uploaded_size += item.content_size;
} }
} }
if (!itemIds.includes(c.item_id)) itemIds.push(c.item_id);
} }
groupedChanges.sort((a, b) => { const groupedChangesList = Array.from(groupedChanges.values());
groupedChangesList.sort((a, b) => {
if (a.total_count > b.total_count) return -1; if (a.total_count > b.total_count) return -1;
if (a.total_count < b.total_count) return +1; if (a.total_count < b.total_count) return +1;
return 0; return 0;
}); });
return groupedChangesList;
return groupedChanges;
}; };