From d1e02fd5f000cdfa12a385946aac6cdd27290211 Mon Sep 17 00:00:00 2001 From: Laurent Date: Tue, 28 Dec 2021 10:55:01 +0100 Subject: [PATCH] Server: Allow deleting complete user data (#5824) --- packages/app-desktop/runForTesting.sh | 4 + packages/server/package.json | 1 + packages/server/public/css/main.css | 7 +- packages/server/schema.sqlite | Bin 319488 -> 327680 bytes packages/server/src/db.ts | 8 +- .../20211204191051_user_deletion.ts | 26 ++ packages/server/src/models/BaseModel.ts | 25 +- packages/server/src/models/ChangeModel.ts | 25 +- packages/server/src/models/ItemModel.test.ts | 92 ++--- packages/server/src/models/ItemModel.ts | 47 ++- .../server/src/models/NotificationModel.ts | 17 +- packages/server/src/models/ShareModel.ts | 10 + packages/server/src/models/ShareUserModel.ts | 10 + .../src/models/UserDeletionModel.test.ts | 146 +++++++ .../server/src/models/UserDeletionModel.ts | 94 +++++ packages/server/src/models/UserFlagModel.ts | 9 + packages/server/src/models/UserItemModel.ts | 12 +- packages/server/src/models/UserModel.test.ts | 36 +- packages/server/src/models/UserModel.ts | 22 +- packages/server/src/models/factory.ts | 5 + .../server/src/models/utils/pagination.ts | 24 +- packages/server/src/routes/api/batch_items.ts | 2 +- packages/server/src/routes/api/debug.ts | 6 +- packages/server/src/routes/api/items.test.ts | 4 +- packages/server/src/routes/api/items.ts | 3 +- .../server/src/routes/api/share_users.test.ts | 2 +- packages/server/src/routes/api/shares.test.ts | 4 +- packages/server/src/routes/index/tasks.ts | 2 + .../server/src/routes/index/user_deletions.ts | 150 +++++++ packages/server/src/routes/index/users.ts | 47 ++- packages/server/src/routes/routes.ts | 2 + packages/server/src/services/BaseService.ts | 16 +- .../server/src/services/TaskService.test.ts | 8 +- packages/server/src/services/TaskService.ts | 13 +- .../src/services/UserDeletionService.test.ts | 137 +++++++ .../src/services/UserDeletionService.ts | 106 +++++ .../server/src/services/database/types.ts | 27 ++ packages/server/src/services/types.ts | 2 + packages/server/src/tools/debugTools.ts | 13 + packages/server/src/tools/generateTypes.ts | 4 + packages/server/src/utils/errors.ts | 14 +- .../src/utils/prettycron.testdisabled.ts | 58 +++ packages/server/src/utils/prettycron.ts | 384 ++++++++++++++++++ packages/server/src/utils/routeUtils.ts | 1 + packages/server/src/utils/setupAppContext.ts | 6 +- packages/server/src/utils/setupTaskService.ts | 12 +- .../server/src/utils/testing/fileApiUtils.ts | 38 +- packages/server/src/utils/time.ts | 1 + packages/server/src/utils/urlUtils.ts | 8 + packages/server/src/utils/views/table.ts | 10 +- packages/server/src/views/index/user.mustache | 4 + .../src/views/index/user_deletions.mustache | 11 + .../server/src/views/index/users.mustache | 4 + .../src/views/partials/tableRowItem.mustache | 2 +- yarn.lock | 20 +- 55 files changed, 1561 insertions(+), 180 deletions(-) create mode 100644 packages/server/src/migrations/20211204191051_user_deletion.ts create mode 100644 packages/server/src/models/UserDeletionModel.test.ts create mode 100644 packages/server/src/models/UserDeletionModel.ts create mode 100644 packages/server/src/routes/index/user_deletions.ts create mode 100644 packages/server/src/services/UserDeletionService.test.ts create mode 100644 packages/server/src/services/UserDeletionService.ts create mode 100644 packages/server/src/utils/prettycron.testdisabled.ts create mode 100644 packages/server/src/utils/prettycron.ts create mode 100644 packages/server/src/views/index/user_deletions.mustache diff --git a/packages/app-desktop/runForTesting.sh b/packages/app-desktop/runForTesting.sh index 8d63d0894..4a8f3371f 100755 --- a/packages/app-desktop/runForTesting.sh +++ b/packages/app-desktop/runForTesting.sh @@ -54,6 +54,10 @@ do curl --data '{"action": "createTestUsers"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug + elif [[ $CMD == "createUserDeletions" ]]; then + + curl --data '{"action": "createUserDeletions"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug + elif [[ $CMD == "createData" ]]; then echo 'mkbook "shared"' >> "$CMD_FILE" diff --git a/packages/server/package.json b/packages/server/package.json index f52dd7e9c..072fa13f1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,6 +47,7 @@ "nodemon": "^2.0.6", "pg": "^8.5.1", "pretty-bytes": "^5.6.0", + "prettycron": "^0.10.0", "query-string": "^6.8.3", "rate-limiter-flexible": "^2.2.4", "raw-body": "^2.4.1", diff --git a/packages/server/public/css/main.css b/packages/server/public/css/main.css index e97cd38b2..3b673fcfb 100644 --- a/packages/server/public/css/main.css +++ b/packages/server/public/css/main.css @@ -87,4 +87,9 @@ h4:hover a.heading-anchor, h5:hover a.heading-anchor, h6:hover a.heading-anchor { opacity: 1; -} \ No newline at end of file +} + +abbr[title] { + text-underline-offset: 2px; + text-decoration: underline dotted; +} diff --git a/packages/server/schema.sqlite b/packages/server/schema.sqlite index c43503892781f75603e6a015771151e0de620217..45bc55a21a8a646e2f2456217b4f7f2199402ec7 100644 GIT binary patch delta 2061 zcmc&#TTI(#6pmw)K%h&ib_*;wv%=cYsw565psmsZNhp^>!Y!0-vQ6Tq3C4yvDNt=A z*66O9g!F6k;}D%D?V%b3Mj}t5CLRapuYtwjZx z4YS)ZTaCTmZmYH9366^4A&REr5tca_<)GIySAO`r%Dm0Bi^oyp!0K@>&c?_vg-4hW zHMYfl4E}ufIYcyi49X3532cB~L!|^BfeAyo1bzjd7@n8F&)`N@^#J@~I3`K%gCoX@ zENC@WNnj0(8=Vrk2NsOA5?BT6#^VyW1B@nz1bzfvCOOtE@Q&$(B>4_xOfu(N@RLan z`VA;C%V~TKYRyL_&R2ji%c|?(l35OV4O}r-OR6uyidoL*Dk!nY+vSI zu2F6@5c3aVh4sm`R|9zK;6}10Zs(mei?ja}ET#cRUYH2;~ z>UUr+>DKa;Rc60(s>!=rG$++P>Xn*$?PCI~Iop7Z4K+0PbEExOwB6m|4g>~ir>oO; z&gEtM^Et_9FRW&e(vqF|rIIUJG@seNNQ*vbAFdV&D8r&Qzi+VXBFQje7kx46PK3uu zhwlRBrQ3SOd+BJsaQh}?)`*k4S;bUxRcpy(v$z7SSn_(FnX=7Hqp9g)UAmZ1evjwv z%i@x9vikoXp)~xPavw7!UM@8ap=rnmKkAUmw;{KBL%y_bW=S_QE#5v0&`$}(_R*|T zg*;SfAI=}N-Pe}s;G;#|&3zCy+gG*~Gt3^pk7?Wf_#n zwIR@BSxF>BSl>+Wg=)Rk3cXz^W=ucfKG(3LvD4$~_MqLahBnVmugVB3EWHdng;UoM z^w4mO4N@G3hr}8XC^tmXv~WED8=l!xkij6EVB$M;Mpf`U6-v;;>>6Y!;NoN~F3g>U zE0q)@%&kFsVL1TngnI$_tzPQ{uU6}u^$>qtE>>84_t($o_ zx4S)kD6yA?aAO@RYv1jKu<{3am0I6ng7`)`-sYB*Lo!rQ+KXb;ScD00HfW2`B9t#c zPkXn1r4;w%XIr#IlbW-ONm$eL0`di1tl|q*@+!B!A*^R0WBPOk`9)b;XLsUue2}5W YaIqT0I7!E;m?%s)Uq#Xt^4RMB0iLt{_5c6? delta 1481 zcmZo@5NS9dJVBn33&fZ_ zxl+a)#F#jFi;O9VF=6ss8Ep`wW3r5_F^JJVIan5KSIguMSsjo_{^T99U^O|DpUZ+R z&7RC72X;o*WK%hHkeZCi336bm^vSJqU`vxGua(mWNhMBxAP06&!ekzKuwt9Z<`9O} zP>p<(HfIa#@(A^PPm%FMo;gv{n*VA#wYaF?GKMd9YR@t-k- z@~89D%&Aocm5u==C7A*G+2)=RL1`)8dX6ApGuCou@;-o&K&NhJzQA~oU!FjCZJJOs zKOxUKEG^sB)43wh*UQM)H`gN2*V4iuzckz3JUPYP)EmXWSOT4p{^<{Qut@MAyKhs& z1AYZU+K5uLEr5{-qqZe5?#NdNGBt?u%rdk8~eSptgzP-tQRJ0s(E zjs+}#_?bmma;CFyV9{jZX69zz94Ju97|y}XI!WBvn6WFcBrz!`H9k8pwIaSGHMbx> zF{dQ8$jHDHi;%B#kgH>et3rsQlaH%H0u~jMI7O!K-^Iecec=ukPp0iTTUfR-bLXbU l=a=WD7R77UO@F(CQD(cx4wjcp5+FMav6!*lb2rO&6#&F@-|7GW diff --git a/packages/server/src/db.ts b/packages/server/src/db.ts index e14d63a20..ecd3d3672 100644 --- a/packages/server/src/db.ts +++ b/packages/server/src/db.ts @@ -49,6 +49,7 @@ export interface DbConfigConnection { export interface QueryContext { uniqueConstraintErrorLoggingDisabled?: boolean; + noSuchTableErrorLoggingDisabled?: boolean; } export interface KnexDatabaseConfig { @@ -227,6 +228,10 @@ export async function connectDb(dbConfig: DatabaseConfig): Promise if (data.queryContext.uniqueConstraintErrorLoggingDisabled && isUniqueConstraintError(response)) { return; } + + if (data.queryContext.noSuchTableErrorLoggingDisabled && isNoSuchTableError(response)) { + return; + } } const msg: string[] = []; @@ -392,7 +397,8 @@ export function isUniqueConstraintError(error: any): boolean { export async function latestMigration(db: DbConnection): Promise { try { - const result = await db('knex_migrations').select('name').orderBy('id', 'desc').first(); + const context: QueryContext = { noSuchTableErrorLoggingDisabled: true }; + const result = await db('knex_migrations').queryContext(context).select('name').orderBy('id', 'desc').first(); return { name: result.name, done: true }; } catch (error) { // If the database has never been initialized, we return null, so diff --git a/packages/server/src/migrations/20211204191051_user_deletion.ts b/packages/server/src/migrations/20211204191051_user_deletion.ts new file mode 100644 index 000000000..59732ea86 --- /dev/null +++ b/packages/server/src/migrations/20211204191051_user_deletion.ts @@ -0,0 +1,26 @@ +import { Knex } from 'knex'; +import { DbConnection } from '../db'; + +export async function up(db: DbConnection): Promise { + await db.schema.createTable('user_deletions', (table: Knex.CreateTableBuilder) => { + table.increments('id').unique().primary().notNullable(); + table.string('user_id', 32).notNullable(); + table.specificType('process_data', 'smallint').defaultTo(0).notNullable(); + table.specificType('process_account', 'smallint').defaultTo(0).notNullable(); + table.bigInteger('scheduled_time').notNullable(); + table.bigInteger('start_time').defaultTo(0).notNullable(); + table.bigInteger('end_time').defaultTo(0).notNullable(); + table.integer('success').defaultTo(0).notNullable(); + table.text('error', 'mediumtext').defaultTo('').notNullable(); + table.bigInteger('updated_time').notNullable(); + table.bigInteger('created_time').notNullable(); + }); + + await db.schema.alterTable('user_deletions', (table: Knex.CreateTableBuilder) => { + table.unique(['user_id']); + }); +} + +export async function down(db: DbConnection): Promise { + await db.schema.dropTable('user_deletions'); +} diff --git a/packages/server/src/models/BaseModel.ts b/packages/server/src/models/BaseModel.ts index 54bf187e2..f00361941 100644 --- a/packages/server/src/models/BaseModel.ts +++ b/packages/server/src/models/BaseModel.ts @@ -9,6 +9,7 @@ import { Config } from '../utils/types'; import personalizedUserContentBaseUrl from '@joplin/lib/services/joplinServer/personalizedUserContentBaseUrl'; import Logger from '@joplin/lib/Logger'; import dbuuid from '../utils/dbuuid'; +import { defaultPagination, PaginatedResults, Pagination } from './utils/pagination'; const logger = Logger.create('BaseModel'); @@ -232,6 +233,28 @@ export default abstract class BaseModel { return rows as T[]; } + public async allPaginated(pagination: Pagination, options: LoadOptions = {}): Promise> { + pagination = { + ...defaultPagination(), + ...pagination, + }; + + const itemCount = await this.count(); + + const items = await this + .db(this.tableName) + .select(this.selectFields(options)) + .orderBy(pagination.order[0].by, pagination.order[0].dir) + .offset((pagination.page - 1) * pagination.limit) + .limit(pagination.limit) as T[]; + + return { + items, + page_count: Math.ceil(itemCount / pagination.limit), + has_more: items.length >= pagination.limit, + }; + } + public async count(): Promise { const r = await this .db(this.tableName) @@ -343,7 +366,7 @@ export default abstract class BaseModel { return !!o; } - public async load(id: string, options: LoadOptions = {}): Promise { + public async load(id: Uuid | number, options: LoadOptions = {}): Promise { if (!id) throw new Error('id cannot be empty'); return this.db(this.tableName).select(options.fields || this.defaultFields).where({ id: id }).first(); diff --git a/packages/server/src/models/ChangeModel.ts b/packages/server/src/models/ChangeModel.ts index 3388cc683..a8a2121bc 100644 --- a/packages/server/src/models/ChangeModel.ts +++ b/packages/server/src/models/ChangeModel.ts @@ -16,13 +16,9 @@ export interface DeltaChange extends Change { jop_updated_time?: number; } -export interface PaginatedDeltaChanges extends PaginatedResults { - items: DeltaChange[]; -} +export type PaginatedDeltaChanges = PaginatedResults; -export interface PaginatedChanges extends PaginatedResults { - items: Change[]; -} +export type PaginatedChanges = PaginatedResults; export interface ChangePagination { limit?: number; @@ -43,6 +39,15 @@ export function defaultDeltaPagination(): ChangePagination { }; } +export function requestDeltaPagination(query: any): ChangePagination { + if (!query) return defaultDeltaPagination(); + + const output: ChangePagination = {}; + if ('limit' in query) output.limit = query.limit; + if ('cursor' in query) output.cursor = query.cursor; + return output; +} + export default class ChangeModel extends BaseModel { public get tableName(): string { @@ -391,4 +396,12 @@ export default class ChangeModel extends BaseModel { return savedChange; } + public async deleteByItemIds(itemIds: Uuid[]) { + if (!itemIds.length) return; + + await this.db(this.tableName) + .whereIn('item_id', itemIds) + .delete(); + } + } diff --git a/packages/server/src/models/ItemModel.test.ts b/packages/server/src/models/ItemModel.test.ts index e93575734..181a341b5 100644 --- a/packages/server/src/models/ItemModel.test.ts +++ b/packages/server/src/models/ItemModel.test.ts @@ -1,4 +1,4 @@ -import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder, createItemTree3, db, tempDir, expectNotThrow, expectHttpError } from '../utils/testing/testUtils'; +import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItemTree, createResource, createNote, createItemTree3, db, tempDir, expectNotThrow, expectHttpError } from '../utils/testing/testUtils'; import { shareFolderWithUser } from '../utils/testing/shareApiUtils'; import { resourceBlobPath } from '../utils/joplinUtils'; import newModelFactory from './factory'; @@ -23,64 +23,64 @@ describe('ItemModel', function() { await beforeEachDb(); }); - test('should find exclusively owned items 1', async function() { - const { user: user1 } = await createUserAndSession(1, true); - const { session: session2, user: user2 } = await createUserAndSession(2); + // test('should find exclusively owned items 1', async function() { + // const { user: user1 } = await createUserAndSession(1, true); + // const { session: session2, user: user2 } = await createUserAndSession(2); - const tree: any = { - '000000000000000000000000000000F1': { - '00000000000000000000000000000001': null, - }, - }; + // const tree: any = { + // '000000000000000000000000000000F1': { + // '00000000000000000000000000000001': null, + // }, + // }; - await createItemTree(user1.id, '', tree); - await createItem(session2.id, 'root:/test.txt:', 'testing'); + // await createItemTree(user1.id, '', tree); + // await createItem(session2.id, 'root:/test.txt:', 'testing'); - { - const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); - expect(itemIds.length).toBe(2); + // { + // const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); + // expect(itemIds.length).toBe(2); - const item1 = await models().item().load(itemIds[0]); - const item2 = await models().item().load(itemIds[1]); + // const item1 = await models().item().load(itemIds[0]); + // const item2 = await models().item().load(itemIds[1]); - expect([item1.jop_id, item2.jop_id].sort()).toEqual(['000000000000000000000000000000F1', '00000000000000000000000000000001'].sort()); - } + // expect([item1.jop_id, item2.jop_id].sort()).toEqual(['000000000000000000000000000000F1', '00000000000000000000000000000001'].sort()); + // } - { - const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id); - expect(itemIds.length).toBe(1); - } - }); + // { + // const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id); + // expect(itemIds.length).toBe(1); + // } + // }); - test('should find exclusively owned items 2', async function() { - const { session: session1, user: user1 } = await createUserAndSession(1, true); - const { session: session2, user: user2 } = await createUserAndSession(2); + // test('should find exclusively owned items 2', async function() { + // const { session: session1, user: user1 } = await createUserAndSession(1, true); + // const { session: session2, user: user2 } = await createUserAndSession(2); - await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', { - '000000000000000000000000000000F1': { - '00000000000000000000000000000001': null, - }, - }); + // await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', { + // '000000000000000000000000000000F1': { + // '00000000000000000000000000000001': null, + // }, + // }); - await createFolder(session2.id, { id: '000000000000000000000000000000F2' }); + // await createFolder(session2.id, { id: '000000000000000000000000000000F2' }); - { - const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); - expect(itemIds.length).toBe(0); - } + // { + // const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); + // expect(itemIds.length).toBe(0); + // } - { - const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id); - expect(itemIds.length).toBe(1); - } + // { + // const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id); + // expect(itemIds.length).toBe(1); + // } - await models().user().delete(user2.id); + // await models().user().delete(user2.id); - { - const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); - expect(itemIds.length).toBe(2); - } - }); + // { + // const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id); + // expect(itemIds.length).toBe(2); + // } + // }); test('should find all items within a shared folder', async function() { const { user: user1, session: session1 } = await createUserAndSession(1); diff --git a/packages/server/src/models/ItemModel.ts b/packages/server/src/models/ItemModel.ts index 405e4ec37..108c02231 100644 --- a/packages/server/src/models/ItemModel.ts +++ b/packages/server/src/models/ItemModel.ts @@ -1,4 +1,4 @@ -import BaseModel, { SaveOptions, LoadOptions, DeleteOptions, ValidateOptions, AclAction } from './BaseModel'; +import BaseModel, { SaveOptions, LoadOptions, DeleteOptions as BaseDeleteOptions, ValidateOptions, AclAction } from './BaseModel'; import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User, UserItem } from '../services/database/types'; import { defaultPagination, paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination'; import { isJoplinItemName, isJoplinResourceBlobPath, linkedResourceIds, serializeJoplinItem, unserializeJoplinItem } from '../utils/joplinUtils'; @@ -21,6 +21,10 @@ const mimeUtils = require('@joplin/lib/mime-utils.js').mime; // Converts "root:/myfile.txt:" to "myfile.txt" const extractNameRegex = /^root:\/(.*):$/; +export interface DeleteOptions extends BaseDeleteOptions { + deleteChanges?: boolean; +} + export interface ImportContentToStorageOptions { batchSize?: number; maxContentSize?: number; @@ -45,9 +49,7 @@ export interface SaveFromRawContentResultItem { export type SaveFromRawContentResult = Record; -export interface PaginatedItems extends PaginatedResults { - items: Item[]; -} +export type PaginatedItems = PaginatedResults; export interface SharedRootInfo { item: Item; @@ -813,22 +815,22 @@ export default class ItemModel extends BaseModel { // Returns the item IDs that are owned only by the given user. In other // words, the items that are not shared with anyone else. Such items // can be safely deleted when the user is deleted. - public async exclusivelyOwnedItemIds(userId: Uuid): Promise { - const query = this - .db('items') - .select(this.db.raw('items.id, count(user_items.item_id) as user_item_count')) - .leftJoin('user_items', 'user_items.item_id', 'items.id') - .whereIn('items.id', this.db('user_items').select('user_items.item_id').where('user_id', '=', userId)) - .groupBy('items.id'); + // public async exclusivelyOwnedItemIds(userId: Uuid): Promise { + // const query = this + // .db('items') + // .select(this.db.raw('items.id, count(user_items.item_id) as user_item_count')) + // .leftJoin('user_items', 'user_items.item_id', 'items.id') + // .whereIn('items.id', this.db('user_items').select('user_items.item_id').where('user_id', '=', userId)) + // .groupBy('items.id'); - const rows: any[] = await query; - return rows.filter(r => r.user_item_count === 1).map(r => r.id); - } + // const rows: any[] = await query; + // return rows.filter(r => r.user_item_count === 1).map(r => r.id); + // } - public async deleteExclusivelyOwnedItems(userId: Uuid) { - const itemIds = await this.exclusivelyOwnedItemIds(userId); - await this.delete(itemIds); - } + // public async deleteExclusivelyOwnedItems(userId: Uuid) { + // const itemIds = await this.exclusivelyOwnedItemIds(userId); + // await this.delete(itemIds); + // } public async deleteAll(userId: Uuid): Promise { while (true) { @@ -839,6 +841,11 @@ export default class ItemModel extends BaseModel { } public async delete(id: string | string[], options: DeleteOptions = {}): Promise { + options = { + deleteChanges: false, + ...options, + }; + const ids = typeof id === 'string' ? [id] : id; if (!ids.length) return; @@ -849,12 +856,14 @@ export default class ItemModel extends BaseModel { await this.withTransaction(async () => { await this.models().share().delete(shares.map(s => s.id)); - await this.models().userItem().deleteByItemIds(ids); + await this.models().userItem().deleteByItemIds(ids, { recordChanges: !options.deleteChanges }); await this.models().itemResource().deleteByItemIds(ids); await storageDriver.delete(ids, { models: this.models() }); if (storageDriverFallback) await storageDriverFallback.delete(ids, { models: this.models() }); await super.delete(ids, options); + + if (options.deleteChanges) await this.models().change().deleteByItemIds(ids); }, 'ItemModel::delete'); } diff --git a/packages/server/src/models/NotificationModel.ts b/packages/server/src/models/NotificationModel.ts index 65722cf74..1a0671ea6 100644 --- a/packages/server/src/models/NotificationModel.ts +++ b/packages/server/src/models/NotificationModel.ts @@ -57,8 +57,13 @@ export default class NotificationModel extends BaseModel { }, }; - const n: Notification = await this.loadUnreadByKey(userId, key); - if (n) return n; + const n: Notification = await this.loadByKey(userId, key); + + if (n) { + if (!n.read) return n; + await this.save({ id: n.id, read: 0 }); + return { ...n, read: 0 }; + } const type = notificationTypes[key]; @@ -83,6 +88,14 @@ export default class NotificationModel extends BaseModel { return this.save({ key: actualKey, message, level, owner_id: userId }); } + public async addInfo(userId: Uuid, message: string) { + return this.add(userId, NotificationKey.Any, NotificationLevel.Normal, message); + } + + public async addError(userId: Uuid, message: string) { + return this.add(userId, NotificationKey.Any, NotificationLevel.Error, message); + } + public async setRead(userId: Uuid, key: NotificationKey, read: boolean = true): Promise { const n = await this.loadByKey(userId, key); if (!n) return; diff --git a/packages/server/src/models/ShareModel.ts b/packages/server/src/models/ShareModel.ts index d2a37d51c..3735d488e 100644 --- a/packages/server/src/models/ShareModel.ts +++ b/packages/server/src/models/ShareModel.ts @@ -411,6 +411,16 @@ export default class ShareModel extends BaseModel { }, 'ShareModel::delete'); } + public async deleteByUserId(userId: Uuid) { + const shares = await this.sharesByUser(userId); + + await this.withTransaction(async () => { + for (const share of shares) { + await this.delete(share.id); + } + }, 'ShareModel::deleteByUserId'); + } + public async itemCountByShareId(shareId: Uuid): Promise { const r = await this .db('items') diff --git a/packages/server/src/models/ShareUserModel.ts b/packages/server/src/models/ShareUserModel.ts index 847d7257d..3f364a69e 100644 --- a/packages/server/src/models/ShareUserModel.ts +++ b/packages/server/src/models/ShareUserModel.ts @@ -145,6 +145,16 @@ export default class ShareUserModel extends BaseModel { }, 'ShareUserModel::deleteByShare'); } + public async deleteByUserId(userId: Uuid) { + const shareUsers = await this.byUserId(userId); + + await this.withTransaction(async () => { + for (const shareUser of shareUsers) { + await this.delete(shareUser.id); + } + }, 'UserShareModel::deleteByUserId'); + } + public async delete(id: string | string[], _options: DeleteOptions = {}): Promise { const ids = typeof id === 'string' ? [id] : id; if (!ids.length) throw new Error('no id provided'); diff --git a/packages/server/src/models/UserDeletionModel.test.ts b/packages/server/src/models/UserDeletionModel.test.ts new file mode 100644 index 000000000..2012d8b60 --- /dev/null +++ b/packages/server/src/models/UserDeletionModel.test.ts @@ -0,0 +1,146 @@ +import { beforeAllDb, afterAllTests, beforeEachDb, models, createUser, expectThrow } from '../utils/testing/testUtils'; + +describe('UserDeletionModel', function() { + + beforeAll(async () => { + await beforeAllDb('UserDeletionModel'); + }); + + afterAll(async () => { + await afterAllTests(); + }); + + beforeEach(async () => { + await beforeEachDb(); + }); + + test('should add a deletion operation', async function() { + { + const user = await createUser(1); + + const scheduleTime = Date.now() + 1000; + await models().userDeletion().add(user.id, scheduleTime); + const deletion = await models().userDeletion().byUserId(user.id); + expect(deletion.user_id).toBe(user.id); + expect(deletion.process_account).toBe(1); + expect(deletion.process_data).toBe(1); + expect(deletion.scheduled_time).toBe(scheduleTime); + expect(deletion.error).toBe(''); + expect(deletion.success).toBe(0); + expect(deletion.start_time).toBe(0); + expect(deletion.end_time).toBe(0); + await models().userDeletion().delete(deletion.id); + } + + { + const user = await createUser(2); + + await models().userDeletion().add(user.id, Date.now() + 1000, { + processData: true, + processAccount: false, + }); + + const deletion = await models().userDeletion().byUserId(user.id); + expect(deletion.process_data).toBe(1); + expect(deletion.process_account).toBe(0); + } + + { + const user = await createUser(3); + await models().userDeletion().add(user.id, Date.now() + 1000); + await expectThrow(async () => models().userDeletion().add(user.id, Date.now() + 1000)); + } + }); + + test('should provide the next deletion operation', async function() { + expect(await models().userDeletion().next()).toBeFalsy(); + + jest.useFakeTimers('modern'); + + const t0 = new Date('2021-12-14').getTime(); + jest.setSystemTime(t0); + + const user1 = await createUser(1); + const user2 = await createUser(2); + + await models().userDeletion().add(user1.id, t0 + 100000); + await models().userDeletion().add(user2.id, t0 + 100); + + expect(await models().userDeletion().next()).toBeFalsy(); + + jest.setSystemTime(t0 + 200); + + expect((await models().userDeletion().next()).user_id).toBe(user2.id); + + jest.setSystemTime(t0 + 200000); + + const next1 = await models().userDeletion().next(); + expect(next1.user_id).toBe(user2.id); + await models().userDeletion().start(next1.id); + await models().userDeletion().end(next1.id, true, null); + + const next2 = await models().userDeletion().next(); + expect(next2.user_id).toBe(user1.id); + await models().userDeletion().start(next2.id); + await models().userDeletion().end(next2.id, true, null); + + const next3 = await models().userDeletion().next(); + expect(next3).toBeFalsy(); + + jest.useRealTimers(); + }); + + test('should start and stop deletion jobs', async function() { + jest.useFakeTimers('modern'); + + const t0 = new Date('2021-12-14').getTime(); + jest.setSystemTime(t0); + + const user1 = await createUser(1); + const user2 = await createUser(2); + + await models().userDeletion().add(user1.id, t0 + 10); + await models().userDeletion().add(user2.id, t0 + 100); + + jest.setSystemTime(t0 + 200); + + const next1 = await models().userDeletion().next(); + await models().userDeletion().start(next1.id); + + { + const d = await models().userDeletion().load(next1.id); + expect(d.start_time).toBe(t0 + 200); + expect(d.updated_time).toBe(t0 + 200); + expect(d.end_time).toBe(0); + } + + jest.setSystemTime(t0 + 300); + + await models().userDeletion().end(next1.id, false, 'error!'); + + { + const d = await models().userDeletion().load(next1.id); + expect(d.start_time).toBe(t0 + 200); + expect(d.updated_time).toBe(t0 + 300); + expect(d.end_time).toBe(t0 + 300); + expect(d.success).toBe(0); + expect(JSON.parse(d.error)).toEqual({ message: 'error!' }); + } + + const next2 = await models().userDeletion().next(); + await models().userDeletion().start(next2.id); + await models().userDeletion().end(next2.id, true, null); + + { + const d = await models().userDeletion().load(next2.id); + expect(d.start_time).toBe(t0 + 300); + expect(d.updated_time).toBe(t0 + 300); + expect(d.end_time).toBe(t0 + 300); + expect(d.success).toBe(1); + expect(d.error).toBe(''); + } + + jest.useRealTimers(); + }); + +}); diff --git a/packages/server/src/models/UserDeletionModel.ts b/packages/server/src/models/UserDeletionModel.ts new file mode 100644 index 000000000..ced1b7977 --- /dev/null +++ b/packages/server/src/models/UserDeletionModel.ts @@ -0,0 +1,94 @@ +import { UserDeletion, Uuid } from '../services/database/types'; +import { errorToString } from '../utils/errors'; +import BaseModel from './BaseModel'; + +export interface AddOptions { + processData?: boolean; + processAccount?: boolean; +} + +export default class UserDeletionModel extends BaseModel { + + protected get tableName(): string { + return 'user_deletions'; + } + + protected hasUuid(): boolean { + return false; + } + + public async byUserId(userId: Uuid): Promise { + return this.db(this.tableName).where('user_id', '=', userId).first(); + } + + public async isScheduledForDeletion(userId: Uuid) { + const r = await this.db(this.tableName).select(['id']).where('user_id', '=', userId).first(); + return !!r; + } + + public async add(userId: Uuid, scheduledTime: number, options: AddOptions = null): Promise { + options = { + processAccount: true, + processData: true, + ...options, + }; + + const now = Date.now(); + + const o: UserDeletion = { + user_id: userId, + scheduled_time: scheduledTime, + created_time: now, + updated_time: now, + process_account: options.processAccount ? 1 : 0, + process_data: options.processData ? 1 : 0, + }; + + await this.db(this.tableName).insert(o); + + return this.byUserId(userId); + } + + public async remove(jobId: number) { + await this.db(this.tableName).where('id', '=', jobId).delete(); + } + + public async next(): Promise { + return this + .db(this.tableName) + .where('scheduled_time', '<=', Date.now()) + .andWhere('start_time', '=', 0) + .orderBy('scheduled_time', 'asc') + .first(); + } + + public async start(deletionId: number) { + const now = Date.now(); + + await this + .db(this.tableName) + .update({ start_time: now, updated_time: now }) + .where('id', deletionId) + .andWhere('start_time', '=', 0); + + const item = await this.load(deletionId); + if (item.start_time !== now) throw new Error('Job was already started'); + } + + public async end(deletionId: number, success: boolean, error: any) { + const now = Date.now(); + + const o: UserDeletion = { + end_time: now, + updated_time: now, + success: success ? 1 : 0, + error: error ? errorToString(error) : '', + }; + + await this + .db(this.tableName) + .update(o) + .where('id', deletionId); + } + +} diff --git a/packages/server/src/models/UserFlagModel.ts b/packages/server/src/models/UserFlagModel.ts index 6176f42b0..90aaffb9e 100644 --- a/packages/server/src/models/UserFlagModel.ts +++ b/packages/server/src/models/UserFlagModel.ts @@ -108,6 +108,7 @@ export default class UserFlagModels extends BaseModel { const failedPaymentFinalFlag = flags.find(f => f.type === UserFlagType.FailedPaymentFinal); const subscriptionCancelledFlag = flags.find(f => f.type === UserFlagType.SubscriptionCancelled); const manuallyDisabledFlag = flags.find(f => f.type === UserFlagType.ManuallyDisabled); + const userDeletionInProgress = flags.find(f => f.type === UserFlagType.UserDeletionInProgress); if (accountWithoutSubscriptionFlag) { newProps.can_upload = 0; @@ -133,6 +134,10 @@ export default class UserFlagModels extends BaseModel { newProps.enabled = 0; } + if (userDeletionInProgress) { + newProps.enabled = 0; + } + if (user.can_upload !== newProps.can_upload || user.enabled !== newProps.enabled) { await this.models().user().save({ id: userId, @@ -152,4 +157,8 @@ export default class UserFlagModels extends BaseModel { return this.db(this.tableName).where('user_id', '=', userId); } + public async deleteByUserId(userId: Uuid) { + await this.db(this.tableName).where('user_id', '=', userId).delete(); + } + } diff --git a/packages/server/src/models/UserItemModel.ts b/packages/server/src/models/UserItemModel.ts index b8ed9ddb4..71b39fa47 100644 --- a/packages/server/src/models/UserItemModel.ts +++ b/packages/server/src/models/UserItemModel.ts @@ -16,6 +16,7 @@ export interface UserItemDeleteOptions extends DeleteOptions { byUserItem?: UserItem; byUserItemIds?: number[]; byShare?: DeleteByShare; + recordChanges?: boolean; } export default class UserItemModel extends BaseModel { @@ -87,8 +88,8 @@ export default class UserItemModel extends BaseModel { await this.deleteBy({ byUserItem: userItem }); } - public async deleteByItemIds(itemIds: Uuid[]): Promise { - await this.deleteBy({ byItemIds: itemIds }); + public async deleteByItemIds(itemIds: Uuid[], options: UserItemDeleteOptions = null): Promise { + await this.deleteBy({ ...options, byItemIds: itemIds }); } public async deleteByShareId(shareId: Uuid): Promise { @@ -152,6 +153,11 @@ export default class UserItemModel extends BaseModel { } private async deleteBy(options: UserItemDeleteOptions = {}): Promise { + options = { + recordChanges: true, + ...options, + }; + let userItems: UserItem[] = []; if (options.byShareId && options.byUserId) { @@ -180,7 +186,7 @@ export default class UserItemModel extends BaseModel { for (const userItem of userItems) { const item = items.find(i => i.id === userItem.item_id); - if (this.models().item().shouldRecordChange(item.name)) { + if (options.recordChanges && this.models().item().shouldRecordChange(item.name)) { await this.models().change().save({ item_type: ItemType.UserItem, item_id: userItem.item_id, diff --git a/packages/server/src/models/UserModel.test.ts b/packages/server/src/models/UserModel.test.ts index 6552c486d..28a79f785 100644 --- a/packages/server/src/models/UserModel.test.ts +++ b/packages/server/src/models/UserModel.test.ts @@ -1,5 +1,5 @@ -import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, checkThrowAsync, createItem, expectThrow } from '../utils/testing/testUtils'; -import { EmailSender, User, UserFlagType } from '../services/database/types'; +import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, checkThrowAsync, expectThrow } from '../utils/testing/testUtils'; +import { EmailSender, UserFlagType } from '../services/database/types'; import { ErrorUnprocessableEntity } from '../utils/errors'; import { betaUserDateRange, stripeConfig } from '../utils/stripe'; import { accountByType, AccountType } from './UserModel'; @@ -51,26 +51,26 @@ describe('UserModel', function() { expect(error instanceof ErrorUnprocessableEntity).toBe(true); }); - test('should delete a user', async () => { - const { session: session1, user: user1 } = await createUserAndSession(2, false); + // test('should delete a user', async () => { + // const { session: session1, user: user1 } = await createUserAndSession(2, false); - const userModel = models().user(); + // const userModel = models().user(); - const allUsers: User[] = await userModel.all(); - const beforeCount: number = allUsers.length; + // const allUsers: User[] = await userModel.all(); + // const beforeCount: number = allUsers.length; - await createItem(session1.id, 'root:/test.txt:', 'testing'); + // await createItem(session1.id, 'root:/test.txt:', 'testing'); - // Admin can delete any user - expect(!!(await models().session().load(session1.id))).toBe(true); - expect((await models().item().all()).length).toBe(1); - expect((await models().userItem().all()).length).toBe(1); - await models().user().delete(user1.id); - expect((await userModel.all()).length).toBe(beforeCount - 1); - expect(!!(await models().session().load(session1.id))).toBe(false); - expect((await models().item().all()).length).toBe(0); - expect((await models().userItem().all()).length).toBe(0); - }); + // // Admin can delete any user + // expect(!!(await models().session().load(session1.id))).toBe(true); + // expect((await models().item().all()).length).toBe(1); + // expect((await models().userItem().all()).length).toBe(1); + // await models().user().delete(user1.id); + // expect((await userModel.all()).length).toBe(beforeCount - 1); + // expect(!!(await models().session().load(session1.id))).toBe(false); + // expect((await models().item().all()).length).toBe(0); + // expect((await models().userItem().all()).length).toBe(0); + // }); test('should push an email when creating a new user', async () => { const { user: user1 } = await createUserAndSession(1); diff --git a/packages/server/src/models/UserModel.ts b/packages/server/src/models/UserModel.ts index 337cc0dcf..002cf348f 100644 --- a/packages/server/src/models/UserModel.ts +++ b/packages/server/src/models/UserModel.ts @@ -275,18 +275,18 @@ export default class UserModel extends BaseModel { return !!s[0].length && !!s[1].length; } - public async delete(id: string): Promise { - const shares = await this.models().share().sharesByUser(id); + // public async delete(id: string): Promise { + // const shares = await this.models().share().sharesByUser(id); - await this.withTransaction(async () => { - await this.models().item().deleteExclusivelyOwnedItems(id); - await this.models().share().delete(shares.map(s => s.id)); - await this.models().userItem().deleteByUserId(id); - await this.models().session().deleteByUserId(id); - await this.models().notification().deleteByUserId(id); - await super.delete(id); - }, 'UserModel::delete'); - } + // await this.withTransaction(async () => { + // await this.models().item().deleteExclusivelyOwnedItems(id); + // await this.models().share().delete(shares.map(s => s.id)); + // await this.models().userItem().deleteByUserId(id); + // await this.models().session().deleteByUserId(id); + // await this.models().notification().deleteByUserId(id); + // await super.delete(id); + // }, 'UserModel::delete'); + // } private async confirmEmail(user: User) { await this.save({ id: user.id, email_confirmed: 1 }); diff --git a/packages/server/src/models/factory.ts b/packages/server/src/models/factory.ts index 4e410d810..01bcee404 100644 --- a/packages/server/src/models/factory.ts +++ b/packages/server/src/models/factory.ts @@ -74,6 +74,7 @@ import EventModel from './EventModel'; import { Config } from '../utils/types'; import LockModel from './LockModel'; import StorageModel from './StorageModel'; +import UserDeletionModel from './UserDeletionModel'; export type NewModelFactoryHandler = (db: DbConnection)=> Models; @@ -165,6 +166,10 @@ export class Models { return new StorageModel(this.db_, this.newModelFactory, this.config_); } + public userDeletion() { + return new UserDeletionModel(this.db_, this.newModelFactory, this.config_); + } + } export default function newModelFactory(db: DbConnection, config: Config): Models { diff --git a/packages/server/src/models/utils/pagination.ts b/packages/server/src/models/utils/pagination.ts index f84099ea6..c708ba0a2 100644 --- a/packages/server/src/models/utils/pagination.ts +++ b/packages/server/src/models/utils/pagination.ts @@ -1,6 +1,5 @@ import { ErrorBadRequest } from '../../utils/errors'; import { decodeBase64, encodeBase64 } from '../../utils/base64'; -import { ChangePagination as DeltaPagination, defaultDeltaPagination } from '../ChangeModel'; import { Knex } from 'knex'; export enum PaginationOrderDir { @@ -28,8 +27,8 @@ export interface PaginationQueryParams { cursor?: string; } -export interface PaginatedResults { - items: any[]; +export interface PaginatedResults { + items: T[]; has_more: boolean; cursor?: string; page_count?: number; @@ -107,15 +106,6 @@ export function requestPagination(query: any): Pagination { return validatePagination({ limit, order, page }); } -export function requestDeltaPagination(query: any): DeltaPagination { - if (!query) return defaultDeltaPagination(); - - const output: DeltaPagination = {}; - if ('limit' in query) output.limit = query.limit; - if ('cursor' in query) output.cursor = query.cursor; - return output; -} - export function paginationToQueryParams(pagination: Pagination): PaginationQueryParams { const output: PaginationQueryParams = {}; if (!pagination) return {}; @@ -152,6 +142,8 @@ export interface PageLink { } export function filterPaginationQueryParams(query: any): PaginationQueryParams { + if (!query) return {}; + const baseUrlQuery: PaginationQueryParams = {}; if (query.limit) baseUrlQuery.limit = query.limit; if (query.order_by) baseUrlQuery.order_by = query.order_by; @@ -221,7 +213,13 @@ export function createPaginationLinks(page: number, pageCount: number, urlTempla // return output; // } -export async function paginateDbQuery(query: Knex.QueryBuilder, pagination: Pagination, mainTable: string = ''): Promise { + +export async function paginateDbQuery(query: Knex.QueryBuilder, pagination: Pagination, mainTable: string = ''): Promise> { + pagination = { + ...defaultPagination(), + ...pagination, + }; + pagination = processCursor(pagination); const orderSql: any[] = pagination.order.map(o => { diff --git a/packages/server/src/routes/api/batch_items.ts b/packages/server/src/routes/api/batch_items.ts index c46b4d7e3..f8f7d3a7c 100644 --- a/packages/server/src/routes/api/batch_items.ts +++ b/packages/server/src/routes/api/batch_items.ts @@ -8,7 +8,7 @@ import { PaginatedResults } from '../../models/utils/pagination'; const router = new Router(RouteType.Api); router.put('api/batch_items', async (path: SubPath, ctx: AppContext) => { - const output: PaginatedResults = { + const output: PaginatedResults = { items: await putItemContents(path, ctx, true) as any, has_more: false, }; diff --git a/packages/server/src/routes/api/debug.ts b/packages/server/src/routes/api/debug.ts index 1c2da24ef..2ab7f8baf 100644 --- a/packages/server/src/routes/api/debug.ts +++ b/packages/server/src/routes/api/debug.ts @@ -1,5 +1,5 @@ import config from '../../config'; -import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools'; +import { clearDatabase, createTestUsers, CreateTestUsersOptions, createUserDeletions } from '../../tools/debugTools'; import { bodyFields } from '../../utils/requestUtils'; import Router from '../../utils/Router'; import { Env, RouteType } from '../../utils/types'; @@ -34,6 +34,10 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => { await createTestUsers(ctx.joplin.db, config(), options); } + if (query.action === 'createUserDeletions') { + await createUserDeletions(ctx.joplin.db, config()); + } + if (query.action === 'clearDatabase') { await clearDatabase(ctx.joplin.db); } diff --git a/packages/server/src/routes/api/items.test.ts b/packages/server/src/routes/api/items.test.ts index 427c49be4..fd8b25f73 100644 --- a/packages/server/src/routes/api/items.test.ts +++ b/packages/server/src/routes/api/items.test.ts @@ -154,7 +154,7 @@ describe('api_items', function() { test('should batch upload items', async function() { const { session: session1 } = await createUserAndSession(1, false); - const result: PaginatedResults = await putApi(session1.id, 'batch_items', { + const result: PaginatedResults = await putApi(session1.id, 'batch_items', { items: [ { name: '00000000000000000000000000000001.md', @@ -177,7 +177,7 @@ describe('api_items', function() { const note1 = makeNoteSerializedBody({ id: '00000000000000000000000000000001' }); await models().user().save({ id: user1.id, max_item_size: note1.length }); - const result: PaginatedResults = await putApi(session1.id, 'batch_items', { + const result: PaginatedResults = await putApi(session1.id, 'batch_items', { items: [ { name: '00000000000000000000000000000001.md', diff --git a/packages/server/src/routes/api/items.ts b/packages/server/src/routes/api/items.ts index 04a1d231c..5014f75b3 100644 --- a/packages/server/src/routes/api/items.ts +++ b/packages/server/src/routes/api/items.ts @@ -7,10 +7,11 @@ import { AppContext } from '../../utils/types'; import * as fs from 'fs-extra'; import { ErrorForbidden, ErrorMethodNotAllowed, ErrorNotFound, ErrorPayloadTooLarge, errorToPlainObject } from '../../utils/errors'; import ItemModel, { ItemSaveOption, SaveFromRawContentItem } from '../../models/ItemModel'; -import { requestDeltaPagination, requestPagination } from '../../models/utils/pagination'; +import { requestPagination } from '../../models/utils/pagination'; import { AclAction } from '../../models/BaseModel'; import { safeRemove } from '../../utils/fileUtils'; import { formatBytes, MB } from '../../utils/bytes'; +import { requestDeltaPagination } from '../../models/ChangeModel'; const router = new Router(RouteType.Api); diff --git a/packages/server/src/routes/api/share_users.test.ts b/packages/server/src/routes/api/share_users.test.ts index 3c9922439..117750238 100644 --- a/packages/server/src/routes/api/share_users.test.ts +++ b/packages/server/src/routes/api/share_users.test.ts @@ -32,7 +32,7 @@ describe('share_users', function() { const { share: share1 } = await shareWithUserAndAccept(session1.id, session2.id, user2, ShareType.Folder, folderItem1); const { share: share2 } = await shareWithUserAndAccept(session1.id, session2.id, user2, ShareType.Folder, folderItem2); - const shareUsers = await getApi(session2.id, 'share_users'); + const shareUsers = await getApi>(session2.id, 'share_users'); expect(shareUsers.items.length).toBe(2); expect(shareUsers.items.find(su => su.share.id === share1.id)).toBeTruthy(); expect(shareUsers.items.find(su => su.share.id === share2.id)).toBeTruthy(); diff --git a/packages/server/src/routes/api/shares.test.ts b/packages/server/src/routes/api/shares.test.ts index 7917a1534..1cdb222ef 100644 --- a/packages/server/src/routes/api/shares.test.ts +++ b/packages/server/src/routes/api/shares.test.ts @@ -49,7 +49,7 @@ describe('shares', function() { }); { - const shares = await getApi(session1.id, 'shares'); + const shares = await getApi>(session1.id, 'shares'); expect(shares.items.length).toBe(2); const share1: Share = shares.items.find(it => it.folder_id === '000000000000000000000000000000F1'); @@ -60,7 +60,7 @@ describe('shares', function() { expect(share2).toBeTruthy(); expect(share2.type).toBe(ShareType.Note); - const shareUsers = await getApi(session1.id, `shares/${share1.id}/users`); + const shareUsers = await getApi>(session1.id, `shares/${share1.id}/users`); expect(shareUsers.items.length).toBe(2); const su2 = shareUsers.items.find(su => su.user.email === 'user2@localhost'); diff --git a/packages/server/src/routes/index/tasks.ts b/packages/server/src/routes/index/tasks.ts index f2fd98990..977c79728 100644 --- a/packages/server/src/routes/index/tasks.ts +++ b/packages/server/src/routes/index/tasks.ts @@ -12,6 +12,7 @@ import { createCsrfTag } from '../../utils/csrf'; import { RunType } from '../../services/TaskService'; import { NotificationKey } from '../../models/NotificationModel'; import { NotificationLevel } from '../../services/database/types'; +const prettyCron = require('prettycron'); const router: Router = new Router(RouteType.Web); @@ -76,6 +77,7 @@ router.get('tasks', async (_path: SubPath, ctx: AppContext) => { }, { value: task.schedule, + hint: prettyCron.toString(task.schedule), }, { value: yesOrNo(state.running), diff --git a/packages/server/src/routes/index/user_deletions.ts b/packages/server/src/routes/index/user_deletions.ts new file mode 100644 index 000000000..d6e179a4d --- /dev/null +++ b/packages/server/src/routes/index/user_deletions.ts @@ -0,0 +1,150 @@ +import { makeUrl, redirect, SubPath, UrlType } from '../../utils/routeUtils'; +import Router from '../../utils/Router'; +import { RouteType } from '../../utils/types'; +import { AppContext } from '../../utils/types'; +import { ErrorBadRequest, ErrorForbidden, ErrorMethodNotAllowed } from '../../utils/errors'; +import defaultView from '../../utils/defaultView'; +import { yesOrNo } from '../../utils/strings'; +import { makeTablePagination, makeTableView, Row, Table } from '../../utils/views/table'; +import { PaginationOrderDir } from '../../models/utils/pagination'; +import { formatDateTime } from '../../utils/time'; +import { userDeletionsUrl, userUrl } from '../../utils/urlUtils'; +import { createCsrfTag } from '../../utils/csrf'; +import { bodyFields } from '../../utils/requestUtils'; + +const router: Router = new Router(RouteType.Web); + +router.get('user_deletions', async (_path: SubPath, ctx: AppContext) => { + const user = ctx.joplin.owner; + if (!user.is_admin) throw new ErrorForbidden(); + + if (ctx.method === 'GET') { + const pagination = makeTablePagination(ctx.query, 'scheduled_time', PaginationOrderDir.ASC); + const page = await ctx.joplin.models.userDeletion().allPaginated(pagination); + const users = await ctx.joplin.models.user().loadByIds(page.items.map(d => d.user_id), { fields: ['id', 'email'] }); + + console.info(page); + + const table: Table = { + baseUrl: userDeletionsUrl(), + requestQuery: ctx.query, + pageCount: page.page_count, + pagination, + headers: [ + { + name: 'select', + label: '', + canSort: false, + }, + { + name: 'email', + label: 'Email', + stretch: true, + }, + { + name: 'process_data', + label: 'Data?', + }, + { + name: 'process_account', + label: 'Account?', + }, + { + name: 'scheduled_time', + label: 'Scheduled', + }, + { + name: 'start_time', + label: 'Start', + }, + { + name: 'end_time', + label: 'End', + }, + { + name: 'success', + label: 'Success?', + }, + { + name: 'error', + label: 'Error', + }, + ], + rows: page.items.map(d => { + const isDone = d.end_time && d.success; + + const row: Row = [ + { + value: `checkbox_${d.id}`, + checkbox: true, + }, + { + value: isDone ? d.user_id : users.find(u => u.id === d.user_id).email, + stretch: true, + url: isDone ? '' : userUrl(d.user_id), + }, + { + value: yesOrNo(d.process_data), + }, + { + value: yesOrNo(d.process_account), + }, + { + value: formatDateTime(d.scheduled_time), + }, + { + value: formatDateTime(d.start_time), + }, + { + value: formatDateTime(d.end_time), + }, + { + value: d.end_time ? yesOrNo(d.success) : '-', + }, + { + value: d.error, + }, + ]; + + return row; + }), + }; + + const view = defaultView('user_deletions', 'User deletions'); + view.content = { + userDeletionTable: makeTableView(table), + postUrl: makeUrl(UrlType.UserDeletions), + csrfTag: await createCsrfTag(ctx), + }; + view.cssFiles = ['index/user_deletions']; + + return view; + } + + throw new ErrorMethodNotAllowed(); +}); + +router.post('user_deletions', async (_path: SubPath, ctx: AppContext) => { + const user = ctx.joplin.owner; + if (!user.is_admin) throw new ErrorForbidden(); + + interface PostFields { + removeButton: string; + } + + const models = ctx.joplin.models; + + const fields: PostFields = await bodyFields(ctx.req); + + if (fields.removeButton) { + const jobIds = Object.keys(fields).filter(f => f.startsWith('checkbox_')).map(f => Number(f.substr(9))); + for (const jobId of jobIds) await models.userDeletion().remove(jobId); + await models.notification().addInfo(user.id, `${jobIds.length} job(s) have been removed`); + } else { + throw new ErrorBadRequest('Invalid action'); + } + + return redirect(ctx, makeUrl(UrlType.UserDeletions)); +}); + +export default router; diff --git a/packages/server/src/routes/index/users.ts b/packages/server/src/routes/index/users.ts index 52be0c262..11cd3899b 100644 --- a/packages/server/src/routes/index/users.ts +++ b/packages/server/src/routes/index/users.ts @@ -15,10 +15,10 @@ import uuidgen from '../../utils/uuidgen'; import { formatMaxItemSize, formatMaxTotalSize, formatTotalSize, formatTotalSizePercent, yesOrNo } from '../../utils/strings'; import { getCanShareFolder, totalSizeClass } from '../../models/utils/user'; import { yesNoDefaultOptions, yesNoOptions } from '../../utils/views/select'; -import { confirmUrl, stripePortalUrl } from '../../utils/urlUtils'; +import { confirmUrl, stripePortalUrl, userDeletionsUrl } from '../../utils/urlUtils'; import { cancelSubscriptionByUserId, updateCustomerEmail, updateSubscriptionType } from '../../utils/stripe'; import { createCsrfTag } from '../../utils/csrf'; -import { formatDateTime } from '../../utils/time'; +import { formatDateTime, Hour } from '../../utils/time'; import { cookieSet } from '../../utils/cookies'; import { startImpersonating, stopImpersonating } from './utils/users/impersonate'; import { userFlagToString } from '../../models/UserFlagModel'; @@ -106,20 +106,23 @@ router.get('users', async (_path: SubPath, ctx: AppContext) => { }); const view: View = defaultView('users', 'Users'); - view.content.users = users.map(user => { - return { - ...user, - displayName: user.full_name ? user.full_name : '(not set)', - formattedItemMaxSize: formatMaxItemSize(user), - formattedTotalSize: formatTotalSize(user), - formattedMaxTotalSize: formatMaxTotalSize(user), - formattedTotalSizePercent: formatTotalSizePercent(user), - totalSizeClass: totalSizeClass(user), - formattedAccountType: accountTypeToString(user.account_type), - formattedCanShareFolder: yesOrNo(getCanShareFolder(user)), - rowClassName: user.enabled ? '' : 'is-disabled', - }; - }); + view.content = { + users: users.map(user => { + return { + ...user, + displayName: user.full_name ? user.full_name : '(not set)', + formattedItemMaxSize: formatMaxItemSize(user), + formattedTotalSize: formatTotalSize(user), + formattedMaxTotalSize: formatMaxTotalSize(user), + formattedTotalSizePercent: formatTotalSizePercent(user), + totalSizeClass: totalSizeClass(user), + formattedAccountType: accountTypeToString(user.account_type), + formattedCanShareFolder: yesOrNo(getCanShareFolder(user)), + rowClassName: user.enabled ? '' : 'is-disabled', + }; + }), + userDeletionUrl: userDeletionsUrl(), + }; return view; }); @@ -163,6 +166,7 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null if (!owner.is_admin) userFlagViews = []; const subscription = !isNew ? await ctx.joplin.models.subscription().byUserId(userId) : null; + const isScheduledForDeletion = await ctx.joplin.models.userDeletion().isScheduledForDeletion(userId); const view: View = defaultView('user', 'Profile'); view.content.user = user; @@ -186,6 +190,7 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null view.content.showImpersonateButton = !isNew && !!owner.is_admin && user.enabled && user.id !== owner.id; view.content.showRestoreButton = !isNew && !!owner.is_admin && !user.enabled; + view.content.showScheduleDeletionButton = !isNew && !!owner.is_admin && !isScheduledForDeletion; view.content.showResetPasswordButton = !isNew && owner.is_admin && user.enabled; view.content.canShareFolderOptions = yesNoDefaultOptions(user, 'can_share_folder'); view.content.canUploadOptions = yesNoOptions(user, 'can_upload'); @@ -308,6 +313,7 @@ interface FormFields { impersonate_button: string; stop_impersonate_button: string; delete_user_flags: string; + schedule_deletion_button: string; } router.post('users', async (path: SubPath, ctx: AppContext) => { @@ -364,6 +370,15 @@ router.post('users', async (path: SubPath, ctx: AppContext) => { await updateSubscriptionType(models, userId, AccountType.Basic); } else if (fields.update_subscription_pro_button) { await updateSubscriptionType(models, userId, AccountType.Pro); + } else if (fields.schedule_deletion_button) { + const deletionDate = Date.now() + 24 * Hour; + + await models.userDeletion().add(userId, deletionDate, { + processAccount: true, + processData: true, + }); + + await models.notification().addInfo(owner.id, `User ${user.email} has been scheduled for deletion on ${formatDateTime(deletionDate)}. [View deletion list](${userDeletionsUrl()})`); } else if (fields.delete_user_flags) { const userFlagTypes: UserFlagType[] = []; for (const key of Object.keys(fields)) { diff --git a/packages/server/src/routes/routes.ts b/packages/server/src/routes/routes.ts index f6a6db1d1..2b97c6f74 100644 --- a/packages/server/src/routes/routes.ts +++ b/packages/server/src/routes/routes.ts @@ -28,6 +28,7 @@ import indexTasks from './index/tasks'; import indexTerms from './index/terms'; import indexUpgrade from './index/upgrade'; import indexUsers from './index/users'; +import indexUserDeletions from './index/user_deletions'; import defaultRoute from './default'; @@ -60,6 +61,7 @@ const routes: Routers = { 'upgrade': indexUpgrade, 'help': indexHelp, 'tasks': indexTasks, + 'user_deletions': indexUserDeletions, '': defaultRoute, }; diff --git a/packages/server/src/services/BaseService.ts b/packages/server/src/services/BaseService.ts index 9d7351645..a12d0a28f 100644 --- a/packages/server/src/services/BaseService.ts +++ b/packages/server/src/services/BaseService.ts @@ -10,6 +10,7 @@ export default class BaseService { private env_: Env; private models_: Models; private config_: Config; + protected name_: string = 'Untitled'; protected enabled_: boolean = true; private destroyed_: boolean = false; protected maintenanceInterval_: number = 10000; @@ -23,8 +24,12 @@ export default class BaseService { this.scheduleMaintenance = this.scheduleMaintenance.bind(this); } + protected get name(): string { + return this.name_; + } + public async destroy() { - if (this.destroyed_) throw new Error('Already destroyed'); + if (this.destroyed_) throw new Error(`${this.name}: Already destroyed`); this.destroyed_ = true; this.scheduledMaintenances_ = []; @@ -75,12 +80,17 @@ export default class BaseService { } } - private async runMaintenance() { + public async runMaintenance() { + if (this.maintenanceInProgress_) { + logger.warn(`${this.name}: Skipping maintenance because it is already in progress`); + return; + } + this.maintenanceInProgress_ = true; try { await this.maintenance(); } catch (error) { - logger.error('Could not run maintenance', error); + logger.error(`${this.name}: Could not run maintenance`, error); } this.maintenanceInProgress_ = false; } diff --git a/packages/server/src/services/TaskService.test.ts b/packages/server/src/services/TaskService.test.ts index b2fca4bd7..23ba02ebf 100644 --- a/packages/server/src/services/TaskService.test.ts +++ b/packages/server/src/services/TaskService.test.ts @@ -5,7 +5,13 @@ import { Env } from '../utils/types'; import TaskService, { RunType, Task } from './TaskService'; const newService = () => { - return new TaskService(Env.Dev, models(), config()); + return new TaskService(Env.Dev, models(), config(), { + share: null, + email: null, + mustache: null, + tasks: null, + userDeletion: null, + }); }; describe('TaskService', function() { diff --git a/packages/server/src/services/TaskService.ts b/packages/server/src/services/TaskService.ts index c9c8ebaa5..b2167e48a 100644 --- a/packages/server/src/services/TaskService.ts +++ b/packages/server/src/services/TaskService.ts @@ -1,7 +1,9 @@ import Logger from '@joplin/lib/Logger'; import { Models } from '../models/factory'; +import { Config, Env } from '../utils/types'; import BaseService from './BaseService'; import { Event, EventType } from './database/types'; +import { Services } from './types'; const cron = require('node-cron'); const logger = Logger.create('TaskService'); @@ -14,6 +16,7 @@ export enum TaskId { HandleFailedPaymentSubscriptions = 5, DeleteExpiredSessions = 6, CompressOldChanges = 7, + ProcessUserDeletions = 8, } export enum RunType { @@ -31,7 +34,7 @@ export interface Task { id: TaskId; description: string; schedule: string; - run(models: Models): void; + run(models: Models, services: Services): void; } export type Tasks = Record; @@ -53,6 +56,12 @@ export default class TaskService extends BaseService { private tasks_: Tasks = {}; private taskStates_: Record = {}; + private services_: Services; + + public constructor(env: Env, models: Models, config: Config, services: Services) { + super(env, models, config); + this.services_ = services; + } public registerTask(task: Task) { if (this.tasks_[task.id]) throw new Error(`Already a task with this ID: ${task.id}`); @@ -106,7 +115,7 @@ export default class TaskService extends BaseService { try { logger.info(`Running ${displayString} (${runTypeToString(runType)})...`); - await this.tasks_[id].run(this.models); + await this.tasks_[id].run(this.models, this.services_); } catch (error) { logger.error(`On ${displayString}`, error); } diff --git a/packages/server/src/services/UserDeletionService.test.ts b/packages/server/src/services/UserDeletionService.test.ts new file mode 100644 index 000000000..6dd2406c9 --- /dev/null +++ b/packages/server/src/services/UserDeletionService.test.ts @@ -0,0 +1,137 @@ +import config from '../config'; +import { shareFolderWithUser } from '../utils/testing/shareApiUtils'; +import { afterAllTests, beforeAllDb, beforeEachDb, createNote, createUserAndSession, models } from '../utils/testing/testUtils'; +import { Env } from '../utils/types'; +import UserDeletionService from './UserDeletionService'; + +const newService = () => { + return new UserDeletionService(Env.Dev, models(), config()); +}; + +describe('UserDeletionService', function() { + + beforeAll(async () => { + await beforeAllDb('UserDeletionService'); + }); + + afterAll(async () => { + await afterAllTests(); + }); + + beforeEach(async () => { + await beforeEachDb(); + }); + + test('should delete user data', async function() { + const { user: user1, session: session1 } = await createUserAndSession(1); + const { user: user2, session: session2 } = await createUserAndSession(2); + await createNote(session1.id, { title: 'testing1' }); + await createNote(session2.id, { title: 'testing2' }); + + const t0 = new Date('2021-12-14').getTime(); + const t1 = t0 + 1000; + + const job = await models().userDeletion().add(user1.id, t1, { + processData: true, + processAccount: false, + }); + + expect(await models().item().count()).toBe(2); + expect(await models().change().count()).toBe(2); + + const service = newService(); + await service.processDeletionJob(job, { sleepBetweenOperations: 0 }); + + expect(await models().item().count()).toBe(1); + expect(await models().change().count()).toBe(1); + + const item = (await models().item().all())[0]; + expect(item.owner_id).toBe(user2.id); + + const change = (await models().change().all())[0]; + expect(change.user_id).toBe(user2.id); + + expect(await models().user().count()).toBe(2); + expect(await models().session().count()).toBe(2); + }); + + test('should delete user account', async function() { + const { user: user1 } = await createUserAndSession(1); + const { user: user2 } = await createUserAndSession(2); + + const t0 = new Date('2021-12-14').getTime(); + const t1 = t0 + 1000; + + const job = await models().userDeletion().add(user1.id, t1, { + processData: false, + processAccount: true, + }); + + expect(await models().user().count()).toBe(2); + expect(await models().session().count()).toBe(2); + + const service = newService(); + await service.processDeletionJob(job, { sleepBetweenOperations: 0 }); + + expect(await models().user().count()).toBe(1); + expect(await models().session().count()).toBe(1); + + const user = (await models().user().all())[0]; + expect(user.id).toBe(user2.id); + }); + + test('should not delete notebooks that are not owned', async function() { + const { session: session1 } = await createUserAndSession(1); + const { user: user2, session: session2 } = await createUserAndSession(2); + + await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F2', [ + { + id: '000000000000000000000000000000F2', + children: [ + { + id: '00000000000000000000000000000001', + }, + ], + }, + ]); + + expect(await models().share().count()).toBe(1); + expect(await models().shareUser().count()).toBe(1); + + const job = await models().userDeletion().add(user2.id, Date.now()); + const service = newService(); + await service.processDeletionJob(job, { sleepBetweenOperations: 0 }); + + expect(await models().share().count()).toBe(1); // The share object has not (and should not) been deleted + expect(await models().shareUser().count()).toBe(0); // However all the invitations are gone + expect(await models().item().count()).toBe(2); + }); + + test('should not delete notebooks that are owned', async function() { + const { user: user1, session: session1 } = await createUserAndSession(1); + const { session: session2 } = await createUserAndSession(2); + + await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F2', [ + { + id: '000000000000000000000000000000F2', + children: [ + { + id: '00000000000000000000000000000001', + }, + ], + }, + ]); + + expect(await models().share().count()).toBe(1); + expect(await models().shareUser().count()).toBe(1); + + const job = await models().userDeletion().add(user1.id, Date.now()); + const service = newService(); + await service.processDeletionJob(job, { sleepBetweenOperations: 0 }); + + expect(await models().share().count()).toBe(0); + expect(await models().shareUser().count()).toBe(0); + expect(await models().item().count()).toBe(0); + }); + +}); diff --git a/packages/server/src/services/UserDeletionService.ts b/packages/server/src/services/UserDeletionService.ts new file mode 100644 index 000000000..c7f248a41 --- /dev/null +++ b/packages/server/src/services/UserDeletionService.ts @@ -0,0 +1,106 @@ +import Logger from '@joplin/lib/Logger'; +import { Pagination } from '../models/utils/pagination'; +import { msleep } from '../utils/time'; +import BaseService from './BaseService'; +import { UserDeletion, UserFlagType, Uuid } from './database/types'; + +const logger = Logger.create('UserDeletionService'); + +export interface DeletionJobOptions { + sleepBetweenOperations?: number; +} + +export default class UserDeletionService extends BaseService { + + protected name_: string = 'UserDeletionService'; + + private async deleteUserData(userId: Uuid, options: DeletionJobOptions) { + // While the "UserDeletionInProgress" flag is on, the account is + // disabled so that no new items or other changes can happen. + await this.models.userFlag().add(userId, UserFlagType.UserDeletionInProgress); + + try { + // --------------------------------------------------------------------- + // Delete own shares and shares participated in. Note that when the + // shares are deleted, the associated user_items are deleted too, so we + // don't need to wait for ShareService to run to continue. + // --------------------------------------------------------------------- + + logger.info(`Deleting shares for user ${userId}`); + + await this.models.share().deleteByUserId(userId); + await this.models.shareUser().deleteByUserId(userId); + + // --------------------------------------------------------------------- + // Delete items. Also delete associated change objects. + // --------------------------------------------------------------------- + + logger.info(`Deleting items for user ${userId}`); + + while (true) { + const pagination: Pagination = { + limit: 1000, + }; + + const page = await this.models.item().children(userId, '', pagination, { fields: ['id'] }); + if (!page.items.length) break; + + await this.models.item().delete(page.items.map(it => it.id), { + deleteChanges: true, + }); + + await msleep(options.sleepBetweenOperations); + } + } finally { + await this.models.userFlag().remove(userId, UserFlagType.UserDeletionInProgress); + } + } + + private async deleteUserAccount(userId: Uuid, _options: DeletionJobOptions = null) { + logger.info(`Deleting user account: ${userId}`); + + await this.models.userFlag().add(userId, UserFlagType.UserDeletionInProgress); + + await this.models.session().deleteByUserId(userId); + await this.models.notification().deleteByUserId(userId); + await this.models.user().delete(userId); + await this.models.userFlag().deleteByUserId(userId); + } + + public async processDeletionJob(deletion: UserDeletion, options: DeletionJobOptions = null) { + options = { + sleepBetweenOperations: 5000, + ...options, + }; + + logger.info('Starting user deletion: ', deletion); + + let error: any = null; + let success: boolean = true; + + try { + await this.models.userDeletion().start(deletion.id); + if (deletion.process_data) await this.deleteUserData(deletion.user_id, options); + if (deletion.process_account) await this.deleteUserAccount(deletion.user_id, options); + } catch (e) { + error = e; + success = false; + logger.error(`Processing deletion ${deletion.id}:`, error); + } + + await this.models.userDeletion().end(deletion.id, success, error); + + logger.info('Completed user deletion: ', deletion.id); + } + + public async processNextDeletionJob() { + const deletion = await this.models.userDeletion().next(); + if (!deletion) return; + await this.processDeletionJob(deletion); + } + + protected async maintenance() { + await this.processNextDeletionJob(); + } + +} diff --git a/packages/server/src/services/database/types.ts b/packages/server/src/services/database/types.ts index d56ec7609..5f23770be 100644 --- a/packages/server/src/services/database/types.ts +++ b/packages/server/src/services/database/types.ts @@ -40,6 +40,7 @@ export enum UserFlagType { AccountWithoutSubscription = 4, SubscriptionCancelled = 5, ManuallyDisabled = 6, + UserDeletionInProgress = 7, } export function userFlagTypeToLabel(t: UserFlagType): string { @@ -50,6 +51,7 @@ export function userFlagTypeToLabel(t: UserFlagType): string { [UserFlagType.AccountWithoutSubscription]: 'Account Without Subscription', [UserFlagType.SubscriptionCancelled]: 'Subscription Cancelled', [UserFlagType.ManuallyDisabled]: 'Manually Disabled', + [UserFlagType.UserDeletionInProgress]: 'User deletion in progress', }; if (!s[t]) throw new Error(`Unknown flag type: ${t}`); @@ -268,6 +270,18 @@ export interface Item extends WithDates, WithUuid { content_storage_id?: number; } +export interface UserDeletion extends WithDates { + id?: number; + user_id?: Uuid; + process_data?: number; + process_account?: number; + scheduled_time?: number; + start_time?: number; + end_time?: number; + success?: number; + error?: string; +} + export const databaseSchema: DatabaseTables = { sessions: { id: { type: 'string' }, @@ -449,5 +463,18 @@ export const databaseSchema: DatabaseTables = { owner_id: { type: 'string' }, content_storage_id: { type: 'number' }, }, + user_deletions: { + id: { type: 'number' }, + user_id: { type: 'string' }, + process_data: { type: 'number' }, + process_account: { type: 'number' }, + scheduled_time: { type: 'string' }, + start_time: { type: 'string' }, + end_time: { type: 'string' }, + success: { type: 'number' }, + error: { type: 'string' }, + updated_time: { type: 'string' }, + created_time: { type: 'string' }, + }, }; // AUTO-GENERATED-TYPES diff --git a/packages/server/src/services/types.ts b/packages/server/src/services/types.ts index 02d278377..41e16d3be 100644 --- a/packages/server/src/services/types.ts +++ b/packages/server/src/services/types.ts @@ -2,10 +2,12 @@ import EmailService from './EmailService'; import MustacheService from './MustacheService'; import ShareService from './ShareService'; import TaskService from './TaskService'; +import UserDeletionService from './UserDeletionService'; export interface Services { share: ShareService; email: EmailService; mustache: MustacheService; tasks: TaskService; + userDeletion: UserDeletionService; } diff --git a/packages/server/src/tools/debugTools.ts b/packages/server/src/tools/debugTools.ts index 6737e5a69..9865d5b35 100644 --- a/packages/server/src/tools/debugTools.ts +++ b/packages/server/src/tools/debugTools.ts @@ -3,6 +3,7 @@ import { DbConnection, dropTables, migrateLatest } from '../db'; import newModelFactory from '../models/factory'; import { AccountType } from '../models/UserModel'; import { User, UserFlagType } from '../services/database/types'; +import { Minute, Second } from '../utils/time'; import { Config } from '../utils/types'; export interface CreateTestUsersOptions { @@ -98,3 +99,15 @@ export async function createTestUsers(db: DbConnection, config: Config, options: } } } + +export async function createUserDeletions(db: DbConnection, config: Config) { + const models = newModelFactory(db, config); + + const users = await models.user().all(); + + for (let i = 0; i < 3; i++) { + if (i >= users.length) break; + if (users[i].is_admin) continue; + await models.userDeletion().add(users[i].id, Date.now() + 60 * Second + (i * 10 * Minute)); + } +} diff --git a/packages/server/src/tools/generateTypes.ts b/packages/server/src/tools/generateTypes.ts index 96e9c1b52..7d80320df 100644 --- a/packages/server/src/tools/generateTypes.ts +++ b/packages/server/src/tools/generateTypes.ts @@ -35,6 +35,7 @@ const config = { 'main.user_items': 'WithDates', 'main.users': 'WithDates, WithUuid', 'main.events': 'WithUuid', + 'main.user_deletions': 'WithDates', }, }; @@ -58,6 +59,9 @@ const propertyTypes: Record = { 'users.total_item_size': 'number', 'events.created_time': 'number', 'events.type': 'EventType', + 'user_deletions.start_time': 'number', + 'user_deletions.end_time': 'number', + 'user_deletions.scheduled_time': 'number', }; function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void { diff --git a/packages/server/src/utils/errors.ts b/packages/server/src/utils/errors.ts index 2bfcb8f91..3ed81d357 100644 --- a/packages/server/src/utils/errors.ts +++ b/packages/server/src/utils/errors.ts @@ -130,23 +130,29 @@ export class ErrorTooManyRequests extends ApiError { } export function errorToString(error: Error): string { - const msg: string[] = []; - msg.push(error.message ? error.message : 'Unknown error'); - if (error.stack) msg.push(error.stack); - return msg.join(': '); + // const msg: string[] = []; + // msg.push(error.message ? error.message : 'Unknown error'); + // if (error.stack) msg.push(error.stack); + // return msg.join(': '); + + return JSON.stringify(errorToPlainObject(error)); } interface PlainObjectError { httpCode?: number; message?: string; code?: string; + stack?: string; } export function errorToPlainObject(error: any): PlainObjectError { + if (typeof error === 'string') return { message: error }; + const output: PlainObjectError = {}; if ('httpCode' in error) output.httpCode = error.httpCode; if ('code' in error) output.code = error.code; if ('message' in error) output.message = error.message; + if ('stack' in error) output.stack = error.stack; return output; } diff --git a/packages/server/src/utils/prettycron.testdisabled.ts b/packages/server/src/utils/prettycron.testdisabled.ts new file mode 100644 index 000000000..d834afcc6 --- /dev/null +++ b/packages/server/src/utils/prettycron.testdisabled.ts @@ -0,0 +1,58 @@ +// const prettycron = require('./prettycron'); + +// describe('prettycron', function() { + +// it('should check if an item is encrypted', async function() { +// const testCases = [ +// { cron: '0 * * * *', readable: 'Every hour, on the hour', sixth: false }, +// { cron: '30 * * * 1', readable: 'Every 30th minute past every hour on Mon', sixth: false }, +// { cron: '15,45 9,21 * * *', readable: '09:15, 09:45, 21:15 and 21:45 every day', sixth: false }, +// { cron: '18,19 7 5 * *', readable: '07:18 and 07:19 on the 5th of every month', sixth: false }, +// { cron: '* * 25 12 *', readable: 'Every minute on the 25th in Dec', sixth: false }, +// { cron: '0 * 1,3 * *', readable: 'Every hour, on the hour on the 1 and 3rd of every month', sixth: false }, +// { cron: '0 17 * 1,4,7,10 *', readable: '17:00 every day in Jan, Apr, Jul and Oct', sixth: false }, +// { cron: '15 * * * 1,2', readable: 'Every 15th minute past every hour on Mon and Tue', sixth: false }, +// { cron: '* 8,10,12,14,16,18,20 * * *', readable: 'Every minute of 8, 10, 12, 14, 16, 18 and 20th hour', sixth: false }, +// { cron: '0 12 15,16 1 3', readable: '12:00 on the 15 and 16th and every Wed in Jan', sixth: false }, +// { cron: '0 4,8,12,4 * * 4,5,6', readable: 'On the 4, 8 and 12th hour on Thu, Fri and Sat', sixth: false }, +// { cron: '0 2,16 1,8,15,22 * 1,2', readable: '02:00 and 16:00 on the 1, 8, 15 and 22nd of every month and every Mon and Tue', sixth: false }, +// { cron: '15 3,8,10,12,14,16,18 16 * *', readable: 'Every 15th minute past the 3, 8, 10, 12, 14, 16 and 18th hour on the 16th of every month', sixth: false }, +// { cron: '2 8,10,12,14,16,18 * 8 0,3', readable: 'Every 2nd minute past the 8, 10, 12, 14, 16 and 18th hour on Sun and Wed in Aug', sixth: false }, +// { cron: '0 0 18 1/1 * ?', readable: '00:00 on the 18th of every month', sixth: false }, +// { cron: '30 10 * * 0', readable: '10:30 on Sun', sixth: false }, +// { cron: '* * * * *', readable: 'Every minute', sixth: false }, +// { cron: '*/2 * * * *', readable: 'Every other minute', sixth: false }, + +// { cron: '0 0 18 1/1 * ? *', readable: '18:00:00 every day', sixth: true }, +// { cron: '* * * * * *', readable: 'Every second', sixth: true }, +// { cron: '0/1 0/1 0/1 0/1 0/1 0/1', readable: 'Every second', sixth: true }, +// { cron: '*/4 2 4 * * *', readable: 'Every 4 seconds on the 2nd minute past the 4th hour', sixth: true }, +// { cron: '30 15 9 * * *', readable: '09:15:30 every day', sixth: true }, +// { cron: '*/30 15 9 * * *', readable: '09:15:00 and 09:15:30 every day', sixth: true }, +// { cron: '*/2 * * * * *', readable: 'Every other second', sixth: true }, +// { cron: '*/3 * * * * *', readable: 'Every 3 seconds', sixth: true }, +// { cron: '*/4 * * * * *', readable: 'Every 4 seconds', sixth: true }, +// { cron: '*/5 * * * * *', readable: 'Every 5 seconds', sixth: true }, +// { cron: '*/6 * * * * *', readable: 'Every 6 seconds', sixth: true }, +// { cron: '*/10 * * * * *', readable: 'Every 10 seconds', sixth: true }, +// { cron: '*/12 * * * * *', readable: 'Every 12 seconds', sixth: true }, +// { cron: '*/15 * * * * *', readable: 'Every 15 seconds', sixth: true }, +// { cron: '*/20 * * * * *', readable: 'Every 20 seconds', sixth: true }, +// { cron: '*/30 * * * * *', readable: 'Every minute starting on the first and 30th second', sixth: true }, +// { cron: '5 * * * * *', readable: 'Every minute starting on the 5th second', sixth: true }, +// { cron: '5 */2 * * * *', readable: 'Every other minute starting on the 5th second', sixth: true }, +// { cron: '30 * * * * *', readable: 'Every minute starting on the 30th second', sixth: true }, +// { cron: '0,2,4,20 * * * * *', readable: 'Every minute starting on the 0, 2, 4 and 20th second', sixth: true }, +// { cron: '5,10/30 * * * 1,3 8', readable: 'Every minute starting on the 5, 10 and 40th second on Sat in Jan and Mar', sixth: true }, +// { cron: '15-17 * * * * *', readable: 'Every minute starting on the 15, 16 and 17th second', sixth: true }, +// ]; + +// for (const t of testCases) { +// const input = t.cron; +// const expected = t.readable; +// const actual = prettycron.toString(input, t.sixth); +// expect(actual).toBe(expected); +// } +// }); + +// }); diff --git a/packages/server/src/utils/prettycron.ts b/packages/server/src/utils/prettycron.ts new file mode 100644 index 000000000..30f5307fd --- /dev/null +++ b/packages/server/src/utils/prettycron.ts @@ -0,0 +1,384 @@ +// ============================================================================== +// 2021-12-15 +// Modified for Joplin Server to use dayjs instead of moment.js +// Unfortunately it still requires the "later" library, which huge, so shouldn't +// be used for now +// ============================================================================== + + + +// ////////////////////////////////////////////////////////////////////////////////// +// +// prettycron.js +// Generates human-readable sentences from a schedule string in cron format +// +// Based on an earlier version by Pehr Johansson +// http://dsysadm.blogspot.com.au/2012/09/human-readable-cron-expressions-using.html +// +// ////////////////////////////////////////////////////////////////////////////////// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program. If not, see . +// ////////////////////////////////////////////////////////////////////////////////// + +const dayjs = require('dayjs'); +const advancedFormat = require('dayjs/plugin/advancedFormat'); +const calendar = require('dayjs/plugin/calendar'); +dayjs.extend(advancedFormat); +dayjs.extend(calendar); + +const later = require('later'); + +(function() { + + const ordinal = { + + ordinalSuffix(num: number) { + const ordinalsArray = ['th', 'st', 'nd', 'rd']; + + // Get reminder of number by hundred so that we can counter number between 11-19 + const offset = num % 100; + + // Calculate position of ordinal to be used. Logic : Array index is calculated based on defined values. + const ordinalPos = ordinalsArray[ (offset - 20) % 10 ] || ordinalsArray[ offset ] || ordinalsArray[0]; + + // Return suffix + return ordinalPos; + }, + + toOrdinal(num: number) { + + // Check if number is valid + // if( !validateNumber(num) ) { + // return `${num} is not a valid number`; + // } + + // If number is zero no need to spend time on calculation + if (num === 0) { + return num.toString(); + } + + return num.toString() + this.ordinalSuffix(num); + }, + }; + + // For an array of numbers, e.g. a list of hours in a schedule, + // return a string listing out all of the values (complete with + // "and" plus ordinal text on the last item). + const numberList = function(numbers: any[]) { + if (numbers.length < 2) { + return ordinal.toOrdinal(numbers[0]); + } + + const last_val = numbers.pop(); + return `${numbers.join(', ')} and ${ordinal.toOrdinal(last_val)}`; + }; + + const stepSize = function(numbers: any[]) { + if (!numbers || numbers.length <= 1) return 0; + + const expectedStep = numbers[1] - numbers[0]; + if (numbers.length == 2) return expectedStep; + + // Check that every number is the previous number + the first number + return numbers.slice(1).every(function(n,i,a) { + return (i === 0 ? n : n - a[i - 1]) === expectedStep; + }) ? expectedStep : 0; + }; + + const isEveryOther = function(stepsize: number, numbers: any[]) { + return numbers.length === 30 && stepsize === 2; + }; + const isTwicePerHour = function(stepsize: number, numbers: any[]) { + return numbers.length === 2 && stepsize === 30; + }; + const isOnTheHour = function(numbers: any[]) { + return numbers.length === 1 && numbers[0] === 0; + }; + const isStepValue = function(stepsize: number, numbers: any[]) { + // Value with slash (https://en.wikipedia.org/wiki/Cron#Non-Standard_Characters) + return numbers.length > 2 && stepsize > 0; + }; + // For an array of numbers of seconds, return a string + // listing all the values unless they represent a frequency divisible by 60: + // /2, /3, /4, /5, /6, /10, /12, /15, /20 and /30 + const getMinutesTextParts = function(numbers: any[]) { + const stepsize = stepSize(numbers); + if (!numbers) { + return { beginning: 'minute', text: '' }; + } + + const minutes = { beginning: '', text: '' }; + if (isOnTheHour(numbers)) { + minutes.text = 'hour, on the hour'; + } else if (isEveryOther(stepsize, numbers)) { + minutes.beginning = 'other minute'; + } else if (isStepValue(stepsize, numbers)) { + minutes.text = `${stepsize} minutes`; + } else if (isTwicePerHour(stepsize, numbers)) { + minutes.text = 'first and 30th minute'; + } else { + minutes.text = `${numberList(numbers)} minute`; + } + return minutes; + }; + // For an array of numbers of seconds, return a string + // listing all the values unless they represent a frequency divisible by 60: + // /2, /3, /4, /5, /6, /10, /12, /15, /20 and /30 + const getSecondsTextParts = function(numbers: any[]) { + const stepsize = stepSize(numbers); + if (!numbers) { + return { beginning: 'second', text: '' }; + } + if (isEveryOther(stepsize, numbers)) { + return { beginning: '', text: 'other second' }; + } else if (isStepValue(stepsize, numbers)) { + return { beginning: '', text: `${stepsize} seconds` }; + } else { + return { beginning: 'minute', text: `starting on the ${numbers.length === 2 && stepsize === 30 ? 'first and 30th second' : `${numberList(numbers)} second`}` }; + } + }; + + // Parse a number into day of week, or a month name; + // used in dateList below. + const numberToDateName = function(value: any, type: any) { + if (type === 'dow') { + return dayjs().day(value - 1).format('ddd'); + } else if (type === 'mon') { + return dayjs().month(value - 1).format('MMM'); + } + }; + + // From an array of numbers corresponding to dates (given in type: either + // days of the week, or months), return a string listing all the values. + const dateList = function(numbers: any[], type: any) { + if (numbers.length < 2) { + return numberToDateName(`${numbers[0]}`, type); + } + + const last_val = `${numbers.pop()}`; + const output_text = ''; + + // No idea what is this nonsense so comenting it out for now. + + // for (let i = 0, value; value = numbers[i]; i++) { + // if (output_text.length > 0) { + // output_text += ', '; + // } + // output_text += numberToDateName(value, type); + // } + return `${output_text} and ${numberToDateName(last_val, type)}`; + }; + + // Pad to equivalent of sprintf('%02d'). Both moment.js and later.js + // have zero-fill functions, but alas, they're private. + // let zeroPad = function(x:any) { + // return (x < 10) ? '0' + x : x; + // }; + + const removeFromSchedule = function(schedule: any, member: any, length: any) { + if (schedule[member] && schedule[member].length === length) { + delete schedule[member]; + } + }; + + // ---------------- + + // Given a schedule from later.js (i.e. after parsing the cronspec), + // generate a friendly sentence description. + const scheduleToSentence = function(schedule: any, useSeconds: boolean) { + let textParts = []; + + // A later.js schedules contains no member for time units where an asterisk is used, + // but schedules that means the same (e.g 0/1 is essentially the same as *) are + // returned with populated members. + // Remove all members that are fully populated to reduce complexity of code + removeFromSchedule(schedule, 'M', 12); + removeFromSchedule(schedule, 'D', 31); + removeFromSchedule(schedule, 'd', 7); + removeFromSchedule(schedule, 'h', 24); + removeFromSchedule(schedule, 'm', 60); + removeFromSchedule(schedule, 's', 60); + + // let everySecond = useSeconds && schedule['s'] === undefined; + // let everyMinute = schedule['m'] === undefined; + // let everyHour = schedule['h'] === undefined; + const everyWeekday = schedule['d'] === undefined; + const everyDayInMonth = schedule['D'] === undefined; + // let everyMonth = schedule['M'] === undefined; + + const oneOrTwoSecondsPerMinute = schedule['s'] && schedule['s'].length <= 2; + const oneOrTwoMinutesPerHour = schedule['m'] && schedule['m'].length <= 2; + const oneOrTwoHoursPerDay = schedule['h'] && schedule['h'].length <= 2; + const onlySpecificDaysOfMonth = schedule['D'] && schedule['D'].length !== 31; + if (oneOrTwoHoursPerDay && oneOrTwoMinutesPerHour && oneOrTwoSecondsPerMinute) { + // If there are only one or two specified values for + // hour or minute, print them in HH:MM format, or HH:MM:ss if seconds are used + // If seconds are not used, later.js returns one element for the seconds (set to zero) + + const hm = []; + // let m = dayjs(new Date()); + for (let i = 0; i < schedule['h'].length; i++) { + for (let j = 0; j < schedule['m'].length; j++) { + for (let k = 0; k < schedule['s'].length; k++) { + + const s = dayjs() + .hour(schedule['h'][i]) + .minute(schedule['m'][j]) + .second(schedule['s'][k]) + .format(useSeconds ? 'HH:mm:ss' : 'HH:mm'); + + hm.push(s); + + // m.hour(schedule['h'][i]); + // m.minute(schedule['m'][j]); + // m.second(schedule['s'][k]); + // hm.push(m.format( useSeconds ? 'HH:mm:ss' : 'HH:mm')); + } + } + } + if (hm.length < 2) { + textParts.push(hm[0]); + } else { + const last_val = hm.pop(); + textParts.push(`${hm.join(', ')} and ${last_val}`); + } + if (everyWeekday && everyDayInMonth) { + textParts.push('every day'); + } + + } else { + const seconds = getSecondsTextParts(schedule['s']); + const minutes = getMinutesTextParts(schedule['m']); + let beginning = ''; + let end = ''; + + textParts.push('Every'); + + // Otherwise, list out every specified hour/minute value. + const hasSpecificSeconds = schedule['s'] && ( + schedule['s'].length > 1 && schedule['s'].length < 60 || + schedule['s'].length === 1 && schedule['s'][0] !== 0); + if (hasSpecificSeconds) { + beginning = seconds.beginning; + end = seconds.text; + } + + if (schedule['h']) { // runs only at specific hours + if (hasSpecificSeconds) { + end += ' on the '; + } + if (schedule['m']) { // and only at specific minutes + const hours = `${numberList(schedule['h'])} hour`; + if (!hasSpecificSeconds && isOnTheHour(schedule['m'])) { + textParts = ['On the']; + end += hours; + } else { + beginning = minutes.beginning; + end += `${minutes.text} past the ${hours}`; + } + } else { // specific hours, but every minute + end += `minute of ${numberList(schedule['h'])} hour`; + } + } else if (schedule['m']) { // every hour, but specific minutes + beginning = minutes.beginning; + end += minutes.text; + if (!isOnTheHour(schedule['m']) && (onlySpecificDaysOfMonth || schedule['d'] || schedule['M'])) { + end += ' past every hour'; + } + } else if (!schedule['s'] && !schedule['m']) { + beginning = seconds.beginning; + } else if (!useSeconds || !hasSpecificSeconds) { // cronspec has "*" for both hour and minute + beginning += minutes.beginning; + } + textParts.push(beginning); + textParts.push(end); + } + + if (onlySpecificDaysOfMonth) { // runs only on specific day(s) of month + textParts.push(`on the ${numberList(schedule['D'])}`); + if (!schedule['M']) { + textParts.push('of every month'); + } + } + + if (schedule['d']) { // runs only on specific day(s) of week + if (schedule['D']) { + // if both day fields are specified, cron uses both; superuser.com/a/348372 + textParts.push('and every'); + } else { + textParts.push('on'); + } + textParts.push(dateList(schedule['d'], 'dow')); + } + + if (schedule['M']) { + if (schedule['M'].length === 12) { + textParts.push('day of every month'); + } else { + // runs only in specific months; put this output last + textParts.push(`in ${dateList(schedule['M'], 'mon')}`); + } + } + + return textParts.filter(function(p) { return p; }).join(' '); + }; + + // ---------------- + + // Given a cronspec, return the human-readable string. + const toString = function(cronspec: any, sixth: boolean) { + const schedule = later.parse.cron(cronspec, sixth); + return scheduleToSentence(schedule['schedules'][0], sixth); + }; + + // Given a cronspec, return the next date for when it will next run. + // (This is just a wrapper for later.js) + const getNextDate = function(cronspec: any, sixth: boolean) { + later.date.localTime(); + const schedule = later.parse.cron(cronspec, sixth); + return later.schedule(schedule).next(); + }; + + // Given a cronspec, return a friendly string for when it will next run. + // (This is just a wrapper for later.js and moment.js) + const getNext = function(cronspec: any, sixth: boolean) { + return dayjs(getNextDate(cronspec, sixth)).calendar(); + }; + + // Given a cronspec and numDates, return a list of formatted dates + // of the next set of runs. + // (This is just a wrapper for later.js and moment.js) + const getNextDates = function(cronspec: any, numDates: any, sixth: boolean) { + const schedule = later.parse.cron(cronspec, sixth); + const nextDates = later.schedule(schedule).next(numDates); + + const nextPrettyDates = []; + for (let i = 0; i < nextDates.length; i++) { + nextPrettyDates.push(dayjs(nextDates[i]).calendar()); + } + + return nextPrettyDates; + }; + + // ---------------- + + // attach ourselves to window in the browser, and to exports in Node, + // so our functions can always be called as prettyCron.toString() + const global_obj = (typeof exports !== 'undefined' && exports !== null) ? exports : (window as any).prettyCron = {}; + + global_obj.toString = toString; + global_obj.getNext = getNext; + global_obj.getNextDate = getNextDate; + global_obj.getNextDates = getNextDates; + +}).call(this); diff --git a/packages/server/src/utils/routeUtils.ts b/packages/server/src/utils/routeUtils.ts index 18919685f..cf8f90083 100644 --- a/packages/server/src/utils/routeUtils.ts +++ b/packages/server/src/utils/routeUtils.ts @@ -289,6 +289,7 @@ export enum UrlType { Terms = 'terms', Privacy = 'privacy', Tasks = 'tasks', + UserDeletions = 'user_deletions', } export function makeUrl(urlType: UrlType): string { diff --git a/packages/server/src/utils/setupAppContext.ts b/packages/server/src/utils/setupAppContext.ts index d3dce8e65..0e33cc34f 100644 --- a/packages/server/src/utils/setupAppContext.ts +++ b/packages/server/src/utils/setupAppContext.ts @@ -9,15 +9,19 @@ import { Services } from '../services/types'; import EmailService from '../services/EmailService'; import MustacheService from '../services/MustacheService'; import setupTaskService from './setupTaskService'; +import UserDeletionService from '../services/UserDeletionService'; async function setupServices(env: Env, models: Models, config: Config): Promise { const output: Services = { share: new ShareService(env, models, config), email: new EmailService(env, models, config), mustache: new MustacheService(config.viewDir, config.baseUrl), - tasks: setupTaskService(env, models, config), + userDeletion: new UserDeletionService(env, models, config), + tasks: null, }; + output.tasks = setupTaskService(env, models, config, output), + await output.mustache.loadPartials(); return output; diff --git a/packages/server/src/utils/setupTaskService.ts b/packages/server/src/utils/setupTaskService.ts index 4e677cbc0..d322851ff 100644 --- a/packages/server/src/utils/setupTaskService.ts +++ b/packages/server/src/utils/setupTaskService.ts @@ -1,9 +1,10 @@ import { Models } from '../models/factory'; import TaskService, { Task, TaskId } from '../services/TaskService'; +import { Services } from '../services/types'; import { Config, Env } from './types'; -export default function(env: Env, models: Models, config: Config): TaskService { - const taskService = new TaskService(env, models, config); +export default function(env: Env, models: Models, config: Config, services: Services): TaskService { + const taskService = new TaskService(env, models, config, services); let tasks: Task[] = [ { @@ -27,6 +28,13 @@ export default function(env: Env, models: Models, config: Config): TaskService { run: (models: Models) => models.change().compressOldChanges(), }, + { + id: TaskId.ProcessUserDeletions, + description: 'Process user deletions', + schedule: '0 */6 * * *', + run: (_models: Models, services: Services) => services.userDeletion.runMaintenance(), + }, + // Need to do it relatively frequently so that if the user fixes // whatever was causing the oversized account, they can get it // re-enabled quickly. Also it's done on minute 30 because it depends on diff --git a/packages/server/src/utils/testing/fileApiUtils.ts b/packages/server/src/utils/testing/fileApiUtils.ts index 8c7516de0..c7191b7ba 100644 --- a/packages/server/src/utils/testing/fileApiUtils.ts +++ b/packages/server/src/utils/testing/fileApiUtils.ts @@ -103,25 +103,25 @@ export async function postDirectory(sessionId: string, parentPath: string, name: return context.response.body; } -export async function getDirectoryChildrenContext(sessionId: string, path: string, pagination: Pagination = null): Promise { - const context = await koaAppContext({ - sessionId: sessionId, - request: { - method: 'GET', - url: `/api/files/${path}/children`, - query: paginationToQueryParams(pagination), - }, - }); +// export async function getDirectoryChildrenContext(sessionId: string, path: string, pagination: Pagination = null): Promise { +// const context = await koaAppContext({ +// sessionId: sessionId, +// request: { +// method: 'GET', +// url: `/api/files/${path}/children`, +// query: paginationToQueryParams(pagination), +// }, +// }); - await routeHandler(context); - return context; -} +// await routeHandler(context); +// return context; +// } -export async function getDirectoryChildren(sessionId: string, path: string, pagination: Pagination = null): Promise { - const context = await getDirectoryChildrenContext(sessionId, path, pagination); - checkContextError(context); - return context.response.body as PaginatedResults; -} +// export async function getDirectoryChildren(sessionId: string, path: string, pagination: Pagination = null): Promise> { +// const context = await getDirectoryChildrenContext(sessionId, path, pagination); +// checkContextError(context); +// return context.response.body as PaginatedResults; +// } export async function putFileContentContext(sessionId: string, path: string, filePath: string): Promise { const context = await koaAppContext({ @@ -194,8 +194,8 @@ export async function getDeltaContext(sessionId: string, path: string, paginatio return context; } -export async function getDelta(sessionId: string, path: string, pagination: Pagination): Promise { +export async function getDelta(sessionId: string, path: string, pagination: Pagination): Promise> { const context = await getDeltaContext(sessionId, path, pagination); checkContextError(context); - return context.response.body as PaginatedResults; + return context.response.body as PaginatedResults; } diff --git a/packages/server/src/utils/time.ts b/packages/server/src/utils/time.ts index 5ce3fe5b2..d5a80eb13 100644 --- a/packages/server/src/utils/time.ts +++ b/packages/server/src/utils/time.ts @@ -30,6 +30,7 @@ export function msleep(ms: number) { } export function formatDateTime(ms: number | Date): string { + if (!ms) return '-'; ms = ms instanceof Date ? ms.getTime() : ms; return `${dayjs(ms).format('D MMM YY HH:mm:ss')} (${defaultTimezone()})`; } diff --git a/packages/server/src/utils/urlUtils.ts b/packages/server/src/utils/urlUtils.ts index ab7f5a33d..68b6d6565 100644 --- a/packages/server/src/utils/urlUtils.ts +++ b/packages/server/src/utils/urlUtils.ts @@ -45,3 +45,11 @@ export function homeUrl(): string { export function loginUrl(): string { return `${config().baseUrl}/login`; } + +export function userDeletionsUrl(): string { + return `${config().baseUrl}/user_deletions`; +} + +export function userUrl(userId: Uuid): string { + return `${config().baseUrl}/users/${userId}`; +} diff --git a/packages/server/src/utils/views/table.ts b/packages/server/src/utils/views/table.ts index 859e446c7..5417f7c8f 100644 --- a/packages/server/src/utils/views/table.ts +++ b/packages/server/src/utils/views/table.ts @@ -26,6 +26,7 @@ export interface Header { name: string; label: string; stretch?: boolean; + canSort?: boolean; } interface HeaderView { @@ -40,6 +41,7 @@ interface RowItem { checkbox?: boolean; url?: string; stretch?: boolean; + hint?: string; } export type Row = RowItem[]; @@ -49,6 +51,7 @@ interface RowItemView { classNames: string[]; url: string; checkbox: boolean; + hint: string; } type RowView = RowItemView[]; @@ -79,9 +82,11 @@ export function makeTablePagination(query: any, defaultOrderField: string, defau } function makeHeaderView(header: Header, parentBaseUrl: string, baseUrlQuery: PaginationQueryParams, pagination: Pagination): HeaderView { + const canSort = header.canSort !== false; + return { label: header.label, - sortLink: !pagination ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }), + sortLink: !pagination || !canSort ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }), classNames: [header.stretch ? 'stretch' : 'nowrap', headerIsSelectedClass(header.name, pagination)], iconDir: headerSortIconDir(header.name, pagination), }; @@ -94,6 +99,7 @@ function makeRowView(row: Row): RowView { classNames: [rowItem.stretch ? 'stretch' : 'nowrap'], url: rowItem.url, checkbox: rowItem.checkbox, + hint: rowItem.hint, }; }); } @@ -104,6 +110,8 @@ export function makeTableView(table: Table): TableView { let pagination: Pagination = null; if (table.pageCount) { + if (!table.baseUrl || !table.requestQuery) throw new Error('Table.baseUrl and Table.requestQuery are required for pagination when there is more than one page'); + baseUrlQuery = filterPaginationQueryParams(table.requestQuery); pagination = table.pagination; paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' })); diff --git a/packages/server/src/views/index/user.mustache b/packages/server/src/views/index/user.mustache index d54cf9724..d5353491c 100644 --- a/packages/server/src/views/index/user.mustache +++ b/packages/server/src/views/index/user.mustache @@ -79,6 +79,7 @@

+
@@ -103,6 +104,9 @@ {{#showRestoreButton}} {{/showRestoreButton}} + {{#showScheduleDeletionButton}} + + {{/showScheduleDeletionButton}}
diff --git a/packages/server/src/views/index/user_deletions.mustache b/packages/server/src/views/index/user_deletions.mustache new file mode 100644 index 000000000..ba9e33d79 --- /dev/null +++ b/packages/server/src/views/index/user_deletions.mustache @@ -0,0 +1,11 @@ +
+ {{{csrfTag}}} + + {{#userDeletionTable}} + {{>table}} + {{/userDeletionTable}} + +
+ +
+
diff --git a/packages/server/src/views/index/users.mustache b/packages/server/src/views/index/users.mustache index c270ba766..fde83a737 100644 --- a/packages/server/src/views/index/users.mustache +++ b/packages/server/src/views/index/users.mustache @@ -1,3 +1,7 @@ +
+ > User deletions +
+
Add user Hide disabled diff --git a/packages/server/src/views/partials/tableRowItem.mustache b/packages/server/src/views/partials/tableRowItem.mustache index 1a2240973..e44409eca 100644 --- a/packages/server/src/views/partials/tableRowItem.mustache +++ b/packages/server/src/views/partials/tableRowItem.mustache @@ -3,6 +3,6 @@ {{/checkbox}} {{^checkbox}} - {{#url}}{{/url}}{{value}} + {{#url}}{{/url}}{{#hint}}{{/hint}}{{value}}{{#hint}}{{/hint}} {{/checkbox}} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 513dc8db3..0ea5c056d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3391,6 +3391,7 @@ __metadata: nodemon: ^2.0.6 pg: ^8.5.1 pretty-bytes: ^5.6.0 + prettycron: ^0.10.0 query-string: ^6.8.3 rate-limiter-flexible: ^2.2.4 raw-body: ^2.4.1 @@ -19299,6 +19300,13 @@ __metadata: languageName: node linkType: hard +"later@npm:^1.1.6": + version: 1.2.0 + resolution: "later@npm:1.2.0" + checksum: 49dda2d5b3acda7a190a756c1cecc959853e8cbd1cc4c4a3b8dc0a3ec3859ec031adafe5e9659e2ad3ee13fb06288cc72d894fb8ce7bca17c0155666fdf6211b + languageName: node + linkType: hard + "latest-version@npm:^5.1.0": version: 5.1.0 resolution: "latest-version@npm:5.1.0" @@ -21469,7 +21477,7 @@ __metadata: languageName: node linkType: hard -"moment@npm:>= 2.9.0, moment@npm:^2.15.1, moment@npm:^2.22.2, moment@npm:^2.24.0, moment@npm:^2.29.1": +"moment@npm:>= 2.9.0, moment@npm:^2.15.1, moment@npm:^2.22.2, moment@npm:^2.24.0, moment@npm:^2.29.1, moment@npm:^2.8.4": version: 2.29.1 resolution: "moment@npm:2.29.1" checksum: 1e14d5f422a2687996be11dd2d50c8de3bd577c4a4ca79ba5d02c397242a933e5b941655de6c8cb90ac18f01cc4127e55b4a12ae3c527a6c0a274e455979345e @@ -24062,6 +24070,16 @@ __metadata: languageName: node linkType: hard +"prettycron@npm:^0.10.0": + version: 0.10.0 + resolution: "prettycron@npm:0.10.0" + dependencies: + later: ^1.1.6 + moment: ^2.8.4 + checksum: a8e9b6e1e59f0b5b5293b21b1ebaea8f043328db9046695d0bca498d1337314ed151ad4b6df0a957513c2b59a5c777e8c2e190fb805a452c18fd2cecb7492c85 + languageName: node + linkType: hard + "private@npm:^0.1.8": version: 0.1.8 resolution: "private@npm:0.1.8"