self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

Server: Allow deleting complete user data (#5824)

Browse Source
This commit is contained in:
Laurent 2021-12-28 10:55:01 +01:00 committed by GitHub
parent b41a3d7f8d
commit d1e02fd5f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
55 changed files with 1561 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/app-desktop/runForTesting.sh
View File

@ -54,6 +54,10 @@ do
curl --data '{"action": "createTestUsers"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
elif [[ $CMD == "createUserDeletions" ]]; then
curl --data '{"action": "createUserDeletions"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
elif [[ $CMD == "createData" ]]; then
echo 'mkbook "shared"' >> "$CMD_FILE"

1
packages/server/package.json
View File

@ -47,6 +47,7 @@
"nodemon": "^2.0.6",
"pg": "^8.5.1",
"pretty-bytes": "^5.6.0",
"prettycron": "^0.10.0",
"query-string": "^6.8.3",
"rate-limiter-flexible": "^2.2.4",
"raw-body": "^2.4.1",

7
packages/server/public/css/main.css
View File

@ -87,4 +87,9 @@ h4:hover a.heading-anchor,
h5:hover a.heading-anchor,
h6:hover a.heading-anchor {
opacity: 1;
}
}
abbr[title] {
text-underline-offset: 2px;
text-decoration: underline dotted;
}

BIN
packages/server/schema.sqlite
View File

Binary file not shown.

8
packages/server/src/db.ts
View File

@ -49,6 +49,7 @@ export interface DbConfigConnection {
export interface QueryContext {
uniqueConstraintErrorLoggingDisabled?: boolean;
noSuchTableErrorLoggingDisabled?: boolean;
}
export interface KnexDatabaseConfig {
@ -227,6 +228,10 @@ export async function connectDb(dbConfig: DatabaseConfig): Promise<DbConnection>
if (data.queryContext.uniqueConstraintErrorLoggingDisabled && isUniqueConstraintError(response)) {
return;
}
if (data.queryContext.noSuchTableErrorLoggingDisabled && isNoSuchTableError(response)) {
return;
}
}
const msg: string[] = [];
@ -392,7 +397,8 @@ export function isUniqueConstraintError(error: any): boolean {
export async function latestMigration(db: DbConnection): Promise<Migration | null> {
try {
const result = await db('knex_migrations').select('name').orderBy('id', 'desc').first();
const context: QueryContext = { noSuchTableErrorLoggingDisabled: true };
const result = await db('knex_migrations').queryContext(context).select('name').orderBy('id', 'desc').first();
return { name: result.name, done: true };
} catch (error) {
// If the database has never been initialized, we return null, so

26
packages/server/src/migrations/20211204191051_user_deletion.ts Normal file
View File

@ -0,0 +1,26 @@
import { Knex } from 'knex';
import { DbConnection } from '../db';
export async function up(db: DbConnection): Promise<any> {
await db.schema.createTable('user_deletions', (table: Knex.CreateTableBuilder) => {
table.increments('id').unique().primary().notNullable();
table.string('user_id', 32).notNullable();
table.specificType('process_data', 'smallint').defaultTo(0).notNullable();
table.specificType('process_account', 'smallint').defaultTo(0).notNullable();
table.bigInteger('scheduled_time').notNullable();
table.bigInteger('start_time').defaultTo(0).notNullable();
table.bigInteger('end_time').defaultTo(0).notNullable();
table.integer('success').defaultTo(0).notNullable();
table.text('error', 'mediumtext').defaultTo('').notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('user_deletions', (table: Knex.CreateTableBuilder) => {
table.unique(['user_id']);
});
}
export async function down(db: DbConnection): Promise<any> {
await db.schema.dropTable('user_deletions');
}

25
packages/server/src/models/BaseModel.ts
View File

@ -9,6 +9,7 @@ import { Config } from '../utils/types';
import personalizedUserContentBaseUrl from '@joplin/lib/services/joplinServer/personalizedUserContentBaseUrl';
import Logger from '@joplin/lib/Logger';
import dbuuid from '../utils/dbuuid';
import { defaultPagination, PaginatedResults, Pagination } from './utils/pagination';
const logger = Logger.create('BaseModel');
@ -232,6 +233,28 @@ export default abstract class BaseModel<T> {
return rows as T[];
}
public async allPaginated(pagination: Pagination, options: LoadOptions = {}): Promise<PaginatedResults<T>> {
pagination = {
...defaultPagination(),
...pagination,
};
const itemCount = await this.count();
const items = await this
.db(this.tableName)
.select(this.selectFields(options))
.orderBy(pagination.order[0].by, pagination.order[0].dir)
.offset((pagination.page - 1) * pagination.limit)
.limit(pagination.limit) as T[];
return {
items,
page_count: Math.ceil(itemCount / pagination.limit),
has_more: items.length >= pagination.limit,
};
}
public async count(): Promise<number> {
const r = await this
.db(this.tableName)
@ -343,7 +366,7 @@ export default abstract class BaseModel<T> {
return !!o;
}
public async load(id: string, options: LoadOptions = {}): Promise<T> {
public async load(id: Uuid | number, options: LoadOptions = {}): Promise<T> {
if (!id) throw new Error('id cannot be empty');
return this.db(this.tableName).select(options.fields || this.defaultFields).where({ id: id }).first();

25
packages/server/src/models/ChangeModel.ts
View File

@ -16,13 +16,9 @@ export interface DeltaChange extends Change {
jop_updated_time?: number;
}
export interface PaginatedDeltaChanges extends PaginatedResults {
items: DeltaChange[];
}
export type PaginatedDeltaChanges = PaginatedResults<DeltaChange>;
export interface PaginatedChanges extends PaginatedResults {
items: Change[];
}
export type PaginatedChanges = PaginatedResults<Change>;
export interface ChangePagination {
limit?: number;
@ -43,6 +39,15 @@ export function defaultDeltaPagination(): ChangePagination {
};
}
export function requestDeltaPagination(query: any): ChangePagination {
if (!query) return defaultDeltaPagination();
const output: ChangePagination = {};
if ('limit' in query) output.limit = query.limit;
if ('cursor' in query) output.cursor = query.cursor;
return output;
}
export default class ChangeModel extends BaseModel<Change> {
public get tableName(): string {
@ -391,4 +396,12 @@ export default class ChangeModel extends BaseModel<Change> {
return savedChange;
}
public async deleteByItemIds(itemIds: Uuid[]) {
if (!itemIds.length) return;
await this.db(this.tableName)
.whereIn('item_id', itemIds)
.delete();
}
}

92
packages/server/src/models/ItemModel.test.ts
View File

@ -1,4 +1,4 @@
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder, createItemTree3, db, tempDir, expectNotThrow, expectHttpError } from '../utils/testing/testUtils';
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItemTree, createResource, createNote, createItemTree3, db, tempDir, expectNotThrow, expectHttpError } from '../utils/testing/testUtils';
import { shareFolderWithUser } from '../utils/testing/shareApiUtils';
import { resourceBlobPath } from '../utils/joplinUtils';
import newModelFactory from './factory';
@ -23,64 +23,64 @@ describe('ItemModel', function() {
await beforeEachDb();
});
test('should find exclusively owned items 1', async function() {
const { user: user1 } = await createUserAndSession(1, true);
const { session: session2, user: user2 } = await createUserAndSession(2);
// test('should find exclusively owned items 1', async function() {
// const { user: user1 } = await createUserAndSession(1, true);
// const { session: session2, user: user2 } = await createUserAndSession(2);
const tree: any = {
'000000000000000000000000000000F1': {
'00000000000000000000000000000001': null,
},
};
// const tree: any = {
// '000000000000000000000000000000F1': {
// '00000000000000000000000000000001': null,
// },
// };
await createItemTree(user1.id, '', tree);
await createItem(session2.id, 'root:/test.txt:', 'testing');
// await createItemTree(user1.id, '', tree);
// await createItem(session2.id, 'root:/test.txt:', 'testing');
{
const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
expect(itemIds.length).toBe(2);
// {
// const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
// expect(itemIds.length).toBe(2);
const item1 = await models().item().load(itemIds[0]);
const item2 = await models().item().load(itemIds[1]);
// const item1 = await models().item().load(itemIds[0]);
// const item2 = await models().item().load(itemIds[1]);
expect([item1.jop_id, item2.jop_id].sort()).toEqual(['000000000000000000000000000000F1', '00000000000000000000000000000001'].sort());
}
// expect([item1.jop_id, item2.jop_id].sort()).toEqual(['000000000000000000000000000000F1', '00000000000000000000000000000001'].sort());
// }
{
const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id);
expect(itemIds.length).toBe(1);
}
});
// {
// const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id);
// expect(itemIds.length).toBe(1);
// }
// });
test('should find exclusively owned items 2', async function() {
const { session: session1, user: user1 } = await createUserAndSession(1, true);
const { session: session2, user: user2 } = await createUserAndSession(2);
// test('should find exclusively owned items 2', async function() {
// const { session: session1, user: user1 } = await createUserAndSession(1, true);
// const { session: session2, user: user2 } = await createUserAndSession(2);
await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', {
'000000000000000000000000000000F1': {
'00000000000000000000000000000001': null,
},
});
// await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', {
// '000000000000000000000000000000F1': {
// '00000000000000000000000000000001': null,
// },
// });
await createFolder(session2.id, { id: '000000000000000000000000000000F2' });
// await createFolder(session2.id, { id: '000000000000000000000000000000F2' });
{
const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
expect(itemIds.length).toBe(0);
}
// {
// const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
// expect(itemIds.length).toBe(0);
// }
{
const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id);
expect(itemIds.length).toBe(1);
}
// {
// const itemIds = await models().item().exclusivelyOwnedItemIds(user2.id);
// expect(itemIds.length).toBe(1);
// }
await models().user().delete(user2.id);
// await models().user().delete(user2.id);
{
const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
expect(itemIds.length).toBe(2);
}
});
// {
// const itemIds = await models().item().exclusivelyOwnedItemIds(user1.id);
// expect(itemIds.length).toBe(2);
// }
// });
test('should find all items within a shared folder', async function() {
const { user: user1, session: session1 } = await createUserAndSession(1);

47
packages/server/src/models/ItemModel.ts
View File

@ -1,4 +1,4 @@
import BaseModel, { SaveOptions, LoadOptions, DeleteOptions, ValidateOptions, AclAction } from './BaseModel';
import BaseModel, { SaveOptions, LoadOptions, DeleteOptions as BaseDeleteOptions, ValidateOptions, AclAction } from './BaseModel';
import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User, UserItem } from '../services/database/types';
import { defaultPagination, paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination';
import { isJoplinItemName, isJoplinResourceBlobPath, linkedResourceIds, serializeJoplinItem, unserializeJoplinItem } from '../utils/joplinUtils';
@ -21,6 +21,10 @@ const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
// Converts "root:/myfile.txt:" to "myfile.txt"
const extractNameRegex = /^root:\/(.*):$/;
export interface DeleteOptions extends BaseDeleteOptions {
deleteChanges?: boolean;
}
export interface ImportContentToStorageOptions {
batchSize?: number;
maxContentSize?: number;
@ -45,9 +49,7 @@ export interface SaveFromRawContentResultItem {
export type SaveFromRawContentResult = Record<string, SaveFromRawContentResultItem>;
export interface PaginatedItems extends PaginatedResults {
items: Item[];
}
export type PaginatedItems = PaginatedResults<Item>;
export interface SharedRootInfo {
item: Item;
@ -813,22 +815,22 @@ export default class ItemModel extends BaseModel<Item> {
// Returns the item IDs that are owned only by the given user. In other
// words, the items that are not shared with anyone else. Such items
// can be safely deleted when the user is deleted.
public async exclusivelyOwnedItemIds(userId: Uuid): Promise<Uuid[]> {
const query = this
.db('items')
.select(this.db.raw('items.id, count(user_items.item_id) as user_item_count'))
.leftJoin('user_items', 'user_items.item_id', 'items.id')
.whereIn('items.id', this.db('user_items').select('user_items.item_id').where('user_id', '=', userId))
.groupBy('items.id');
// public async exclusivelyOwnedItemIds(userId: Uuid): Promise<Uuid[]> {
// const query = this
// .db('items')
// .select(this.db.raw('items.id, count(user_items.item_id) as user_item_count'))
// .leftJoin('user_items', 'user_items.item_id', 'items.id')
// .whereIn('items.id', this.db('user_items').select('user_items.item_id').where('user_id', '=', userId))
// .groupBy('items.id');
const rows: any[] = await query;
return rows.filter(r => r.user_item_count === 1).map(r => r.id);
}
// const rows: any[] = await query;
// return rows.filter(r => r.user_item_count === 1).map(r => r.id);
// }
public async deleteExclusivelyOwnedItems(userId: Uuid) {
const itemIds = await this.exclusivelyOwnedItemIds(userId);
await this.delete(itemIds);
}
// public async deleteExclusivelyOwnedItems(userId: Uuid) {
// const itemIds = await this.exclusivelyOwnedItemIds(userId);
// await this.delete(itemIds);
// }
public async deleteAll(userId: Uuid): Promise<void> {
while (true) {
@ -839,6 +841,11 @@ export default class ItemModel extends BaseModel<Item> {
}
public async delete(id: string | string[], options: DeleteOptions = {}): Promise<void> {
options = {
deleteChanges: false,
...options,
};
const ids = typeof id === 'string' ? [id] : id;
if (!ids.length) return;
@ -849,12 +856,14 @@ export default class ItemModel extends BaseModel<Item> {
await this.withTransaction(async () => {
await this.models().share().delete(shares.map(s => s.id));
await this.models().userItem().deleteByItemIds(ids);
await this.models().userItem().deleteByItemIds(ids, { recordChanges: !options.deleteChanges });
await this.models().itemResource().deleteByItemIds(ids);
await storageDriver.delete(ids, { models: this.models() });
if (storageDriverFallback) await storageDriverFallback.delete(ids, { models: this.models() });
await super.delete(ids, options);
if (options.deleteChanges) await this.models().change().deleteByItemIds(ids);
}, 'ItemModel::delete');
}

17
packages/server/src/models/NotificationModel.ts
View File

@ -57,8 +57,13 @@ export default class NotificationModel extends BaseModel<Notification> {
},
};
const n: Notification = await this.loadUnreadByKey(userId, key);
if (n) return n;
const n: Notification = await this.loadByKey(userId, key);
if (n) {
if (!n.read) return n;
await this.save({ id: n.id, read: 0 });
return { ...n, read: 0 };
}
const type = notificationTypes[key];
@ -83,6 +88,14 @@ export default class NotificationModel extends BaseModel<Notification> {
return this.save({ key: actualKey, message, level, owner_id: userId });
}
public async addInfo(userId: Uuid, message: string) {
return this.add(userId, NotificationKey.Any, NotificationLevel.Normal, message);
}
public async addError(userId: Uuid, message: string) {
return this.add(userId, NotificationKey.Any, NotificationLevel.Error, message);
}
public async setRead(userId: Uuid, key: NotificationKey, read: boolean = true): Promise<void> {
const n = await this.loadByKey(userId, key);
if (!n) return;

10
packages/server/src/models/ShareModel.ts
View File

@ -411,6 +411,16 @@ export default class ShareModel extends BaseModel<Share> {
}, 'ShareModel::delete');
}
public async deleteByUserId(userId: Uuid) {
const shares = await this.sharesByUser(userId);
await this.withTransaction(async () => {
for (const share of shares) {
await this.delete(share.id);
}
}, 'ShareModel::deleteByUserId');
}
public async itemCountByShareId(shareId: Uuid): Promise<number> {
const r = await this
.db('items')

10
packages/server/src/models/ShareUserModel.ts
View File

@ -145,6 +145,16 @@ export default class ShareUserModel extends BaseModel<ShareUser> {
}, 'ShareUserModel::deleteByShare');
}
public async deleteByUserId(userId: Uuid) {
const shareUsers = await this.byUserId(userId);
await this.withTransaction(async () => {
for (const shareUser of shareUsers) {
await this.delete(shareUser.id);
}
}, 'UserShareModel::deleteByUserId');
}
public async delete(id: string | string[], _options: DeleteOptions = {}): Promise<void> {
const ids = typeof id === 'string' ? [id] : id;
if (!ids.length) throw new Error('no id provided');

146
packages/server/src/models/UserDeletionModel.test.ts Normal file
View File

@ -0,0 +1,146 @@
import { beforeAllDb, afterAllTests, beforeEachDb, models, createUser, expectThrow } from '../utils/testing/testUtils';
describe('UserDeletionModel', function() {
beforeAll(async () => {
await beforeAllDb('UserDeletionModel');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should add a deletion operation', async function() {
{
const user = await createUser(1);
const scheduleTime = Date.now() + 1000;
await models().userDeletion().add(user.id, scheduleTime);
const deletion = await models().userDeletion().byUserId(user.id);
expect(deletion.user_id).toBe(user.id);
expect(deletion.process_account).toBe(1);
expect(deletion.process_data).toBe(1);
expect(deletion.scheduled_time).toBe(scheduleTime);
expect(deletion.error).toBe('');
expect(deletion.success).toBe(0);
expect(deletion.start_time).toBe(0);
expect(deletion.end_time).toBe(0);
await models().userDeletion().delete(deletion.id);
}
{
const user = await createUser(2);
await models().userDeletion().add(user.id, Date.now() + 1000, {
processData: true,
processAccount: false,
});
const deletion = await models().userDeletion().byUserId(user.id);
expect(deletion.process_data).toBe(1);
expect(deletion.process_account).toBe(0);
}
{
const user = await createUser(3);
await models().userDeletion().add(user.id, Date.now() + 1000);
await expectThrow(async () => models().userDeletion().add(user.id, Date.now() + 1000));
}
});
test('should provide the next deletion operation', async function() {
expect(await models().userDeletion().next()).toBeFalsy();
jest.useFakeTimers('modern');
const t0 = new Date('2021-12-14').getTime();
jest.setSystemTime(t0);
const user1 = await createUser(1);
const user2 = await createUser(2);
await models().userDeletion().add(user1.id, t0 + 100000);
await models().userDeletion().add(user2.id, t0 + 100);
expect(await models().userDeletion().next()).toBeFalsy();
jest.setSystemTime(t0 + 200);
expect((await models().userDeletion().next()).user_id).toBe(user2.id);
jest.setSystemTime(t0 + 200000);
const next1 = await models().userDeletion().next();
expect(next1.user_id).toBe(user2.id);
await models().userDeletion().start(next1.id);
await models().userDeletion().end(next1.id, true, null);
const next2 = await models().userDeletion().next();
expect(next2.user_id).toBe(user1.id);
await models().userDeletion().start(next2.id);
await models().userDeletion().end(next2.id, true, null);
const next3 = await models().userDeletion().next();
expect(next3).toBeFalsy();
jest.useRealTimers();
});
test('should start and stop deletion jobs', async function() {
jest.useFakeTimers('modern');
const t0 = new Date('2021-12-14').getTime();
jest.setSystemTime(t0);
const user1 = await createUser(1);
const user2 = await createUser(2);
await models().userDeletion().add(user1.id, t0 + 10);
await models().userDeletion().add(user2.id, t0 + 100);
jest.setSystemTime(t0 + 200);
const next1 = await models().userDeletion().next();
await models().userDeletion().start(next1.id);
{
const d = await models().userDeletion().load(next1.id);
expect(d.start_time).toBe(t0 + 200);
expect(d.updated_time).toBe(t0 + 200);
expect(d.end_time).toBe(0);
}
jest.setSystemTime(t0 + 300);
await models().userDeletion().end(next1.id, false, 'error!');
{
const d = await models().userDeletion().load(next1.id);
expect(d.start_time).toBe(t0 + 200);
expect(d.updated_time).toBe(t0 + 300);
expect(d.end_time).toBe(t0 + 300);
expect(d.success).toBe(0);
expect(JSON.parse(d.error)).toEqual({ message: 'error!' });
}
const next2 = await models().userDeletion().next();
await models().userDeletion().start(next2.id);
await models().userDeletion().end(next2.id, true, null);
{
const d = await models().userDeletion().load(next2.id);
expect(d.start_time).toBe(t0 + 300);
expect(d.updated_time).toBe(t0 + 300);
expect(d.end_time).toBe(t0 + 300);
expect(d.success).toBe(1);
expect(d.error).toBe('');
}
jest.useRealTimers();
});
});

94
packages/server/src/models/UserDeletionModel.ts Normal file
View File

@ -0,0 +1,94 @@
import { UserDeletion, Uuid } from '../services/database/types';
import { errorToString } from '../utils/errors';
import BaseModel from './BaseModel';
export interface AddOptions {
processData?: boolean;
processAccount?: boolean;
}
export default class UserDeletionModel extends BaseModel<UserDeletion> {
protected get tableName(): string {
return 'user_deletions';
}
protected hasUuid(): boolean {
return false;
}
public async byUserId(userId: Uuid): Promise<UserDeletion> {
return this.db(this.tableName).where('user_id', '=', userId).first();
}
public async isScheduledForDeletion(userId: Uuid) {
const r = await this.db(this.tableName).select(['id']).where('user_id', '=', userId).first();
return !!r;
}
public async add(userId: Uuid, scheduledTime: number, options: AddOptions = null): Promise<UserDeletion> {
options = {
processAccount: true,
processData: true,
...options,
};
const now = Date.now();
const o: UserDeletion = {
user_id: userId,
scheduled_time: scheduledTime,
created_time: now,
updated_time: now,
process_account: options.processAccount ? 1 : 0,
process_data: options.processData ? 1 : 0,
};
await this.db(this.tableName).insert(o);
return this.byUserId(userId);
}
public async remove(jobId: number) {
await this.db(this.tableName).where('id', '=', jobId).delete();
}
public async next(): Promise<UserDeletion> {
return this
.db(this.tableName)
.where('scheduled_time', '<=', Date.now())
.andWhere('start_time', '=', 0)
.orderBy('scheduled_time', 'asc')
.first();
}
public async start(deletionId: number) {
const now = Date.now();
await this
.db(this.tableName)
.update({ start_time: now, updated_time: now })
.where('id', deletionId)
.andWhere('start_time', '=', 0);
const item = await this.load(deletionId);
if (item.start_time !== now) throw new Error('Job was already started');
}
public async end(deletionId: number, success: boolean, error: any) {
const now = Date.now();
const o: UserDeletion = {
end_time: now,
updated_time: now,
success: success ? 1 : 0,
error: error ? errorToString(error) : '',
};
await this
.db(this.tableName)
.update(o)
.where('id', deletionId);
}
}

9
packages/server/src/models/UserFlagModel.ts
View File

@ -108,6 +108,7 @@ export default class UserFlagModels extends BaseModel<UserFlag> {
const failedPaymentFinalFlag = flags.find(f => f.type === UserFlagType.FailedPaymentFinal);
const subscriptionCancelledFlag = flags.find(f => f.type === UserFlagType.SubscriptionCancelled);
const manuallyDisabledFlag = flags.find(f => f.type === UserFlagType.ManuallyDisabled);
const userDeletionInProgress = flags.find(f => f.type === UserFlagType.UserDeletionInProgress);
if (accountWithoutSubscriptionFlag) {
newProps.can_upload = 0;
@ -133,6 +134,10 @@ export default class UserFlagModels extends BaseModel<UserFlag> {
newProps.enabled = 0;
}
if (userDeletionInProgress) {
newProps.enabled = 0;
}
if (user.can_upload !== newProps.can_upload || user.enabled !== newProps.enabled) {
await this.models().user().save({
id: userId,
@ -152,4 +157,8 @@ export default class UserFlagModels extends BaseModel<UserFlag> {
return this.db(this.tableName).where('user_id', '=', userId);
}
public async deleteByUserId(userId: Uuid) {
await this.db(this.tableName).where('user_id', '=', userId).delete();
}
}

12
packages/server/src/models/UserItemModel.ts
View File

@ -16,6 +16,7 @@ export interface UserItemDeleteOptions extends DeleteOptions {
byUserItem?: UserItem;
byUserItemIds?: number[];
byShare?: DeleteByShare;
recordChanges?: boolean;
}
export default class UserItemModel extends BaseModel<UserItem> {
@ -87,8 +88,8 @@ export default class UserItemModel extends BaseModel<UserItem> {
await this.deleteBy({ byUserItem: userItem });
}
public async deleteByItemIds(itemIds: Uuid[]): Promise<void> {
await this.deleteBy({ byItemIds: itemIds });
public async deleteByItemIds(itemIds: Uuid[], options: UserItemDeleteOptions = null): Promise<void> {
await this.deleteBy({ ...options, byItemIds: itemIds });
}
public async deleteByShareId(shareId: Uuid): Promise<void> {
@ -152,6 +153,11 @@ export default class UserItemModel extends BaseModel<UserItem> {
}
private async deleteBy(options: UserItemDeleteOptions = {}): Promise<void> {
options = {
recordChanges: true,
...options,
};
let userItems: UserItem[] = [];
if (options.byShareId && options.byUserId) {
@ -180,7 +186,7 @@ export default class UserItemModel extends BaseModel<UserItem> {
for (const userItem of userItems) {
const item = items.find(i => i.id === userItem.item_id);
if (this.models().item().shouldRecordChange(item.name)) {
if (options.recordChanges && this.models().item().shouldRecordChange(item.name)) {
await this.models().change().save({
item_type: ItemType.UserItem,
item_id: userItem.item_id,

36
packages/server/src/models/UserModel.test.ts
View File

@ -1,5 +1,5 @@
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, checkThrowAsync, createItem, expectThrow } from '../utils/testing/testUtils';
import { EmailSender, User, UserFlagType } from '../services/database/types';
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, checkThrowAsync, expectThrow } from '../utils/testing/testUtils';
import { EmailSender, UserFlagType } from '../services/database/types';
import { ErrorUnprocessableEntity } from '../utils/errors';
import { betaUserDateRange, stripeConfig } from '../utils/stripe';
import { accountByType, AccountType } from './UserModel';
@ -51,26 +51,26 @@ describe('UserModel', function() {
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
});
test('should delete a user', async () => {
const { session: session1, user: user1 } = await createUserAndSession(2, false);
// test('should delete a user', async () => {
// const { session: session1, user: user1 } = await createUserAndSession(2, false);
const userModel = models().user();
// const userModel = models().user();
const allUsers: User[] = await userModel.all();
const beforeCount: number = allUsers.length;
// const allUsers: User[] = await userModel.all();
// const beforeCount: number = allUsers.length;
await createItem(session1.id, 'root:/test.txt:', 'testing');
// await createItem(session1.id, 'root:/test.txt:', 'testing');
// Admin can delete any user
expect(!!(await models().session().load(session1.id))).toBe(true);
expect((await models().item().all()).length).toBe(1);
expect((await models().userItem().all()).length).toBe(1);
await models().user().delete(user1.id);
expect((await userModel.all()).length).toBe(beforeCount - 1);
expect(!!(await models().session().load(session1.id))).toBe(false);
expect((await models().item().all()).length).toBe(0);
expect((await models().userItem().all()).length).toBe(0);
});
// // Admin can delete any user
// expect(!!(await models().session().load(session1.id))).toBe(true);
// expect((await models().item().all()).length).toBe(1);
// expect((await models().userItem().all()).length).toBe(1);
// await models().user().delete(user1.id);
// expect((await userModel.all()).length).toBe(beforeCount - 1);
// expect(!!(await models().session().load(session1.id))).toBe(false);
// expect((await models().item().all()).length).toBe(0);
// expect((await models().userItem().all()).length).toBe(0);
// });
test('should push an email when creating a new user', async () => {
const { user: user1 } = await createUserAndSession(1);

22
packages/server/src/models/UserModel.ts
View File

@ -275,18 +275,18 @@ export default class UserModel extends BaseModel<User> {
return !!s[0].length && !!s[1].length;
}
public async delete(id: string): Promise<void> {
const shares = await this.models().share().sharesByUser(id);
// public async delete(id: string): Promise<void> {
// const shares = await this.models().share().sharesByUser(id);
await this.withTransaction(async () => {
await this.models().item().deleteExclusivelyOwnedItems(id);
await this.models().share().delete(shares.map(s => s.id));
await this.models().userItem().deleteByUserId(id);
await this.models().session().deleteByUserId(id);
await this.models().notification().deleteByUserId(id);
await super.delete(id);
}, 'UserModel::delete');
}
// await this.withTransaction(async () => {
// await this.models().item().deleteExclusivelyOwnedItems(id);
// await this.models().share().delete(shares.map(s => s.id));
// await this.models().userItem().deleteByUserId(id);
// await this.models().session().deleteByUserId(id);
// await this.models().notification().deleteByUserId(id);
// await super.delete(id);
// }, 'UserModel::delete');
// }
private async confirmEmail(user: User) {
await this.save({ id: user.id, email_confirmed: 1 });

5
packages/server/src/models/factory.ts
View File

@ -74,6 +74,7 @@ import EventModel from './EventModel';
import { Config } from '../utils/types';
import LockModel from './LockModel';
import StorageModel from './StorageModel';
import UserDeletionModel from './UserDeletionModel';
export type NewModelFactoryHandler = (db: DbConnection)=> Models;
@ -165,6 +166,10 @@ export class Models {
return new StorageModel(this.db_, this.newModelFactory, this.config_);
}
public userDeletion() {
return new UserDeletionModel(this.db_, this.newModelFactory, this.config_);
}
}
export default function newModelFactory(db: DbConnection, config: Config): Models {

24
packages/server/src/models/utils/pagination.ts
View File

@ -1,6 +1,5 @@
import { ErrorBadRequest } from '../../utils/errors';
import { decodeBase64, encodeBase64 } from '../../utils/base64';
import { ChangePagination as DeltaPagination, defaultDeltaPagination } from '../ChangeModel';
import { Knex } from 'knex';
export enum PaginationOrderDir {
@ -28,8 +27,8 @@ export interface PaginationQueryParams {
cursor?: string;
}
export interface PaginatedResults {
items: any[];
export interface PaginatedResults<T> {
items: T[];
has_more: boolean;
cursor?: string;
page_count?: number;
@ -107,15 +106,6 @@ export function requestPagination(query: any): Pagination {
return validatePagination({ limit, order, page });
}
export function requestDeltaPagination(query: any): DeltaPagination {
if (!query) return defaultDeltaPagination();
const output: DeltaPagination = {};
if ('limit' in query) output.limit = query.limit;
if ('cursor' in query) output.cursor = query.cursor;
return output;
}
export function paginationToQueryParams(pagination: Pagination): PaginationQueryParams {
const output: PaginationQueryParams = {};
if (!pagination) return {};
@ -152,6 +142,8 @@ export interface PageLink {
}
export function filterPaginationQueryParams(query: any): PaginationQueryParams {
if (!query) return {};
const baseUrlQuery: PaginationQueryParams = {};
if (query.limit) baseUrlQuery.limit = query.limit;
if (query.order_by) baseUrlQuery.order_by = query.order_by;
@ -221,7 +213,13 @@ export function createPaginationLinks(page: number, pageCount: number, urlTempla
// return output;
// }
export async function paginateDbQuery(query: Knex.QueryBuilder, pagination: Pagination, mainTable: string = ''): Promise<PaginatedResults> {
export async function paginateDbQuery(query: Knex.QueryBuilder, pagination: Pagination, mainTable: string = ''): Promise<PaginatedResults<any>> {
pagination = {
...defaultPagination(),
...pagination,
};
pagination = processCursor(pagination);
const orderSql: any[] = pagination.order.map(o => {

2
packages/server/src/routes/api/batch_items.ts
View File

@ -8,7 +8,7 @@ import { PaginatedResults } from '../../models/utils/pagination';
const router = new Router(RouteType.Api);
router.put('api/batch_items', async (path: SubPath, ctx: AppContext) => {
const output: PaginatedResults = {
const output: PaginatedResults<any> = {
items: await putItemContents(path, ctx, true) as any,
has_more: false,
};

6
packages/server/src/routes/api/debug.ts
View File

@ -1,5 +1,5 @@
import config from '../../config';
import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
import { clearDatabase, createTestUsers, CreateTestUsersOptions, createUserDeletions } from '../../tools/debugTools';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { Env, RouteType } from '../../utils/types';
@ -34,6 +34,10 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
await createTestUsers(ctx.joplin.db, config(), options);
}
if (query.action === 'createUserDeletions') {
await createUserDeletions(ctx.joplin.db, config());
}
if (query.action === 'clearDatabase') {
await clearDatabase(ctx.joplin.db);
}

4
packages/server/src/routes/api/items.test.ts
View File

@ -154,7 +154,7 @@ describe('api_items', function() {
test('should batch upload items', async function() {
const { session: session1 } = await createUserAndSession(1, false);
const result: PaginatedResults = await putApi(session1.id, 'batch_items', {
const result: PaginatedResults<any> = await putApi(session1.id, 'batch_items', {
items: [
{
name: '00000000000000000000000000000001.md',
@ -177,7 +177,7 @@ describe('api_items', function() {
const note1 = makeNoteSerializedBody({ id: '00000000000000000000000000000001' });
await models().user().save({ id: user1.id, max_item_size: note1.length });
const result: PaginatedResults = await putApi(session1.id, 'batch_items', {
const result: PaginatedResults<any> = await putApi(session1.id, 'batch_items', {
items: [
{
name: '00000000000000000000000000000001.md',

3
packages/server/src/routes/api/items.ts
View File

@ -7,10 +7,11 @@ import { AppContext } from '../../utils/types';
import * as fs from 'fs-extra';
import { ErrorForbidden, ErrorMethodNotAllowed, ErrorNotFound, ErrorPayloadTooLarge, errorToPlainObject } from '../../utils/errors';
import ItemModel, { ItemSaveOption, SaveFromRawContentItem } from '../../models/ItemModel';
import { requestDeltaPagination, requestPagination } from '../../models/utils/pagination';
import { requestPagination } from '../../models/utils/pagination';
import { AclAction } from '../../models/BaseModel';
import { safeRemove } from '../../utils/fileUtils';
import { formatBytes, MB } from '../../utils/bytes';
import { requestDeltaPagination } from '../../models/ChangeModel';
const router = new Router(RouteType.Api);

2
packages/server/src/routes/api/share_users.test.ts
View File

@ -32,7 +32,7 @@ describe('share_users', function() {
const { share: share1 } = await shareWithUserAndAccept(session1.id, session2.id, user2, ShareType.Folder, folderItem1);
const { share: share2 } = await shareWithUserAndAccept(session1.id, session2.id, user2, ShareType.Folder, folderItem2);
const shareUsers = await getApi<PaginatedResults>(session2.id, 'share_users');
const shareUsers = await getApi<PaginatedResults<any>>(session2.id, 'share_users');
expect(shareUsers.items.length).toBe(2);
expect(shareUsers.items.find(su => su.share.id === share1.id)).toBeTruthy();
expect(shareUsers.items.find(su => su.share.id === share2.id)).toBeTruthy();

4
packages/server/src/routes/api/shares.test.ts
View File

@ -49,7 +49,7 @@ describe('shares', function() {
});
{
const shares = await getApi<PaginatedResults>(session1.id, 'shares');
const shares = await getApi<PaginatedResults<any>>(session1.id, 'shares');
expect(shares.items.length).toBe(2);
const share1: Share = shares.items.find(it => it.folder_id === '000000000000000000000000000000F1');
@ -60,7 +60,7 @@ describe('shares', function() {
expect(share2).toBeTruthy();
expect(share2.type).toBe(ShareType.Note);
const shareUsers = await getApi<PaginatedResults>(session1.id, `shares/${share1.id}/users`);
const shareUsers = await getApi<PaginatedResults<any>>(session1.id, `shares/${share1.id}/users`);
expect(shareUsers.items.length).toBe(2);
const su2 = shareUsers.items.find(su => su.user.email === 'user2@localhost');

2
packages/server/src/routes/index/tasks.ts
View File

@ -12,6 +12,7 @@ import { createCsrfTag } from '../../utils/csrf';
import { RunType } from '../../services/TaskService';
import { NotificationKey } from '../../models/NotificationModel';
import { NotificationLevel } from '../../services/database/types';
const prettyCron = require('prettycron');
const router: Router = new Router(RouteType.Web);
@ -76,6 +77,7 @@ router.get('tasks', async (_path: SubPath, ctx: AppContext) => {
},
{
value: task.schedule,
hint: prettyCron.toString(task.schedule),
},
{
value: yesOrNo(state.running),

150
packages/server/src/routes/index/user_deletions.ts Normal file
View File

@ -0,0 +1,150 @@
import { makeUrl, redirect, SubPath, UrlType } from '../../utils/routeUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
import { AppContext } from '../../utils/types';
import { ErrorBadRequest, ErrorForbidden, ErrorMethodNotAllowed } from '../../utils/errors';
import defaultView from '../../utils/defaultView';
import { yesOrNo } from '../../utils/strings';
import { makeTablePagination, makeTableView, Row, Table } from '../../utils/views/table';
import { PaginationOrderDir } from '../../models/utils/pagination';
import { formatDateTime } from '../../utils/time';
import { userDeletionsUrl, userUrl } from '../../utils/urlUtils';
import { createCsrfTag } from '../../utils/csrf';
import { bodyFields } from '../../utils/requestUtils';
const router: Router = new Router(RouteType.Web);
router.get('user_deletions', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
if (ctx.method === 'GET') {
const pagination = makeTablePagination(ctx.query, 'scheduled_time', PaginationOrderDir.ASC);
const page = await ctx.joplin.models.userDeletion().allPaginated(pagination);
const users = await ctx.joplin.models.user().loadByIds(page.items.map(d => d.user_id), { fields: ['id', 'email'] });
console.info(page);
const table: Table = {
baseUrl: userDeletionsUrl(),
requestQuery: ctx.query,
pageCount: page.page_count,
pagination,
headers: [
{
name: 'select',
label: '',
canSort: false,
},
{
name: 'email',
label: 'Email',
stretch: true,
},
{
name: 'process_data',
label: 'Data?',
},
{
name: 'process_account',
label: 'Account?',
},
{
name: 'scheduled_time',
label: 'Scheduled',
},
{
name: 'start_time',
label: 'Start',
},
{
name: 'end_time',
label: 'End',
},
{
name: 'success',
label: 'Success?',
},
{
name: 'error',
label: 'Error',
},
],
rows: page.items.map(d => {
const isDone = d.end_time && d.success;
const row: Row = [
{
value: `checkbox_${d.id}`,
checkbox: true,
},
{
value: isDone ? d.user_id : users.find(u => u.id === d.user_id).email,
stretch: true,
url: isDone ? '' : userUrl(d.user_id),
},
{
value: yesOrNo(d.process_data),
},
{
value: yesOrNo(d.process_account),
},
{
value: formatDateTime(d.scheduled_time),
},
{
value: formatDateTime(d.start_time),
},
{
value: formatDateTime(d.end_time),
},
{
value: d.end_time ? yesOrNo(d.success) : '-',
},
{
value: d.error,
},
];
return row;
}),
};
const view = defaultView('user_deletions', 'User deletions');
view.content = {
userDeletionTable: makeTableView(table),
postUrl: makeUrl(UrlType.UserDeletions),
csrfTag: await createCsrfTag(ctx),
};
view.cssFiles = ['index/user_deletions'];
return view;
}
throw new ErrorMethodNotAllowed();
});
router.post('user_deletions', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
interface PostFields {
removeButton: string;
}
const models = ctx.joplin.models;
const fields: PostFields = await bodyFields<PostFields>(ctx.req);
if (fields.removeButton) {
const jobIds = Object.keys(fields).filter(f => f.startsWith('checkbox_')).map(f => Number(f.substr(9)));
for (const jobId of jobIds) await models.userDeletion().remove(jobId);
await models.notification().addInfo(user.id, `${jobIds.length} job(s) have been removed`);
} else {
throw new ErrorBadRequest('Invalid action');
}
return redirect(ctx, makeUrl(UrlType.UserDeletions));
});
export default router;

47
packages/server/src/routes/index/users.ts
View File

@ -15,10 +15,10 @@ import uuidgen from '../../utils/uuidgen';
import { formatMaxItemSize, formatMaxTotalSize, formatTotalSize, formatTotalSizePercent, yesOrNo } from '../../utils/strings';
import { getCanShareFolder, totalSizeClass } from '../../models/utils/user';
import { yesNoDefaultOptions, yesNoOptions } from '../../utils/views/select';
import { confirmUrl, stripePortalUrl } from '../../utils/urlUtils';
import { confirmUrl, stripePortalUrl, userDeletionsUrl } from '../../utils/urlUtils';
import { cancelSubscriptionByUserId, updateCustomerEmail, updateSubscriptionType } from '../../utils/stripe';
import { createCsrfTag } from '../../utils/csrf';
import { formatDateTime } from '../../utils/time';
import { formatDateTime, Hour } from '../../utils/time';
import { cookieSet } from '../../utils/cookies';
import { startImpersonating, stopImpersonating } from './utils/users/impersonate';
import { userFlagToString } from '../../models/UserFlagModel';
@ -106,20 +106,23 @@ router.get('users', async (_path: SubPath, ctx: AppContext) => {
});
const view: View = defaultView('users', 'Users');
view.content.users = users.map(user => {
return {
...user,
displayName: user.full_name ? user.full_name : '(not set)',
formattedItemMaxSize: formatMaxItemSize(user),
formattedTotalSize: formatTotalSize(user),
formattedMaxTotalSize: formatMaxTotalSize(user),
formattedTotalSizePercent: formatTotalSizePercent(user),
totalSizeClass: totalSizeClass(user),
formattedAccountType: accountTypeToString(user.account_type),
formattedCanShareFolder: yesOrNo(getCanShareFolder(user)),
rowClassName: user.enabled ? '' : 'is-disabled',
};
});
view.content = {
users: users.map(user => {
return {
...user,
displayName: user.full_name ? user.full_name : '(not set)',
formattedItemMaxSize: formatMaxItemSize(user),
formattedTotalSize: formatTotalSize(user),
formattedMaxTotalSize: formatMaxTotalSize(user),
formattedTotalSizePercent: formatTotalSizePercent(user),
totalSizeClass: totalSizeClass(user),
formattedAccountType: accountTypeToString(user.account_type),
formattedCanShareFolder: yesOrNo(getCanShareFolder(user)),
rowClassName: user.enabled ? '' : 'is-disabled',
};
}),
userDeletionUrl: userDeletionsUrl(),
};
return view;
});
@ -163,6 +166,7 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null
if (!owner.is_admin) userFlagViews = [];
const subscription = !isNew ? await ctx.joplin.models.subscription().byUserId(userId) : null;
const isScheduledForDeletion = await ctx.joplin.models.userDeletion().isScheduledForDeletion(userId);
const view: View = defaultView('user', 'Profile');
view.content.user = user;
@ -186,6 +190,7 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null
view.content.showImpersonateButton = !isNew && !!owner.is_admin && user.enabled && user.id !== owner.id;
view.content.showRestoreButton = !isNew && !!owner.is_admin && !user.enabled;
view.content.showScheduleDeletionButton = !isNew && !!owner.is_admin && !isScheduledForDeletion;
view.content.showResetPasswordButton = !isNew && owner.is_admin && user.enabled;
view.content.canShareFolderOptions = yesNoDefaultOptions(user, 'can_share_folder');
view.content.canUploadOptions = yesNoOptions(user, 'can_upload');
@ -308,6 +313,7 @@ interface FormFields {
impersonate_button: string;
stop_impersonate_button: string;
delete_user_flags: string;
schedule_deletion_button: string;
}
router.post('users', async (path: SubPath, ctx: AppContext) => {
@ -364,6 +370,15 @@ router.post('users', async (path: SubPath, ctx: AppContext) => {
await updateSubscriptionType(models, userId, AccountType.Basic);
} else if (fields.update_subscription_pro_button) {
await updateSubscriptionType(models, userId, AccountType.Pro);
} else if (fields.schedule_deletion_button) {
const deletionDate = Date.now() + 24 * Hour;
await models.userDeletion().add(userId, deletionDate, {
processAccount: true,
processData: true,
});
await models.notification().addInfo(owner.id, `User ${user.email} has been scheduled for deletion on ${formatDateTime(deletionDate)}. [View deletion list](${userDeletionsUrl()})`);
} else if (fields.delete_user_flags) {
const userFlagTypes: UserFlagType[] = [];
for (const key of Object.keys(fields)) {

2
packages/server/src/routes/routes.ts
View File

@ -28,6 +28,7 @@ import indexTasks from './index/tasks';
import indexTerms from './index/terms';
import indexUpgrade from './index/upgrade';
import indexUsers from './index/users';
import indexUserDeletions from './index/user_deletions';
import defaultRoute from './default';
@ -60,6 +61,7 @@ const routes: Routers = {
'upgrade': indexUpgrade,
'help': indexHelp,
'tasks': indexTasks,
'user_deletions': indexUserDeletions,
'': defaultRoute,
};

16
packages/server/src/services/BaseService.ts
View File

@ -10,6 +10,7 @@ export default class BaseService {
private env_: Env;
private models_: Models;
private config_: Config;
protected name_: string = 'Untitled';
protected enabled_: boolean = true;
private destroyed_: boolean = false;
protected maintenanceInterval_: number = 10000;
@ -23,8 +24,12 @@ export default class BaseService {
this.scheduleMaintenance = this.scheduleMaintenance.bind(this);
}
protected get name(): string {
return this.name_;
}
public async destroy() {
if (this.destroyed_) throw new Error('Already destroyed');
if (this.destroyed_) throw new Error(`${this.name}: Already destroyed`);
this.destroyed_ = true;
this.scheduledMaintenances_ = [];
@ -75,12 +80,17 @@ export default class BaseService {
}
}
private async runMaintenance() {
public async runMaintenance() {
if (this.maintenanceInProgress_) {
logger.warn(`${this.name}: Skipping maintenance because it is already in progress`);
return;
}
this.maintenanceInProgress_ = true;
try {
await this.maintenance();
} catch (error) {
logger.error('Could not run maintenance', error);
logger.error(`${this.name}: Could not run maintenance`, error);
}
this.maintenanceInProgress_ = false;
}

8
packages/server/src/services/TaskService.test.ts
View File

@ -5,7 +5,13 @@ import { Env } from '../utils/types';
import TaskService, { RunType, Task } from './TaskService';
const newService = () => {
return new TaskService(Env.Dev, models(), config());
return new TaskService(Env.Dev, models(), config(), {
share: null,
email: null,
mustache: null,
tasks: null,
userDeletion: null,
});
};
describe('TaskService', function() {

13
packages/server/src/services/TaskService.ts
View File

@ -1,7 +1,9 @@
import Logger from '@joplin/lib/Logger';
import { Models } from '../models/factory';
import { Config, Env } from '../utils/types';
import BaseService from './BaseService';
import { Event, EventType } from './database/types';
import { Services } from './types';
const cron = require('node-cron');
const logger = Logger.create('TaskService');
@ -14,6 +16,7 @@ export enum TaskId {
HandleFailedPaymentSubscriptions = 5,
DeleteExpiredSessions = 6,
CompressOldChanges = 7,
ProcessUserDeletions = 8,
}
export enum RunType {
@ -31,7 +34,7 @@ export interface Task {
id: TaskId;
description: string;
schedule: string;
run(models: Models): void;
run(models: Models, services: Services): void;
}
export type Tasks = Record<number, Task>;
@ -53,6 +56,12 @@ export default class TaskService extends BaseService {
private tasks_: Tasks = {};
private taskStates_: Record<number, TaskState> = {};
private services_: Services;
public constructor(env: Env, models: Models, config: Config, services: Services) {
super(env, models, config);
this.services_ = services;
}
public registerTask(task: Task) {
if (this.tasks_[task.id]) throw new Error(`Already a task with this ID: ${task.id}`);
@ -106,7 +115,7 @@ export default class TaskService extends BaseService {
try {
logger.info(`Running ${displayString} (${runTypeToString(runType)})...`);
await this.tasks_[id].run(this.models);
await this.tasks_[id].run(this.models, this.services_);
} catch (error) {
logger.error(`On ${displayString}`, error);
}

137
packages/server/src/services/UserDeletionService.test.ts Normal file
View File

@ -0,0 +1,137 @@
import config from '../config';
import { shareFolderWithUser } from '../utils/testing/shareApiUtils';
import { afterAllTests, beforeAllDb, beforeEachDb, createNote, createUserAndSession, models } from '../utils/testing/testUtils';
import { Env } from '../utils/types';
import UserDeletionService from './UserDeletionService';
const newService = () => {
return new UserDeletionService(Env.Dev, models(), config());
};
describe('UserDeletionService', function() {
beforeAll(async () => {
await beforeAllDb('UserDeletionService');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should delete user data', async function() {
const { user: user1, session: session1 } = await createUserAndSession(1);
const { user: user2, session: session2 } = await createUserAndSession(2);
await createNote(session1.id, { title: 'testing1' });
await createNote(session2.id, { title: 'testing2' });
const t0 = new Date('2021-12-14').getTime();
const t1 = t0 + 1000;
const job = await models().userDeletion().add(user1.id, t1, {
processData: true,
processAccount: false,
});
expect(await models().item().count()).toBe(2);
expect(await models().change().count()).toBe(2);
const service = newService();
await service.processDeletionJob(job, { sleepBetweenOperations: 0 });
expect(await models().item().count()).toBe(1);
expect(await models().change().count()).toBe(1);
const item = (await models().item().all())[0];
expect(item.owner_id).toBe(user2.id);
const change = (await models().change().all())[0];
expect(change.user_id).toBe(user2.id);
expect(await models().user().count()).toBe(2);
expect(await models().session().count()).toBe(2);
});
test('should delete user account', async function() {
const { user: user1 } = await createUserAndSession(1);
const { user: user2 } = await createUserAndSession(2);
const t0 = new Date('2021-12-14').getTime();
const t1 = t0 + 1000;
const job = await models().userDeletion().add(user1.id, t1, {
processData: false,
processAccount: true,
});
expect(await models().user().count()).toBe(2);
expect(await models().session().count()).toBe(2);
const service = newService();
await service.processDeletionJob(job, { sleepBetweenOperations: 0 });
expect(await models().user().count()).toBe(1);
expect(await models().session().count()).toBe(1);
const user = (await models().user().all())[0];
expect(user.id).toBe(user2.id);
});
test('should not delete notebooks that are not owned', async function() {
const { session: session1 } = await createUserAndSession(1);
const { user: user2, session: session2 } = await createUserAndSession(2);
await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F2', [
{
id: '000000000000000000000000000000F2',
children: [
{
id: '00000000000000000000000000000001',
},
],
},
]);
expect(await models().share().count()).toBe(1);
expect(await models().shareUser().count()).toBe(1);
const job = await models().userDeletion().add(user2.id, Date.now());
const service = newService();
await service.processDeletionJob(job, { sleepBetweenOperations: 0 });
expect(await models().share().count()).toBe(1); // The share object has not (and should not) been deleted
expect(await models().shareUser().count()).toBe(0); // However all the invitations are gone
expect(await models().item().count()).toBe(2);
});
test('should not delete notebooks that are owned', async function() {
const { user: user1, session: session1 } = await createUserAndSession(1);
const { session: session2 } = await createUserAndSession(2);
await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F2', [
{
id: '000000000000000000000000000000F2',
children: [
{
id: '00000000000000000000000000000001',
},
],
},
]);
expect(await models().share().count()).toBe(1);
expect(await models().shareUser().count()).toBe(1);
const job = await models().userDeletion().add(user1.id, Date.now());
const service = newService();
await service.processDeletionJob(job, { sleepBetweenOperations: 0 });
expect(await models().share().count()).toBe(0);
expect(await models().shareUser().count()).toBe(0);
expect(await models().item().count()).toBe(0);
});
});

106
packages/server/src/services/UserDeletionService.ts Normal file
View File

@ -0,0 +1,106 @@
import Logger from '@joplin/lib/Logger';
import { Pagination } from '../models/utils/pagination';
import { msleep } from '../utils/time';
import BaseService from './BaseService';
import { UserDeletion, UserFlagType, Uuid } from './database/types';
const logger = Logger.create('UserDeletionService');
export interface DeletionJobOptions {
sleepBetweenOperations?: number;
}
export default class UserDeletionService extends BaseService {
protected name_: string = 'UserDeletionService';
private async deleteUserData(userId: Uuid, options: DeletionJobOptions) {
// While the "UserDeletionInProgress" flag is on, the account is
// disabled so that no new items or other changes can happen.
await this.models.userFlag().add(userId, UserFlagType.UserDeletionInProgress);
try {
// ---------------------------------------------------------------------
// Delete own shares and shares participated in. Note that when the
// shares are deleted, the associated user_items are deleted too, so we
// don't need to wait for ShareService to run to continue.
// ---------------------------------------------------------------------
logger.info(`Deleting shares for user ${userId}`);
await this.models.share().deleteByUserId(userId);
await this.models.shareUser().deleteByUserId(userId);
// ---------------------------------------------------------------------
// Delete items. Also delete associated change objects.
// ---------------------------------------------------------------------
logger.info(`Deleting items for user ${userId}`);
while (true) {
const pagination: Pagination = {
limit: 1000,
};
const page = await this.models.item().children(userId, '', pagination, { fields: ['id'] });
if (!page.items.length) break;
await this.models.item().delete(page.items.map(it => it.id), {
deleteChanges: true,
});
await msleep(options.sleepBetweenOperations);
}
} finally {
await this.models.userFlag().remove(userId, UserFlagType.UserDeletionInProgress);
}
}
private async deleteUserAccount(userId: Uuid, _options: DeletionJobOptions = null) {
logger.info(`Deleting user account: ${userId}`);
await this.models.userFlag().add(userId, UserFlagType.UserDeletionInProgress);
await this.models.session().deleteByUserId(userId);
await this.models.notification().deleteByUserId(userId);
await this.models.user().delete(userId);
await this.models.userFlag().deleteByUserId(userId);
}
public async processDeletionJob(deletion: UserDeletion, options: DeletionJobOptions = null) {
options = {
sleepBetweenOperations: 5000,
...options,
};
logger.info('Starting user deletion: ', deletion);
let error: any = null;
let success: boolean = true;
try {
await this.models.userDeletion().start(deletion.id);
if (deletion.process_data) await this.deleteUserData(deletion.user_id, options);
if (deletion.process_account) await this.deleteUserAccount(deletion.user_id, options);
} catch (e) {
error = e;
success = false;
logger.error(`Processing deletion ${deletion.id}:`, error);
}
await this.models.userDeletion().end(deletion.id, success, error);
logger.info('Completed user deletion: ', deletion.id);
}
public async processNextDeletionJob() {
const deletion = await this.models.userDeletion().next();
if (!deletion) return;
await this.processDeletionJob(deletion);
}
protected async maintenance() {
await this.processNextDeletionJob();
}
}

27
packages/server/src/services/database/types.ts
View File

@ -40,6 +40,7 @@ export enum UserFlagType {
AccountWithoutSubscription = 4,
SubscriptionCancelled = 5,
ManuallyDisabled = 6,
UserDeletionInProgress = 7,
}
export function userFlagTypeToLabel(t: UserFlagType): string {
@ -50,6 +51,7 @@ export function userFlagTypeToLabel(t: UserFlagType): string {
[UserFlagType.AccountWithoutSubscription]: 'Account Without Subscription',
[UserFlagType.SubscriptionCancelled]: 'Subscription Cancelled',
[UserFlagType.ManuallyDisabled]: 'Manually Disabled',
[UserFlagType.UserDeletionInProgress]: 'User deletion in progress',
};
if (!s[t]) throw new Error(`Unknown flag type: ${t}`);
@ -268,6 +270,18 @@ export interface Item extends WithDates, WithUuid {
content_storage_id?: number;
}
export interface UserDeletion extends WithDates {
id?: number;
user_id?: Uuid;
process_data?: number;
process_account?: number;
scheduled_time?: number;
start_time?: number;
end_time?: number;
success?: number;
error?: string;
}
export const databaseSchema: DatabaseTables = {
sessions: {
id: { type: 'string' },
@ -449,5 +463,18 @@ export const databaseSchema: DatabaseTables = {
owner_id: { type: 'string' },
content_storage_id: { type: 'number' },
},
user_deletions: {
id: { type: 'number' },
user_id: { type: 'string' },
process_data: { type: 'number' },
process_account: { type: 'number' },
scheduled_time: { type: 'string' },
start_time: { type: 'string' },
end_time: { type: 'string' },
success: { type: 'number' },
error: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
};
// AUTO-GENERATED-TYPES

2
packages/server/src/services/types.ts
View File

@ -2,10 +2,12 @@ import EmailService from './EmailService';
import MustacheService from './MustacheService';
import ShareService from './ShareService';
import TaskService from './TaskService';
import UserDeletionService from './UserDeletionService';
export interface Services {
share: ShareService;
email: EmailService;
mustache: MustacheService;
tasks: TaskService;
userDeletion: UserDeletionService;
}

13
packages/server/src/tools/debugTools.ts
View File

@ -3,6 +3,7 @@ import { DbConnection, dropTables, migrateLatest } from '../db';
import newModelFactory from '../models/factory';
import { AccountType } from '../models/UserModel';
import { User, UserFlagType } from '../services/database/types';
import { Minute, Second } from '../utils/time';
import { Config } from '../utils/types';
export interface CreateTestUsersOptions {
@ -98,3 +99,15 @@ export async function createTestUsers(db: DbConnection, config: Config, options:
}
}
}
export async function createUserDeletions(db: DbConnection, config: Config) {
const models = newModelFactory(db, config);
const users = await models.user().all();
for (let i = 0; i < 3; i++) {
if (i >= users.length) break;
if (users[i].is_admin) continue;
await models.userDeletion().add(users[i].id, Date.now() + 60 * Second + (i * 10 * Minute));
}
}

4
packages/server/src/tools/generateTypes.ts
View File

@ -35,6 +35,7 @@ const config = {
'main.user_items': 'WithDates',
'main.users': 'WithDates, WithUuid',
'main.events': 'WithUuid',
'main.user_deletions': 'WithDates',
},
};
@ -58,6 +59,9 @@ const propertyTypes: Record<string, string> = {
'users.total_item_size': 'number',
'events.created_time': 'number',
'events.type': 'EventType',
'user_deletions.start_time': 'number',
'user_deletions.end_time': 'number',
'user_deletions.scheduled_time': 'number',
};
function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void {

14
packages/server/src/utils/errors.ts
View File

@ -130,23 +130,29 @@ export class ErrorTooManyRequests extends ApiError {
}
export function errorToString(error: Error): string {
const msg: string[] = [];
msg.push(error.message ? error.message : 'Unknown error');
if (error.stack) msg.push(error.stack);
return msg.join(': ');
// const msg: string[] = [];
// msg.push(error.message ? error.message : 'Unknown error');
// if (error.stack) msg.push(error.stack);
// return msg.join(': ');
return JSON.stringify(errorToPlainObject(error));
}
interface PlainObjectError {
httpCode?: number;
message?: string;
code?: string;
stack?: string;
}
export function errorToPlainObject(error: any): PlainObjectError {
if (typeof error === 'string') return { message: error };
const output: PlainObjectError = {};
if ('httpCode' in error) output.httpCode = error.httpCode;
if ('code' in error) output.code = error.code;
if ('message' in error) output.message = error.message;
if ('stack' in error) output.stack = error.stack;
return output;
}

58
packages/server/src/utils/prettycron.testdisabled.ts Normal file
View File

@ -0,0 +1,58 @@
// const prettycron = require('./prettycron');
// describe('prettycron', function() {
// it('should check if an item is encrypted', async function() {
// const testCases = [
// { cron: '0 * * * *', readable: 'Every hour, on the hour', sixth: false },
// { cron: '30 * * * 1', readable: 'Every 30th minute past every hour on Mon', sixth: false },
// { cron: '15,45 9,21 * * *', readable: '09:15, 09:45, 21:15 and 21:45 every day', sixth: false },
// { cron: '18,19 7 5 * *', readable: '07:18 and 07:19 on the 5th of every month', sixth: false },
// { cron: '* * 25 12 *', readable: 'Every minute on the 25th in Dec', sixth: false },
// { cron: '0 * 1,3 * *', readable: 'Every hour, on the hour on the 1 and 3rd of every month', sixth: false },
// { cron: '0 17 * 1,4,7,10 *', readable: '17:00 every day in Jan, Apr, Jul and Oct', sixth: false },
// { cron: '15 * * * 1,2', readable: 'Every 15th minute past every hour on Mon and Tue', sixth: false },
// { cron: '* 8,10,12,14,16,18,20 * * *', readable: 'Every minute of 8, 10, 12, 14, 16, 18 and 20th hour', sixth: false },
// { cron: '0 12 15,16 1 3', readable: '12:00 on the 15 and 16th and every Wed in Jan', sixth: false },
// { cron: '0 4,8,12,4 * * 4,5,6', readable: 'On the 4, 8 and 12th hour on Thu, Fri and Sat', sixth: false },
// { cron: '0 2,16 1,8,15,22 * 1,2', readable: '02:00 and 16:00 on the 1, 8, 15 and 22nd of every month and every Mon and Tue', sixth: false },
// { cron: '15 3,8,10,12,14,16,18 16 * *', readable: 'Every 15th minute past the 3, 8, 10, 12, 14, 16 and 18th hour on the 16th of every month', sixth: false },
// { cron: '2 8,10,12,14,16,18 * 8 0,3', readable: 'Every 2nd minute past the 8, 10, 12, 14, 16 and 18th hour on Sun and Wed in Aug', sixth: false },
// { cron: '0 0 18 1/1 * ?', readable: '00:00 on the 18th of every month', sixth: false },
// { cron: '30 10 * * 0', readable: '10:30 on Sun', sixth: false },
// { cron: '* * * * *', readable: 'Every minute', sixth: false },
// { cron: '*/2 * * * *', readable: 'Every other minute', sixth: false },
// { cron: '0 0 18 1/1 * ? *', readable: '18:00:00 every day', sixth: true },
// { cron: '* * * * * *', readable: 'Every second', sixth: true },
// { cron: '0/1 0/1 0/1 0/1 0/1 0/1', readable: 'Every second', sixth: true },
// { cron: '*/4 2 4 * * *', readable: 'Every 4 seconds on the 2nd minute past the 4th hour', sixth: true },
// { cron: '30 15 9 * * *', readable: '09:15:30 every day', sixth: true },
// { cron: '*/30 15 9 * * *', readable: '09:15:00 and 09:15:30 every day', sixth: true },
// { cron: '*/2 * * * * *', readable: 'Every other second', sixth: true },
// { cron: '*/3 * * * * *', readable: 'Every 3 seconds', sixth: true },
// { cron: '*/4 * * * * *', readable: 'Every 4 seconds', sixth: true },
// { cron: '*/5 * * * * *', readable: 'Every 5 seconds', sixth: true },
// { cron: '*/6 * * * * *', readable: 'Every 6 seconds', sixth: true },
// { cron: '*/10 * * * * *', readable: 'Every 10 seconds', sixth: true },
// { cron: '*/12 * * * * *', readable: 'Every 12 seconds', sixth: true },
// { cron: '*/15 * * * * *', readable: 'Every 15 seconds', sixth: true },
// { cron: '*/20 * * * * *', readable: 'Every 20 seconds', sixth: true },
// { cron: '*/30 * * * * *', readable: 'Every minute starting on the first and 30th second', sixth: true },
// { cron: '5 * * * * *', readable: 'Every minute starting on the 5th second', sixth: true },
// { cron: '5 */2 * * * *', readable: 'Every other minute starting on the 5th second', sixth: true },
// { cron: '30 * * * * *', readable: 'Every minute starting on the 30th second', sixth: true },
// { cron: '0,2,4,20 * * * * *', readable: 'Every minute starting on the 0, 2, 4 and 20th second', sixth: true },
// { cron: '5,10/30 * * * 1,3 8', readable: 'Every minute starting on the 5, 10 and 40th second on Sat in Jan and Mar', sixth: true },
// { cron: '15-17 * * * * *', readable: 'Every minute starting on the 15, 16 and 17th second', sixth: true },
// ];
// for (const t of testCases) {
// const input = t.cron;
// const expected = t.readable;
// const actual = prettycron.toString(input, t.sixth);
// expect(actual).toBe(expected);
// }
// });
// });

384
packages/server/src/utils/prettycron.ts Normal file
View File

@ -0,0 +1,384 @@
// ==============================================================================
// 2021-12-15
// Modified for Joplin Server to use dayjs instead of moment.js
// Unfortunately it still requires the "later" library, which huge, so shouldn't
// be used for now
// ==============================================================================
// //////////////////////////////////////////////////////////////////////////////////
//
// prettycron.js
// Generates human-readable sentences from a schedule string in cron format
//
// Based on an earlier version by Pehr Johansson
// http://dsysadm.blogspot.com.au/2012/09/human-readable-cron-expressions-using.html
//
// //////////////////////////////////////////////////////////////////////////////////
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// //////////////////////////////////////////////////////////////////////////////////
const dayjs = require('dayjs');
const advancedFormat = require('dayjs/plugin/advancedFormat');
const calendar = require('dayjs/plugin/calendar');
dayjs.extend(advancedFormat);
dayjs.extend(calendar);
const later = require('later');
(function() {
const ordinal = {
ordinalSuffix(num: number) {
const ordinalsArray = ['th', 'st', 'nd', 'rd'];
// Get reminder of number by hundred so that we can counter number between 11-19
const offset = num % 100;
// Calculate position of ordinal to be used. Logic : Array index is calculated based on defined values.
const ordinalPos = ordinalsArray[ (offset - 20) % 10 ] || ordinalsArray[ offset ] || ordinalsArray[0];
// Return suffix
return ordinalPos;
},
toOrdinal(num: number) {
// Check if number is valid
// if( !validateNumber(num) ) {
// return `${num} is not a valid number`;
// }
// If number is zero no need to spend time on calculation
if (num === 0) {
return num.toString();
}
return num.toString() + this.ordinalSuffix(num);
},
};
// For an array of numbers, e.g. a list of hours in a schedule,
// return a string listing out all of the values (complete with
// "and" plus ordinal text on the last item).
const numberList = function(numbers: any[]) {
if (numbers.length < 2) {
return ordinal.toOrdinal(numbers[0]);
}
const last_val = numbers.pop();
return `${numbers.join(', ')} and ${ordinal.toOrdinal(last_val)}`;
};
const stepSize = function(numbers: any[]) {
if (!numbers || numbers.length <= 1) return 0;
const expectedStep = numbers[1] - numbers[0];
if (numbers.length == 2) return expectedStep;
// Check that every number is the previous number + the first number
return numbers.slice(1).every(function(n,i,a) {
return (i === 0 ? n : n - a[i - 1]) === expectedStep;
}) ? expectedStep : 0;
};
const isEveryOther = function(stepsize: number, numbers: any[]) {
return numbers.length === 30 && stepsize === 2;
};
const isTwicePerHour = function(stepsize: number, numbers: any[]) {
return numbers.length === 2 && stepsize === 30;
};
const isOnTheHour = function(numbers: any[]) {
return numbers.length === 1 && numbers[0] === 0;
};
const isStepValue = function(stepsize: number, numbers: any[]) {
// Value with slash (https://en.wikipedia.org/wiki/Cron#Non-Standard_Characters)
return numbers.length > 2 && stepsize > 0;
};
// For an array of numbers of seconds, return a string
// listing all the values unless they represent a frequency divisible by 60:
// /2, /3, /4, /5, /6, /10, /12, /15, /20 and /30
const getMinutesTextParts = function(numbers: any[]) {
const stepsize = stepSize(numbers);
if (!numbers) {
return { beginning: 'minute', text: '' };
}
const minutes = { beginning: '', text: '' };
if (isOnTheHour(numbers)) {
minutes.text = 'hour, on the hour';
} else if (isEveryOther(stepsize, numbers)) {
minutes.beginning = 'other minute';
} else if (isStepValue(stepsize, numbers)) {
minutes.text = `${stepsize} minutes`;
} else if (isTwicePerHour(stepsize, numbers)) {
minutes.text = 'first and 30th minute';
} else {
minutes.text = `${numberList(numbers)} minute`;
}
return minutes;
};
// For an array of numbers of seconds, return a string
// listing all the values unless they represent a frequency divisible by 60:
// /2, /3, /4, /5, /6, /10, /12, /15, /20 and /30
const getSecondsTextParts = function(numbers: any[]) {
const stepsize = stepSize(numbers);
if (!numbers) {
return { beginning: 'second', text: '' };
}
if (isEveryOther(stepsize, numbers)) {
return { beginning: '', text: 'other second' };
} else if (isStepValue(stepsize, numbers)) {
return { beginning: '', text: `${stepsize} seconds` };
} else {
return { beginning: 'minute', text: `starting on the ${numbers.length === 2 && stepsize === 30 ? 'first and 30th second' : `${numberList(numbers)} second`}` };
}
};
// Parse a number into day of week, or a month name;
// used in dateList below.
const numberToDateName = function(value: any, type: any) {
if (type === 'dow') {
return dayjs().day(value - 1).format('ddd');
} else if (type === 'mon') {
return dayjs().month(value - 1).format('MMM');
}
};
// From an array of numbers corresponding to dates (given in type: either
// days of the week, or months), return a string listing all the values.
const dateList = function(numbers: any[], type: any) {
if (numbers.length < 2) {
return numberToDateName(`${numbers[0]}`, type);
}
const last_val = `${numbers.pop()}`;
const output_text = '';
// No idea what is this nonsense so comenting it out for now.
// for (let i = 0, value; value = numbers[i]; i++) {
// if (output_text.length > 0) {
// output_text += ', ';
// }
// output_text += numberToDateName(value, type);
// }
return `${output_text} and ${numberToDateName(last_val, type)}`;
};
// Pad to equivalent of sprintf('%02d'). Both moment.js and later.js
// have zero-fill functions, but alas, they're private.
// let zeroPad = function(x:any) {
// return (x < 10) ? '0' + x : x;
// };
const removeFromSchedule = function(schedule: any, member: any, length: any) {
if (schedule[member] && schedule[member].length === length) {
delete schedule[member];
}
};
// ----------------
// Given a schedule from later.js (i.e. after parsing the cronspec),
// generate a friendly sentence description.
const scheduleToSentence = function(schedule: any, useSeconds: boolean) {
let textParts = [];
// A later.js schedules contains no member for time units where an asterisk is used,
// but schedules that means the same (e.g 0/1 is essentially the same as *) are
// returned with populated members.
// Remove all members that are fully populated to reduce complexity of code
removeFromSchedule(schedule, 'M', 12);
removeFromSchedule(schedule, 'D', 31);
removeFromSchedule(schedule, 'd', 7);
removeFromSchedule(schedule, 'h', 24);
removeFromSchedule(schedule, 'm', 60);
removeFromSchedule(schedule, 's', 60);
// let everySecond = useSeconds && schedule['s'] === undefined;
// let everyMinute = schedule['m'] === undefined;
// let everyHour = schedule['h'] === undefined;
const everyWeekday = schedule['d'] === undefined;
const everyDayInMonth = schedule['D'] === undefined;
// let everyMonth = schedule['M'] === undefined;
const oneOrTwoSecondsPerMinute = schedule['s'] && schedule['s'].length <= 2;
const oneOrTwoMinutesPerHour = schedule['m'] && schedule['m'].length <= 2;
const oneOrTwoHoursPerDay = schedule['h'] && schedule['h'].length <= 2;
const onlySpecificDaysOfMonth = schedule['D'] && schedule['D'].length !== 31;
if (oneOrTwoHoursPerDay && oneOrTwoMinutesPerHour && oneOrTwoSecondsPerMinute) {
// If there are only one or two specified values for
// hour or minute, print them in HH:MM format, or HH:MM:ss if seconds are used
// If seconds are not used, later.js returns one element for the seconds (set to zero)
const hm = [];
// let m = dayjs(new Date());
for (let i = 0; i < schedule['h'].length; i++) {
for (let j = 0; j < schedule['m'].length; j++) {
for (let k = 0; k < schedule['s'].length; k++) {
const s = dayjs()
.hour(schedule['h'][i])
.minute(schedule['m'][j])
.second(schedule['s'][k])
.format(useSeconds ? 'HH:mm:ss' : 'HH:mm');
hm.push(s);
// m.hour(schedule['h'][i]);
// m.minute(schedule['m'][j]);
// m.second(schedule['s'][k]);
// hm.push(m.format( useSeconds ? 'HH:mm:ss' : 'HH:mm'));
}
}
}
if (hm.length < 2) {
textParts.push(hm[0]);
} else {
const last_val = hm.pop();
textParts.push(`${hm.join(', ')} and ${last_val}`);
}
if (everyWeekday && everyDayInMonth) {
textParts.push('every day');
}
} else {
const seconds = getSecondsTextParts(schedule['s']);
const minutes = getMinutesTextParts(schedule['m']);
let beginning = '';
let end = '';
textParts.push('Every');
// Otherwise, list out every specified hour/minute value.
const hasSpecificSeconds = schedule['s'] && (
schedule['s'].length > 1 && schedule['s'].length < 60 ||
schedule['s'].length === 1 && schedule['s'][0] !== 0);
if (hasSpecificSeconds) {
beginning = seconds.beginning;
end = seconds.text;
}
if (schedule['h']) { // runs only at specific hours
if (hasSpecificSeconds) {
end += ' on the ';
}
if (schedule['m']) { // and only at specific minutes
const hours = `${numberList(schedule['h'])} hour`;
if (!hasSpecificSeconds && isOnTheHour(schedule['m'])) {
textParts = ['On the'];
end += hours;
} else {
beginning = minutes.beginning;
end += `${minutes.text} past the ${hours}`;
}
} else { // specific hours, but every minute
end += `minute of ${numberList(schedule['h'])} hour`;
}
} else if (schedule['m']) { // every hour, but specific minutes
beginning = minutes.beginning;
end += minutes.text;
if (!isOnTheHour(schedule['m']) && (onlySpecificDaysOfMonth || schedule['d'] || schedule['M'])) {
end += ' past every hour';
}
} else if (!schedule['s'] && !schedule['m']) {
beginning = seconds.beginning;
} else if (!useSeconds || !hasSpecificSeconds) { // cronspec has "*" for both hour and minute
beginning += minutes.beginning;
}
textParts.push(beginning);
textParts.push(end);
}
if (onlySpecificDaysOfMonth) { // runs only on specific day(s) of month
textParts.push(`on the ${numberList(schedule['D'])}`);
if (!schedule['M']) {
textParts.push('of every month');
}
}
if (schedule['d']) { // runs only on specific day(s) of week
if (schedule['D']) {
// if both day fields are specified, cron uses both; superuser.com/a/348372
textParts.push('and every');
} else {
textParts.push('on');
}
textParts.push(dateList(schedule['d'], 'dow'));
}
if (schedule['M']) {
if (schedule['M'].length === 12) {
textParts.push('day of every month');
} else {
// runs only in specific months; put this output last
textParts.push(`in ${dateList(schedule['M'], 'mon')}`);
}
}
return textParts.filter(function(p) { return p; }).join(' ');
};
// ----------------
// Given a cronspec, return the human-readable string.
const toString = function(cronspec: any, sixth: boolean) {
const schedule = later.parse.cron(cronspec, sixth);
return scheduleToSentence(schedule['schedules'][0], sixth);
};
// Given a cronspec, return the next date for when it will next run.
// (This is just a wrapper for later.js)
const getNextDate = function(cronspec: any, sixth: boolean) {
later.date.localTime();
const schedule = later.parse.cron(cronspec, sixth);
return later.schedule(schedule).next();
};
// Given a cronspec, return a friendly string for when it will next run.
// (This is just a wrapper for later.js and moment.js)
const getNext = function(cronspec: any, sixth: boolean) {
return dayjs(getNextDate(cronspec, sixth)).calendar();
};
// Given a cronspec and numDates, return a list of formatted dates
// of the next set of runs.
// (This is just a wrapper for later.js and moment.js)
const getNextDates = function(cronspec: any, numDates: any, sixth: boolean) {
const schedule = later.parse.cron(cronspec, sixth);
const nextDates = later.schedule(schedule).next(numDates);
const nextPrettyDates = [];
for (let i = 0; i < nextDates.length; i++) {
nextPrettyDates.push(dayjs(nextDates[i]).calendar());
}
return nextPrettyDates;
};
// ----------------
// attach ourselves to window in the browser, and to exports in Node,
// so our functions can always be called as prettyCron.toString()
const global_obj = (typeof exports !== 'undefined' && exports !== null) ? exports : (window as any).prettyCron = {};
global_obj.toString = toString;
global_obj.getNext = getNext;
global_obj.getNextDate = getNextDate;
global_obj.getNextDates = getNextDates;
}).call(this);

1
packages/server/src/utils/routeUtils.ts
View File

@ -289,6 +289,7 @@ export enum UrlType {
Terms = 'terms',
Privacy = 'privacy',
Tasks = 'tasks',
UserDeletions = 'user_deletions',
}
export function makeUrl(urlType: UrlType): string {

6
packages/server/src/utils/setupAppContext.ts
View File

@ -9,15 +9,19 @@ import { Services } from '../services/types';
import EmailService from '../services/EmailService';
import MustacheService from '../services/MustacheService';
import setupTaskService from './setupTaskService';
import UserDeletionService from '../services/UserDeletionService';
async function setupServices(env: Env, models: Models, config: Config): Promise<Services> {
const output: Services = {
share: new ShareService(env, models, config),
email: new EmailService(env, models, config),
mustache: new MustacheService(config.viewDir, config.baseUrl),
tasks: setupTaskService(env, models, config),
userDeletion: new UserDeletionService(env, models, config),
tasks: null,
};
output.tasks = setupTaskService(env, models, config, output),
await output.mustache.loadPartials();
return output;

12
packages/server/src/utils/setupTaskService.ts
View File

@ -1,9 +1,10 @@
import { Models } from '../models/factory';
import TaskService, { Task, TaskId } from '../services/TaskService';
import { Services } from '../services/types';
import { Config, Env } from './types';
export default function(env: Env, models: Models, config: Config): TaskService {
const taskService = new TaskService(env, models, config);
export default function(env: Env, models: Models, config: Config, services: Services): TaskService {
const taskService = new TaskService(env, models, config, services);
let tasks: Task[] = [
{
@ -27,6 +28,13 @@ export default function(env: Env, models: Models, config: Config): TaskService {
run: (models: Models) => models.change().compressOldChanges(),
},
{
id: TaskId.ProcessUserDeletions,
description: 'Process user deletions',
schedule: '0 */6 * * *',
run: (_models: Models, services: Services) => services.userDeletion.runMaintenance(),
},
// Need to do it relatively frequently so that if the user fixes
// whatever was causing the oversized account, they can get it
// re-enabled quickly. Also it's done on minute 30 because it depends on

38
packages/server/src/utils/testing/fileApiUtils.ts
View File

@ -103,25 +103,25 @@ export async function postDirectory(sessionId: string, parentPath: string, name:
return context.response.body;
}
export async function getDirectoryChildrenContext(sessionId: string, path: string, pagination: Pagination = null): Promise<AppContext> {
const context = await koaAppContext({
sessionId: sessionId,
request: {
method: 'GET',
url: `/api/files/${path}/children`,
query: paginationToQueryParams(pagination),
},
});
// export async function getDirectoryChildrenContext(sessionId: string, path: string, pagination: Pagination = null): Promise<AppContext> {
// const context = await koaAppContext({
// sessionId: sessionId,
// request: {
// method: 'GET',
// url: `/api/files/${path}/children`,
// query: paginationToQueryParams(pagination),
// },
// });
await routeHandler(context);
return context;
}
// await routeHandler(context);
// return context;
// }
export async function getDirectoryChildren(sessionId: string, path: string, pagination: Pagination = null): Promise<PaginatedResults> {
const context = await getDirectoryChildrenContext(sessionId, path, pagination);
checkContextError(context);
return context.response.body as PaginatedResults;
}
// export async function getDirectoryChildren(sessionId: string, path: string, pagination: Pagination = null): Promise<PaginatedResults<any>> {
// const context = await getDirectoryChildrenContext(sessionId, path, pagination);
// checkContextError(context);
// return context.response.body as PaginatedResults;
// }
export async function putFileContentContext(sessionId: string, path: string, filePath: string): Promise<AppContext> {
const context = await koaAppContext({
@ -194,8 +194,8 @@ export async function getDeltaContext(sessionId: string, path: string, paginatio
return context;
}
export async function getDelta(sessionId: string, path: string, pagination: Pagination): Promise<PaginatedResults> {
export async function getDelta(sessionId: string, path: string, pagination: Pagination): Promise<PaginatedResults<any>> {
const context = await getDeltaContext(sessionId, path, pagination);
checkContextError(context);
return context.response.body as PaginatedResults;
return context.response.body as PaginatedResults<any>;
}

1
packages/server/src/utils/time.ts
View File

@ -30,6 +30,7 @@ export function msleep(ms: number) {
}
export function formatDateTime(ms: number | Date): string {
if (!ms) return '-';
ms = ms instanceof Date ? ms.getTime() : ms;
return `${dayjs(ms).format('D MMM YY HH:mm:ss')} (${defaultTimezone()})`;
}

8
packages/server/src/utils/urlUtils.ts
View File

@ -45,3 +45,11 @@ export function homeUrl(): string {
export function loginUrl(): string {
return `${config().baseUrl}/login`;
}
export function userDeletionsUrl(): string {
return `${config().baseUrl}/user_deletions`;
}
export function userUrl(userId: Uuid): string {
return `${config().baseUrl}/users/${userId}`;
}

10
packages/server/src/utils/views/table.ts
View File

@ -26,6 +26,7 @@ export interface Header {
name: string;
label: string;
stretch?: boolean;
canSort?: boolean;
}
interface HeaderView {
@ -40,6 +41,7 @@ interface RowItem {
checkbox?: boolean;
url?: string;
stretch?: boolean;
hint?: string;
}
export type Row = RowItem[];
@ -49,6 +51,7 @@ interface RowItemView {
classNames: string[];
url: string;
checkbox: boolean;
hint: string;
}
type RowView = RowItemView[];
@ -79,9 +82,11 @@ export function makeTablePagination(query: any, defaultOrderField: string, defau
}
function makeHeaderView(header: Header, parentBaseUrl: string, baseUrlQuery: PaginationQueryParams, pagination: Pagination): HeaderView {
const canSort = header.canSort !== false;
return {
label: header.label,
sortLink: !pagination ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
sortLink: !pagination || !canSort ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
classNames: [header.stretch ? 'stretch' : 'nowrap', headerIsSelectedClass(header.name, pagination)],
iconDir: headerSortIconDir(header.name, pagination),
};
@ -94,6 +99,7 @@ function makeRowView(row: Row): RowView {
classNames: [rowItem.stretch ? 'stretch' : 'nowrap'],
url: rowItem.url,
checkbox: rowItem.checkbox,
hint: rowItem.hint,
};
});
}
@ -104,6 +110,8 @@ export function makeTableView(table: Table): TableView {
let pagination: Pagination = null;
if (table.pageCount) {
if (!table.baseUrl || !table.requestQuery) throw new Error('Table.baseUrl and Table.requestQuery are required for pagination when there is more than one page');
baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
pagination = table.pagination;
paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));

4
packages/server/src/views/index/user.mustache
View File

@ -79,6 +79,7 @@
</div>
<p id="password_strength" class="help"></p>
</div>
<div class="field">
<label class="label">Repeat password</label>
<div class="control">
@ -103,6 +104,9 @@
{{#showRestoreButton}}
<input type="submit" name="restore_button" class="button is-danger" value="Restore" />
{{/showRestoreButton}}
{{#showScheduleDeletionButton}}
<input type="submit" name="schedule_deletion_button" class="button is-danger" value="Schedule for deletion" />
{{/showScheduleDeletionButton}}
</div>
</div>

11
packages/server/src/views/index/user_deletions.mustache Normal file
View File

@ -0,0 +1,11 @@
<form method='POST' action="{{postUrl}}">
{{{csrfTag}}}
{{#userDeletionTable}}
{{>table}}
{{/userDeletionTable}}
<div class="block">
<input class="button is-link" type="submit" value="Remove selected jobs" name="removeButton"/>
</div>
</form>

4
packages/server/src/views/index/users.mustache
View File

@ -1,3 +1,7 @@
<div class="block">
<a href="{{{userDeletionUrl}}}">&gt; User deletions</a>
</div>
<div class="block">
<a class="button is-primary" href="{{{global.baseUrl}}}/users/new">Add user</a>
<a class="button is-link toggle-disabled-button hide-disabled" href="#">Hide disabled</a>

2
packages/server/src/views/partials/tableRowItem.mustache
View File

@ -3,6 +3,6 @@
<input type="checkbox" name="{{value}}"/>
{{/checkbox}}
{{^checkbox}}
{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
{{#url}}<a href="{{.}}"></span>{{/url}}{{#hint}}<abbr title="{{.}}">{{/hint}}{{value}}</a>{{#hint}}</abbr>{{/hint}}
{{/checkbox}}
</td>

20
yarn.lock
View File

@ -3391,6 +3391,7 @@ __metadata:
nodemon: ^2.0.6
pg: ^8.5.1
pretty-bytes: ^5.6.0
prettycron: ^0.10.0
query-string: ^6.8.3
rate-limiter-flexible: ^2.2.4
raw-body: ^2.4.1
@ -19299,6 +19300,13 @@ __metadata:
languageName: node
linkType: hard
"later@npm:^1.1.6":
version: 1.2.0
resolution: "later@npm:1.2.0"
checksum: 49dda2d5b3acda7a190a756c1cecc959853e8cbd1cc4c4a3b8dc0a3ec3859ec031adafe5e9659e2ad3ee13fb06288cc72d894fb8ce7bca17c0155666fdf6211b
languageName: node
linkType: hard
"latest-version@npm:^5.1.0":
version: 5.1.0
resolution: "latest-version@npm:5.1.0"
@ -21469,7 +21477,7 @@ __metadata:
languageName: node
linkType: hard
"moment@npm:>= 2.9.0, moment@npm:^2.15.1, moment@npm:^2.22.2, moment@npm:^2.24.0, moment@npm:^2.29.1":
"moment@npm:>= 2.9.0, moment@npm:^2.15.1, moment@npm:^2.22.2, moment@npm:^2.24.0, moment@npm:^2.29.1, moment@npm:^2.8.4":
version: 2.29.1
resolution: "moment@npm:2.29.1"
checksum: 1e14d5f422a2687996be11dd2d50c8de3bd577c4a4ca79ba5d02c397242a933e5b941655de6c8cb90ac18f01cc4127e55b4a12ae3c527a6c0a274e455979345e
@ -24062,6 +24070,16 @@ __metadata:
languageName: node
linkType: hard
"prettycron@npm:^0.10.0":
version: 0.10.0
resolution: "prettycron@npm:0.10.0"
dependencies:
later: ^1.1.6
moment: ^2.8.4
checksum: a8e9b6e1e59f0b5b5293b21b1ebaea8f043328db9046695d0bca498d1337314ed151ad4b6df0a957513c2b59a5c777e8c2e190fb805a452c18fd2cecb7492c85
languageName: node
linkType: hard
"private@npm:^0.1.8":
version: 0.1.8
resolution: "private@npm:0.1.8"