diff --git a/packages/server/schema.sqlite b/packages/server/schema.sqlite index 061ec0fe73..26e3a2fd49 100644 Binary files a/packages/server/schema.sqlite and b/packages/server/schema.sqlite differ diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index b5e9703852..f6beff6757 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -208,7 +208,7 @@ async function main() { await initializeJoplinUtils(config(), ctx.models, ctx.joplin.services.mustache); appLogger().info('Migrating database...'); - await migrateDb(ctx.db); + await migrateDb(ctx.joplin.db); appLogger().info('Starting services...'); await startServices(ctx); diff --git a/packages/server/src/db.ts b/packages/server/src/db.ts index ddde737c75..bb76594a7f 100644 --- a/packages/server/src/db.ts +++ b/packages/server/src/db.ts @@ -291,20 +291,6 @@ interface DatabaseTables { // AUTO-GENERATED-TYPES // Auto-generated using `npm run generate-types` -export interface User extends WithDates, WithUuid { - email?: string; - password?: string; - full_name?: string; - is_admin?: number; - max_item_size?: number; - can_share_folder?: number; - email_confirmed?: number; - must_set_password?: number; - account_type?: number; - can_upload?: number; - can_share_note?: number; -} - export interface Session extends WithDates, WithUuid { user_id?: Uuid; auth_code?: string; @@ -427,23 +413,23 @@ export interface Subscription { created_time?: string; } +export interface User extends WithDates, WithUuid { + email?: string; + password?: string; + full_name?: string; + is_admin?: number; + email_confirmed?: number; + must_set_password?: number; + account_type?: number; + can_upload?: number; + max_item_size?: number | null; + can_share_folder?: number | null; + can_share_note?: number | null; + max_total_item_size?: number | null; + total_item_size?: number; +} + export const databaseSchema: DatabaseTables = { - users: { - id: { type: 'string' }, - email: { type: 'string' }, - password: { type: 'string' }, - full_name: { type: 'string' }, - is_admin: { type: 'number' }, - updated_time: { type: 'string' }, - created_time: { type: 'string' }, - max_item_size: { type: 'number' }, - can_share_folder: { type: 'number' }, - email_confirmed: { type: 'number' }, - must_set_password: { type: 'number' }, - account_type: { type: 'number' }, - can_upload: { type: 'number' }, - can_share_note: { type: 'number' }, - }, sessions: { id: { type: 'string' }, user_id: { type: 'string' }, @@ -579,5 +565,23 @@ export const databaseSchema: DatabaseTables = { updated_time: { type: 'string' }, created_time: { type: 'string' }, }, + users: { + id: { type: 'string' }, + email: { type: 'string' }, + password: { type: 'string' }, + full_name: { type: 'string' }, + is_admin: { type: 'number' }, + updated_time: { type: 'string' }, + created_time: { type: 'string' }, + email_confirmed: { type: 'number' }, + must_set_password: { type: 'number' }, + account_type: { type: 'number' }, + can_upload: { type: 'number' }, + max_item_size: { type: 'number' }, + can_share_folder: { type: 'number' }, + can_share_note: { type: 'number' }, + max_total_item_size: { type: 'string' }, + total_item_size: { type: 'number' }, + }, }; // AUTO-GENERATED-TYPES diff --git a/packages/server/src/migrations/20210702182439_account_max_total_size.ts b/packages/server/src/migrations/20210702182439_account_max_total_size.ts new file mode 100644 index 0000000000..341767f27c --- /dev/null +++ b/packages/server/src/migrations/20210702182439_account_max_total_size.ts @@ -0,0 +1,18 @@ +import { Knex } from 'knex'; +import { DbConnection } from '../db'; + +// Due to a bug in Knex.js, it is not possible to drop a column and +// recreate it in the same migration. So this is split into two migrations. +// https://github.com/knex/knex/issues/2581 + +export async function up(db: DbConnection): Promise { + await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) { + table.dropColumn('max_item_size'); + table.dropColumn('can_share_folder'); + table.dropColumn('can_share_note'); + }); +} + +export async function down(_db: DbConnection): Promise { + +} diff --git a/packages/server/src/migrations/20210702182440_account_max_total_size_2.ts b/packages/server/src/migrations/20210702182440_account_max_total_size_2.ts new file mode 100644 index 0000000000..da9022ffd2 --- /dev/null +++ b/packages/server/src/migrations/20210702182440_account_max_total_size_2.ts @@ -0,0 +1,19 @@ +import { Knex } from 'knex'; +import { DbConnection } from '../db'; + +export async function up(db: DbConnection): Promise { + await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) { + table.integer('max_item_size').defaultTo(null).nullable(); + table.specificType('can_share_folder', 'smallint').defaultTo(null).nullable(); + table.specificType('can_share_note', 'smallint').defaultTo(null).nullable(); + table.bigInteger('max_total_item_size').defaultTo(null).nullable(); + }); + + await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) { + table.integer('total_item_size').defaultTo(0).notNullable(); + }); +} + +export async function down(_db: DbConnection): Promise { + +} diff --git a/packages/server/src/models/ItemModel.test.ts b/packages/server/src/models/ItemModel.test.ts index fce1e3312c..cab1681005 100644 --- a/packages/server/src/models/ItemModel.test.ts +++ b/packages/server/src/models/ItemModel.test.ts @@ -1,4 +1,4 @@ -import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder } from '../utils/testing/testUtils'; +import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder, createItemTree3 } from '../utils/testing/testUtils'; import { shareFolderWithUser } from '../utils/testing/shareApiUtils'; import { resourceBlobPath } from '../utils/joplinUtils'; @@ -142,4 +142,99 @@ describe('ItemModel', function() { expect(await models().item().childrenCount(user1.id)).toBe(2); }); + test('should calculate the total size', async function() { + const { user: user1 } = await createUserAndSession(1); + const { user: user2 } = await createUserAndSession(2); + const { user: user3 } = await createUserAndSession(3); + + await createItemTree3(user1.id, '', '', [ + { + id: '000000000000000000000000000000F1', + children: [ + { + id: '00000000000000000000000000000001', + }, + ], + }, + ]); + + await createItemTree3(user2.id, '', '', [ + { + id: '000000000000000000000000000000F2', + children: [ + { + id: '00000000000000000000000000000002', + }, + { + id: '00000000000000000000000000000003', + }, + ], + }, + ]); + + const folder1 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F1'); + const folder2 = await models().item().loadByJopId(user2.id, '000000000000000000000000000000F2'); + const note1 = await models().item().loadByJopId(user1.id, '00000000000000000000000000000001'); + const note2 = await models().item().loadByJopId(user2.id, '00000000000000000000000000000002'); + const note3 = await models().item().loadByJopId(user2.id, '00000000000000000000000000000003'); + + const totalSize1 = await models().item().calculateUserTotalSize(user1.id); + const totalSize2 = await models().item().calculateUserTotalSize(user2.id); + const totalSize3 = await models().item().calculateUserTotalSize(user3.id); + + const expected1 = folder1.content_size + note1.content_size; + const expected2 = folder2.content_size + note2.content_size + note3.content_size; + const expected3 = 0; + + expect(totalSize1).toBe(expected1); + expect(totalSize2).toBe(expected2); + expect(totalSize3).toBe(expected3); + + await models().item().updateTotalSizes(); + expect((await models().user().load(user1.id)).total_item_size).toBe(totalSize1); + expect((await models().user().load(user2.id)).total_item_size).toBe(totalSize2); + expect((await models().user().load(user3.id)).total_item_size).toBe(totalSize3); + }); + + test('should include shared items in total size calculation', async function() { + const { user: user1, session: session1 } = await createUserAndSession(1); + const { user: user2, session: session2 } = await createUserAndSession(2); + const { user: user3 } = await createUserAndSession(3); + + await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', [ + { + id: '000000000000000000000000000000F1', + children: [ + { + id: '00000000000000000000000000000001', + }, + ], + }, + { + id: '000000000000000000000000000000F2', + }, + ]); + + const folder1 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F1'); + const folder2 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F2'); + const note1 = await models().item().loadByJopId(user1.id, '00000000000000000000000000000001'); + + const totalSize1 = await models().item().calculateUserTotalSize(user1.id); + const totalSize2 = await models().item().calculateUserTotalSize(user2.id); + const totalSize3 = await models().item().calculateUserTotalSize(user3.id); + + const expected1 = folder1.content_size + folder2.content_size + note1.content_size; + const expected2 = folder1.content_size + note1.content_size; + const expected3 = 0; + + expect(totalSize1).toBe(expected1); + expect(totalSize2).toBe(expected2); + expect(totalSize3).toBe(expected3); + + await models().item().updateTotalSizes(); + expect((await models().user().load(user1.id)).total_item_size).toBe(expected1); + expect((await models().user().load(user2.id)).total_item_size).toBe(expected2); + expect((await models().user().load(user3.id)).total_item_size).toBe(expected3); + }); + }); diff --git a/packages/server/src/models/ItemModel.ts b/packages/server/src/models/ItemModel.ts index c7e2799183..b54d8009cd 100644 --- a/packages/server/src/models/ItemModel.ts +++ b/packages/server/src/models/ItemModel.ts @@ -1,11 +1,12 @@ import BaseModel, { SaveOptions, LoadOptions, DeleteOptions, ValidateOptions, AclAction } from './BaseModel'; -import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User } from '../db'; +import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User, UserItem } from '../db'; import { defaultPagination, paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination'; import { isJoplinItemName, isJoplinResourceBlobPath, linkedResourceIds, serializeJoplinItem, unserializeJoplinItem } from '../utils/joplinUtils'; import { ModelType } from '@joplin/lib/BaseModel'; import { ApiError, ErrorForbidden, ErrorUnprocessableEntity } from '../utils/errors'; import { Knex } from 'knex'; import { ChangePreviousItem } from './ChangeModel'; +import { unique } from '../utils/array'; const mimeUtils = require('@joplin/lib/mime-utils.js').mime; @@ -39,6 +40,8 @@ export interface ItemSaveOption extends SaveOptions { export default class ItemModel extends BaseModel { + private updatingTotalSizes_: boolean = false; + protected get tableName(): string { return 'items'; } @@ -591,6 +594,69 @@ export default class ItemModel extends BaseModel { }, 'ItemModel::saveForUser'); } + public async updateTotalSizes(): Promise { + interface TotalSizeRow { + userId: Uuid; + totalSize: number; + } + + // Total sizes are updated once an hour, so unless there's something + // very wrong this error shouldn't happen. + if (this.updatingTotalSizes_) throw new Error('Already updating total sizes'); + + this.updatingTotalSizes_ = true; + + const doneUserIds: Record = {}; + + try { + while (true) { + const latestProcessedChange = await this.models().keyValue().value('ItemModel::updateTotalSizes::latestProcessedChange'); + + const changes = await this.models().change().allFromId(latestProcessedChange || ''); + if (!changes.length) break; + + const itemIds: Uuid[] = unique(changes.map(c => c.item_id)); + const userItems: UserItem[] = await this.db('user_items').select('user_id').whereIn('item_id', itemIds); + const userIds: Uuid[] = unique(userItems.map(u => u.user_id)); + + const totalSizes: TotalSizeRow[] = []; + for (const userId of userIds) { + if (doneUserIds[userId]) continue; + + totalSizes.push({ + userId, + totalSize: await this.calculateUserTotalSize(userId), + }); + + doneUserIds[userId] = true; + } + + await this.withTransaction(async () => { + for (const row of totalSizes) { + await this.models().user().save({ + id: row.userId, + total_item_size: row.totalSize, + }); + } + + await this.models().keyValue().setValue('ItemModel::updateTotalSizes::latestProcessedChange', changes[changes.length - 1].id); + }, 'ItemModel::updateTotalSizes'); + } + } finally { + this.updatingTotalSizes_ = false; + } + } + + public async calculateUserTotalSize(userId: Uuid): Promise { + const result = await this.db('items') + .sum('items.content_size', { as: 'total' }) + .leftJoin('user_items', 'items.id', 'user_items.item_id') + .where('user_items.user_id', userId) + .first(); + + return result && result.total ? result.total : 0; + } + public async save(_item: Item, _options: SaveOptions = {}): Promise { throw new Error('Use saveForUser()'); // return this.saveForUser('', item, options); diff --git a/packages/server/src/models/ShareModel.ts b/packages/server/src/models/ShareModel.ts index b5fac48cb5..fe28c18a73 100644 --- a/packages/server/src/models/ShareModel.ts +++ b/packages/server/src/models/ShareModel.ts @@ -6,6 +6,7 @@ import { ErrorBadRequest, ErrorForbidden, ErrorNotFound } from '../utils/errors' import { setQueryParameters } from '../utils/urlUtils'; import BaseModel, { AclAction, DeleteOptions, ValidateOptions } from './BaseModel'; import { userIdFromUserContentUrl } from '../utils/routeUtils'; +import { getCanShareFolder } from './utils/user'; export default class ShareModel extends BaseModel { @@ -15,7 +16,7 @@ export default class ShareModel extends BaseModel { public async checkIfAllowed(user: User, action: AclAction, resource: Share = null): Promise { if (action === AclAction.Create) { - if (resource.type === ShareType.Folder && !user.can_share_folder) throw new ErrorForbidden('The sharing feature is not enabled for this account'); + if (resource.type === ShareType.Folder && !getCanShareFolder(user)) throw new ErrorForbidden('The sharing feature is not enabled for this account'); // Note that currently all users can always share notes by URL so // there's no check on the permission diff --git a/packages/server/src/models/ShareUserModel.ts b/packages/server/src/models/ShareUserModel.ts index e194285666..083227a796 100644 --- a/packages/server/src/models/ShareUserModel.ts +++ b/packages/server/src/models/ShareUserModel.ts @@ -1,6 +1,7 @@ import { Item, Share, ShareType, ShareUser, ShareUserStatus, User, Uuid } from '../db'; import { ErrorForbidden, ErrorNotFound } from '../utils/errors'; import BaseModel, { AclAction, DeleteOptions } from './BaseModel'; +import { getCanShareFolder } from './utils/user'; export default class ShareUserModel extends BaseModel { @@ -10,8 +11,8 @@ export default class ShareUserModel extends BaseModel { public async checkIfAllowed(user: User, action: AclAction, resource: ShareUser = null): Promise { if (action === AclAction.Create) { - const recipient = await this.models().user().load(resource.user_id, { fields: ['can_share_folder'] }); - if (!recipient.can_share_folder) throw new ErrorForbidden('The sharing feature is not enabled for the recipient account'); + const recipient = await this.models().user().load(resource.user_id, { fields: ['account_type', 'can_share_folder'] }); + if (!getCanShareFolder(recipient)) throw new ErrorForbidden('The sharing feature is not enabled for the recipient account'); const share = await this.models().share().load(resource.share_id); if (share.owner_id !== user.id) throw new ErrorForbidden('no access to the share object'); diff --git a/packages/server/src/models/SubscriptionModel.test.ts b/packages/server/src/models/SubscriptionModel.test.ts index a48e165972..9ee12f43a2 100644 --- a/packages/server/src/models/SubscriptionModel.test.ts +++ b/packages/server/src/models/SubscriptionModel.test.ts @@ -1,6 +1,7 @@ import { beforeAllDb, afterAllTests, beforeEachDb, models } from '../utils/testing/testUtils'; import { AccountType } from './UserModel'; import { MB } from '../utils/bytes'; +import { getCanShareFolder, getMaxItemSize } from './utils/user'; describe('SubscriptionModel', function() { @@ -29,8 +30,8 @@ describe('SubscriptionModel', function() { expect(user.account_type).toBe(AccountType.Pro); expect(user.email).toBe('toto@example.com'); - expect(user.can_share_folder).toBe(1); - expect(user.max_item_size).toBe(200 * MB); + expect(getCanShareFolder(user)).toBe(1); + expect(getMaxItemSize(user)).toBe(200 * MB); expect(sub.stripe_subscription_id).toBe('STRIPE_SUB_ID'); expect(sub.stripe_user_id).toBe('STRIPE_USER_ID'); diff --git a/packages/server/src/models/SubscriptionModel.ts b/packages/server/src/models/SubscriptionModel.ts index eba819702c..7c57efe48a 100644 --- a/packages/server/src/models/SubscriptionModel.ts +++ b/packages/server/src/models/SubscriptionModel.ts @@ -2,7 +2,7 @@ import { EmailSender, Subscription, Uuid } from '../db'; import { ErrorNotFound } from '../utils/errors'; import uuidgen from '../utils/uuidgen'; import BaseModel from './BaseModel'; -import { AccountType, accountTypeProperties } from './UserModel'; +import { AccountType } from './UserModel'; export default class SubscriptionModel extends BaseModel { @@ -51,7 +51,7 @@ export default class SubscriptionModel extends BaseModel { public async saveUserAndSubscription(email: string, accountType: AccountType, stripeUserId: string, stripeSubscriptionId: string) { return this.withTransaction(async () => { const user = await this.models().user().save({ - ...accountTypeProperties(accountType), + account_type: accountType, email, email_confirmed: 1, password: uuidgen(), diff --git a/packages/server/src/models/UserModel.ts b/packages/server/src/models/UserModel.ts index 5e4556ec45..97513da898 100644 --- a/packages/server/src/models/UserModel.ts +++ b/packages/server/src/models/UserModel.ts @@ -4,8 +4,9 @@ import * as auth from '../utils/auth'; import { ErrorUnprocessableEntity, ErrorForbidden, ErrorPayloadTooLarge, ErrorNotFound } from '../utils/errors'; import { ModelType } from '@joplin/lib/BaseModel'; import { _ } from '@joplin/lib/locale'; -import { formatBytes, MB } from '../utils/bytes'; +import { formatBytes, GB, MB } from '../utils/bytes'; import { itemIsEncrypted } from '../utils/joplinUtils'; +import { getMaxItemSize, getMaxTotalItemSize } from './utils/user'; export enum AccountType { Default = 0, @@ -13,10 +14,11 @@ export enum AccountType { Pro = 2, } -interface AccountTypeProperties { +export interface Account { account_type: number; can_share_folder: number; max_item_size: number; + max_total_item_size: number; } interface AccountTypeSelectOptions { @@ -24,22 +26,25 @@ interface AccountTypeSelectOptions { label: string; } -export function accountTypeProperties(accountType: AccountType): AccountTypeProperties { - const types: AccountTypeProperties[] = [ +export function accountByType(accountType: AccountType): Account { + const types: Account[] = [ { account_type: AccountType.Default, can_share_folder: 1, max_item_size: 0, + max_total_item_size: 0, }, { account_type: AccountType.Basic, can_share_folder: 0, max_item_size: 10 * MB, + max_total_item_size: 1 * GB, }, { account_type: AccountType.Pro, can_share_folder: 1, max_item_size: 200 * MB, + max_total_item_size: 10 * GB, }, ]; @@ -99,6 +104,7 @@ export default class UserModel extends BaseModel { if ('is_admin' in object) user.is_admin = object.is_admin; if ('full_name' in object) user.full_name = object.full_name; if ('max_item_size' in object) user.max_item_size = object.max_item_size; + if ('max_total_item_size' in object) user.max_total_item_size = object.max_total_item_size; if ('can_share_folder' in object) user.can_share_folder = object.can_share_folder; if ('account_type' in object) user.account_type = object.account_type; if ('must_set_password' in object) user.must_set_password = object.must_set_password; @@ -128,7 +134,10 @@ export default class UserModel extends BaseModel { if (!user.is_admin && resource.id !== user.id) throw new ErrorForbidden('non-admin user cannot modify another user'); if (!user.is_admin && 'is_admin' in resource) throw new ErrorForbidden('non-admin user cannot make themselves an admin'); if (user.is_admin && user.id === resource.id && 'is_admin' in resource && !resource.is_admin) throw new ErrorForbidden('admin user cannot make themselves a non-admin'); + + // TODO: Maybe define a whitelist of properties that can be changed if ('max_item_size' in resource && !user.is_admin && resource.max_item_size !== previousResource.max_item_size) throw new ErrorForbidden('non-admin user cannot change max_item_size'); + if ('max_total_item_size' in resource && !user.is_admin && resource.max_total_item_size !== previousResource.max_total_item_size) throw new ErrorForbidden('non-admin user cannot change max_total_item_size'); if ('can_share_folder' in resource && !user.is_admin && resource.can_share_folder !== previousResource.can_share_folder) throw new ErrorForbidden('non-admin user cannot change can_share_folder'); if ('account_type' in resource && !user.is_admin && resource.account_type !== previousResource.account_type) throw new ErrorForbidden('non-admin user cannot change account_type'); if ('must_set_password' in resource && !user.is_admin && resource.must_set_password !== previousResource.must_set_password) throw new ErrorForbidden('non-admin user cannot change must_set_password'); @@ -149,37 +158,31 @@ export default class UserModel extends BaseModel { // items can be much larger (seems to be up to twice the size but for // safety let's go with 2.2). - const maxSize = user.max_item_size * (itemIsEncrypted(item) ? 2.2 : 1); - if (maxSize && buffer.byteLength > maxSize) { - const itemTitle = joplinItem ? joplinItem.title || '' : ''; - const isNote = joplinItem && joplinItem.type_ === ModelType.Note; + const itemSize = buffer.byteLength; + const itemTitle = joplinItem ? joplinItem.title || '' : ''; + const isNote = joplinItem && joplinItem.type_ === ModelType.Note; + const maxItemSize = getMaxItemSize(user); + const maxSize = maxItemSize * (itemIsEncrypted(item) ? 2.2 : 1); + if (maxSize && itemSize > maxSize) { throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it is larger than than the allowed limit (%s)', isNote ? _('note') : _('attachment'), itemTitle ? itemTitle : item.name, - formatBytes(user.max_item_size) + formatBytes(maxItemSize) + )); + } + + // Also apply a multiplier to take into account E2EE overhead + const maxTotalItemSize = getMaxTotalItemSize(user) * 1.5; + if (maxTotalItemSize && user.total_item_size + itemSize >= maxTotalItemSize) { + throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it would go over the total allowed size (%s) for this account', + isNote ? _('note') : _('attachment'), + itemTitle ? itemTitle : item.name, + formatBytes(maxTotalItemSize) )); } } - // public async checkCanShare(share:Share) { - - // // const itemTitle = joplinItem ? joplinItem.title || '' : ''; - // // const isNote = joplinItem && joplinItem.type_ === ModelType.Note; - - // // // If the item is encrypted, we apply a multipler because encrypted - // // // items can be much larger (seems to be up to twice the size but for - // // // safety let's go with 2.2). - // // const maxSize = user.max_item_size * (item.jop_encryption_applied ? 2.2 : 1); - // // if (maxSize && buffer.byteLength > maxSize) { - // // throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it is larger than than the allowed limit (%s)', - // // isNote ? _('note') : _('attachment'), - // // itemTitle ? itemTitle : name, - // // prettyBytes(user.max_item_size) - // // )); - // // } - // } - protected async validate(object: User, options: ValidateOptions = {}): Promise { const user: User = await super.validate(object, options); diff --git a/packages/server/src/models/utils/user.ts b/packages/server/src/models/utils/user.ts new file mode 100644 index 0000000000..33d22a7555 --- /dev/null +++ b/packages/server/src/models/utils/user.ts @@ -0,0 +1,20 @@ +import { User } from '../../db'; +import { accountByType } from '../UserModel'; + +export function getCanShareFolder(user: User): number { + if (!('account_type' in user) || !('can_share_folder' in user)) throw new Error('Missing account_type or can_share_folder property'); + const account = accountByType(user.account_type); + return user.can_share_folder !== null ? user.can_share_folder : account.can_share_folder; +} + +export function getMaxItemSize(user: User): number { + if (!('account_type' in user) || !('max_item_size' in user)) throw new Error('Missing account_type or max_item_size property'); + const account = accountByType(user.account_type); + return user.max_item_size !== null ? user.max_item_size : account.max_item_size; +} + +export function getMaxTotalItemSize(user: User): number { + if (!('account_type' in user) || !('max_total_item_size' in user)) throw new Error('Missing account_type or max_total_item_size property'); + const account = accountByType(user.account_type); + return user.max_total_item_size !== null ? user.max_total_item_size : account.max_total_item_size; +} diff --git a/packages/server/src/routes/api/items.test.ts b/packages/server/src/routes/api/items.test.ts index 761d2541ea..aa55d90d5e 100644 --- a/packages/server/src/routes/api/items.test.ts +++ b/packages/server/src/routes/api/items.test.ts @@ -327,6 +327,44 @@ describe('api_items', function() { } }); + test('should check permissions - uploaded item should not make the account go over the allowed max limit', async function() { + const { user: user1, session: session1 } = await createUserAndSession(1); + + { + await models().user().save({ id: user1.id, max_total_item_size: 4 }); + + await expectHttpError( + async () => createNote(session1.id, { + id: '00000000000000000000000000000001', + body: '12345', + }), + ErrorPayloadTooLarge.httpCode + ); + } + + { + await models().user().save({ id: user1.id, max_total_item_size: 1000 }); + + await expectNoHttpError( + async () => createNote(session1.id, { + id: '00000000000000000000000000000002', + body: '12345', + }) + ); + } + + { + await models().user().save({ id: user1.id, max_total_item_size: 0 }); + + await expectNoHttpError( + async () => createNote(session1.id, { + id: '00000000000000000000000000000003', + body: '12345', + }) + ); + } + }); + test('should check permissions - should not allow uploading items if disabled', async function() { const { user: user1, session: session1 } = await createUserAndSession(1); diff --git a/packages/server/src/routes/index/home.ts b/packages/server/src/routes/index/home.ts index 6554b94680..9ef6684921 100644 --- a/packages/server/src/routes/index/home.ts +++ b/packages/server/src/routes/index/home.ts @@ -5,9 +5,8 @@ import { AppContext } from '../../utils/types'; import { contextSessionId } from '../../utils/requestUtils'; import { ErrorMethodNotAllowed } from '../../utils/errors'; import defaultView from '../../utils/defaultView'; -import { accountTypeProperties, accountTypeToString } from '../../models/UserModel'; -import { formatBytes } from '../../utils/bytes'; -import { yesOrNo } from '../../utils/strings'; +import { accountByType, accountTypeToString } from '../../models/UserModel'; +import { formatMaxItemSize, yesOrNo } from '../../utils/strings'; const router: Router = new Router(RouteType.Web); @@ -15,7 +14,7 @@ router.get('home', async (_path: SubPath, ctx: AppContext) => { contextSessionId(ctx); if (ctx.method === 'GET') { - const accountProps = accountTypeProperties(ctx.joplin.owner.account_type); + const accountProps = accountByType(ctx.joplin.owner.account_type); const view = defaultView('home', 'Home'); view.content = { @@ -30,7 +29,7 @@ router.get('home', async (_path: SubPath, ctx: AppContext) => { }, { label: 'Max Item Size', - value: accountProps.max_item_size ? formatBytes(accountProps.max_item_size) : '∞', + value: formatMaxItemSize(ctx.joplin.owner), }, { label: 'Can Share Note', diff --git a/packages/server/src/routes/index/signup.test.ts b/packages/server/src/routes/index/signup.test.ts index 47290137e5..870215365b 100644 --- a/packages/server/src/routes/index/signup.test.ts +++ b/packages/server/src/routes/index/signup.test.ts @@ -1,6 +1,7 @@ import config from '../../config'; import { NotificationKey } from '../../models/NotificationModel'; import { AccountType } from '../../models/UserModel'; +import { getCanShareFolder, getMaxItemSize } from '../../models/utils/user'; import { MB } from '../../utils/bytes'; import { execRequestC } from '../../utils/testing/apiUtils'; import { beforeAllDb, afterAllTests, beforeEachDb, models } from '../../utils/testing/testUtils'; @@ -43,8 +44,8 @@ describe('index_signup', function() { expect(user).toBeTruthy(); expect(user.account_type).toBe(AccountType.Basic); expect(user.email_confirmed).toBe(0); - expect(user.can_share_folder).toBe(0); - expect(user.max_item_size).toBe(10 * MB); + expect(getCanShareFolder(user)).toBe(0); + expect(getMaxItemSize(user)).toBe(10 * MB); // Check that the user is logged in const session = await models().session().load(context.cookies.get('sessionId')); diff --git a/packages/server/src/routes/index/signup.ts b/packages/server/src/routes/index/signup.ts index c5dd116c96..04e410f84b 100644 --- a/packages/server/src/routes/index/signup.ts +++ b/packages/server/src/routes/index/signup.ts @@ -8,7 +8,7 @@ import defaultView from '../../utils/defaultView'; import { View } from '../../services/MustacheService'; import { checkPassword } from './users'; import { NotificationKey } from '../../models/NotificationModel'; -import { AccountType, accountTypeProperties } from '../../models/UserModel'; +import { AccountType } from '../../models/UserModel'; import { ErrorForbidden } from '../../utils/errors'; function makeView(error: Error = null): View { @@ -44,7 +44,7 @@ router.post('signup', async (_path: SubPath, ctx: AppContext) => { const password = checkPassword(formUser, true); const user = await ctx.joplin.models.user().save({ - ...accountTypeProperties(AccountType.Basic), + account_type: AccountType.Basic, email: formUser.email, full_name: formUser.full_name, password, diff --git a/packages/server/src/routes/index/users.test.ts b/packages/server/src/routes/index/users.test.ts index c39732dc4e..f15af964a3 100644 --- a/packages/server/src/routes/index/users.test.ts +++ b/packages/server/src/routes/index/users.test.ts @@ -58,7 +58,7 @@ export async function getUserHtml(sessionId: string, userId: string): Promise { await beforeAllDb('index_users'); @@ -84,7 +84,7 @@ describe('index_users', function() { expect(!!newUser.id).toBe(true); expect(!!newUser.is_admin).toBe(false); expect(!!newUser.email).toBe(true); - expect(newUser.max_item_size).toBe(0); + expect(newUser.max_item_size).toBe(null); expect(newUser.must_set_password).toBe(0); const userModel = models().user(); @@ -94,6 +94,18 @@ describe('index_users', function() { expect(userFromModel.password === '123456').toBe(false); // Password has been hashed }); + test('should create a user with null properties if they are not explicitly set', async function() { + const { session } = await createUserAndSession(1, true); + + await postUser(session.id, 'test@example.com', '123456'); + const newUser = await models().user().loadByEmail('test@example.com'); + + expect(newUser.max_item_size).toBe(null); + expect(newUser.can_share_folder).toBe(null); + expect(newUser.can_share_note).toBe(null); + expect(newUser.max_total_item_size).toBe(null); + }); + test('should ask user to set password if not set on creation', async function() { const { session } = await createUserAndSession(1, true); diff --git a/packages/server/src/routes/index/users.ts b/packages/server/src/routes/index/users.ts index b7de1b5d0b..2a8e98520b 100644 --- a/packages/server/src/routes/index/users.ts +++ b/packages/server/src/routes/index/users.ts @@ -10,9 +10,11 @@ import { View } from '../../services/MustacheService'; import defaultView from '../../utils/defaultView'; import { AclAction } from '../../models/BaseModel'; import { NotificationKey } from '../../models/NotificationModel'; -import { formatBytes } from '../../utils/bytes'; -import { accountTypeOptions, accountTypeProperties } from '../../models/UserModel'; +import { accountTypeOptions, accountTypeToString } from '../../models/UserModel'; import uuidgen from '../../utils/uuidgen'; +import { formatMaxItemSize, formatMaxTotalSize, formatTotalSize, yesOrNo } from '../../utils/strings'; +import { getCanShareFolder, getMaxTotalItemSize } from '../../models/utils/user'; +import { yesNoDefaultOptions } from '../../utils/views/select'; interface CheckPasswordInput { password: string; @@ -30,22 +32,30 @@ export function checkPassword(fields: CheckPasswordInput, required: boolean): st return ''; } +function boolOrDefaultToValue(fields: any, fieldName: string): number | null { + if (fields[fieldName] === '') return null; + const output = Number(fields[fieldName]); + if (isNaN(output) || (output !== 0 && output !== 1)) throw new Error(`Invalid value for ${fieldName}`); + return output; +} + +function intOrDefaultToValue(fields: any, fieldName: string): number | null { + if (fields[fieldName] === '') return null; + const output = Number(fields[fieldName]); + if (isNaN(output)) throw new Error(`Invalid value for ${fieldName}`); + return output; +} + function makeUser(isNew: boolean, fields: any): User { - let user: User = {}; + const user: User = {}; if ('email' in fields) user.email = fields.email; if ('full_name' in fields) user.full_name = fields.full_name; if ('is_admin' in fields) user.is_admin = fields.is_admin; - if ('max_item_size' in fields) user.max_item_size = fields.max_item_size || 0; - if ('can_share_folder' in fields) user.can_share_folder = fields.can_share_folder ? 1 : 0; - - if ('account_type' in fields) { - user.account_type = Number(fields.account_type); - user = { - ...user, - ...accountTypeProperties(user.account_type), - }; - } + if ('max_item_size' in fields) user.max_item_size = intOrDefaultToValue(fields, 'max_item_size'); + if ('max_total_item_size' in fields) user.max_total_item_size = intOrDefaultToValue(fields, 'max_total_item_size'); + if ('can_share_folder' in fields) user.can_share_folder = boolOrDefaultToValue(fields, 'can_share_folder'); + if ('account_type' in fields) user.account_type = Number(fields.account_type); const password = checkPassword(fields, false); if (password) user.password = password; @@ -61,10 +71,7 @@ function makeUser(isNew: boolean, fields: any): User { } function defaultUser(): User { - return { - can_share_folder: 1, - max_item_size: 0, - }; + return {}; } function userIsNew(path: SubPath): boolean { @@ -77,6 +84,19 @@ function userIsMe(path: SubPath): boolean { const router = new Router(RouteType.Web); +function totalSizePercent(user: User): number { + const maxTotalSize = getMaxTotalItemSize(user); + if (!maxTotalSize) return 0; + return user.total_item_size / maxTotalSize; +} + +function totalSizeClass(user: User) { + const d = totalSizePercent(user); + if (d >= 1) return 'is-danger'; + if (d >= .7) return 'is-warning'; + return ''; +} + router.get('users', async (_path: SubPath, ctx: AppContext) => { const userModel = ctx.joplin.models.user(); await userModel.checkIfAllowed(ctx.joplin.owner, AclAction.List); @@ -94,7 +114,13 @@ router.get('users', async (_path: SubPath, ctx: AppContext) => { view.content.users = users.map(user => { return { ...user, - formattedItemMaxSize: user.max_item_size ? formatBytes(user.max_item_size) : '∞', + formattedItemMaxSize: formatMaxItemSize(user), + formattedTotalSize: formatTotalSize(user), + formattedMaxTotalSize: formatMaxTotalSize(user), + formattedTotalSizePercent: `${Math.round(totalSizePercent(user) * 100)}%`, + totalSizeClass: totalSizeClass(user), + formattedAccountType: accountTypeToString(user.account_type), + formattedCanShareFolder: yesOrNo(getCanShareFolder(user)), }; }); return view; @@ -131,6 +157,8 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null view.content.showDeleteButton = !isNew && !!owner.is_admin && owner.id !== user.id; view.content.showResetPasswordButton = !isNew && owner.is_admin; + view.content.canShareFolderOptions = yesNoDefaultOptions(user, 'can_share_folder'); + if (config().accountTypesEnabled) { view.content.showAccountTypes = true; view.content.accountTypes = accountTypeOptions().map((o: any) => { diff --git a/packages/server/src/services/CronService.ts b/packages/server/src/services/CronService.ts index 53a08eef26..c1d5ee602d 100644 --- a/packages/server/src/services/CronService.ts +++ b/packages/server/src/services/CronService.ts @@ -1,11 +1,26 @@ +import Logger from '@joplin/lib/Logger'; import BaseService from './BaseService'; const cron = require('node-cron'); +const logger = Logger.create('cron'); + export default class CronService extends BaseService { public async runInBackground() { + await this.models.item().updateTotalSizes(); + cron.schedule('0 */6 * * *', async () => { + const startTime = Date.now(); + logger.info('Deleting expired tokens...'); await this.models.token().deleteExpiredTokens(); + logger.info(`Expired tokens deleted in ${Date.now() - startTime}ms`); + }); + + cron.schedule('0 * * * *', async () => { + const startTime = Date.now(); + logger.info('Updating total sizes...'); + await this.models.item().updateTotalSizes(); + logger.info(`Total sizes updated in ${Date.now() - startTime}ms`); }); } diff --git a/packages/server/src/tools/generateTypes.ts b/packages/server/src/tools/generateTypes.ts index 9a9719272c..4983952f7a 100644 --- a/packages/server/src/tools/generateTypes.ts +++ b/packages/server/src/tools/generateTypes.ts @@ -48,6 +48,10 @@ const propertyTypes: Record = { 'emails.sent_time': 'number', 'subscriptions.last_payment_time': 'number', 'subscriptions.last_payment_failed_time': 'number', + 'users.can_share_folder': 'number | null', + 'users.can_share_note': 'number | null', + 'users.max_total_item_size': 'number | null', + 'users.max_item_size': 'number | null', }; function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void { diff --git a/packages/server/src/utils/strings.ts b/packages/server/src/utils/strings.ts index 995c643004..0608ff1826 100644 --- a/packages/server/src/utils/strings.ts +++ b/packages/server/src/utils/strings.ts @@ -1,3 +1,7 @@ +import { User } from '../db'; +import { getMaxItemSize, getMaxTotalItemSize } from '../models/utils/user'; +import { formatBytes } from './bytes'; + export function yesOrNo(value: any): string { return value ? 'yes' : 'no'; } @@ -5,3 +9,17 @@ export function yesOrNo(value: any): string { export function nothing() { return ''; } + +export function formatMaxItemSize(user: User): string { + const size = getMaxItemSize(user); + return size ? formatBytes(size) : '∞'; +} + +export function formatMaxTotalSize(user: User): string { + const size = getMaxTotalItemSize(user); + return size ? formatBytes(size) : '∞'; +} + +export function formatTotalSize(user: User): string { + return formatBytes(user.total_item_size); +} diff --git a/packages/server/src/utils/types.ts b/packages/server/src/utils/types.ts index 20716325c9..21120a30c5 100644 --- a/packages/server/src/utils/types.ts +++ b/packages/server/src/utils/types.ts @@ -2,6 +2,7 @@ import { LoggerWrapper } from '@joplin/lib/Logger'; import * as Koa from 'koa'; import { DbConnection, User, Uuid } from '../db'; import { Models } from '../models/factory'; +import { Account } from '../models/UserModel'; import { Services } from '../services/types'; import { Routers } from './routeUtils'; @@ -25,6 +26,7 @@ interface AppContextJoplin { appLogger(): LoggerWrapper; notifications: NotificationView[]; owner: User; + account: Account; routes: Routers; services: Services; } diff --git a/packages/server/src/utils/views/select.ts b/packages/server/src/utils/views/select.ts new file mode 100644 index 0000000000..b23f65b7e6 --- /dev/null +++ b/packages/server/src/utils/views/select.ts @@ -0,0 +1,36 @@ +interface Option { + value: any; + label: string; + selected: boolean; +} + +type LabelFn = (key: string, value: any)=> string; + +export function yesNoDefaultLabel(_key: string, value: any): string { + if (value === '') return 'Default'; + return value ? 'Yes' : 'No'; +} + +export function objectToSelectOptions(object: any, selectedValue: any, labelFn: LabelFn): Option[] { + const output: Option[] = []; + for (const [key, value] of Object.entries(object)) { + output.push({ + label: labelFn(key, value), + selected: value === selectedValue, + value: value, + }); + } + return output; +} + +export function selectOption(label: string, value: any, selected: boolean): Option { + return { label, value, selected }; +} + +export function yesNoDefaultOptions(object: any, key: string): Option[] { + return [ + selectOption('Default', '', object[key] === null), + selectOption('Yes', '1', object[key] === 1), + selectOption('No', '0', object[key] === 0), + ]; +} diff --git a/packages/server/src/views/index/user.mustache b/packages/server/src/views/index/user.mustache index ff16263d1e..db69e68e07 100644 --- a/packages/server/src/views/index/user.mustache +++ b/packages/server/src/views/index/user.mustache @@ -14,6 +14,7 @@ + {{#global.owner.is_admin}} {{#showAccountTypes}}
@@ -25,23 +26,36 @@ {{/accountTypes}}
-

If the account type is anything other than Default, the account-specific properties will be applied.

+

If the below properties are left to their default (empty) values, the account-specific properties will apply.

{{/showAccountTypes}}
- +
- + +
+ +
+
+ +
+ +
+ +
{{/global.owner.is_admin}} +
diff --git a/packages/server/src/views/index/users.mustache b/packages/server/src/views/index/users.mustache index b17f8015fd..63509e19b4 100644 --- a/packages/server/src/views/index/users.mustache +++ b/packages/server/src/views/index/users.mustache @@ -1,9 +1,16 @@ +
+ Add user +
+ + + + @@ -14,8 +21,11 @@ + - + + +
Full name EmailAccount Max Item SizeTotal SizeMax Total Size Can share Is admin? Actions
{{full_name}} {{email}}{{formattedAccountType}} {{formattedItemMaxSize}}{{can_share_folder}}{{formattedTotalSize}} ({{formattedTotalSizePercent}}){{formattedMaxTotalSize}}{{formattedCanShareFolder}} {{is_admin}} Edit