Server: Add support for account max total size

packages/server/src/app.ts

@@ -208,7 +208,7 @@ async function main() {
 		await initializeJoplinUtils(config(), ctx.models, ctx.joplin.services.mustache);
 
 		appLogger().info('Migrating database...');
-		await migrateDb(ctx.db);
+		await migrateDb(ctx.joplin.db);
 
 		appLogger().info('Starting services...');
 		await startServices(ctx);

packages/server/src/db.ts

@@ -291,20 +291,6 @@ interface DatabaseTables {
 
 // AUTO-GENERATED-TYPES
 // Auto-generated using `npm run generate-types`
-export interface User extends WithDates, WithUuid {
-	email?: string;
-	password?: string;
-	full_name?: string;
-	is_admin?: number;
-	max_item_size?: number;
-	can_share_folder?: number;
-	email_confirmed?: number;
-	must_set_password?: number;
-	account_type?: number;
-	can_upload?: number;
-	can_share_note?: number;
-}
-
 export interface Session extends WithDates, WithUuid {
 	user_id?: Uuid;
 	auth_code?: string;
@@ -427,23 +413,23 @@ export interface Subscription {
 	created_time?: string;
 }
 
+export interface User extends WithDates, WithUuid {
+	email?: string;
+	password?: string;
+	full_name?: string;
+	is_admin?: number;
+	email_confirmed?: number;
+	must_set_password?: number;
+	account_type?: number;
+	can_upload?: number;
+	max_item_size?: number | null;
+	can_share_folder?: number | null;
+	can_share_note?: number | null;
+	max_total_item_size?: number | null;
+	total_item_size?: number;
+}
+
 export const databaseSchema: DatabaseTables = {
-	users: {
-		id: { type: 'string' },
-		email: { type: 'string' },
-		password: { type: 'string' },
-		full_name: { type: 'string' },
-		is_admin: { type: 'number' },
-		updated_time: { type: 'string' },
-		created_time: { type: 'string' },
-		max_item_size: { type: 'number' },
-		can_share_folder: { type: 'number' },
-		email_confirmed: { type: 'number' },
-		must_set_password: { type: 'number' },
-		account_type: { type: 'number' },
-		can_upload: { type: 'number' },
-		can_share_note: { type: 'number' },
-	},
 	sessions: {
 		id: { type: 'string' },
 		user_id: { type: 'string' },
@@ -579,5 +565,23 @@ export const databaseSchema: DatabaseTables = {
 		updated_time: { type: 'string' },
 		created_time: { type: 'string' },
 	},
+	users: {
+		id: { type: 'string' },
+		email: { type: 'string' },
+		password: { type: 'string' },
+		full_name: { type: 'string' },
+		is_admin: { type: 'number' },
+		updated_time: { type: 'string' },
+		created_time: { type: 'string' },
+		email_confirmed: { type: 'number' },
+		must_set_password: { type: 'number' },
+		account_type: { type: 'number' },
+		can_upload: { type: 'number' },
+		max_item_size: { type: 'number' },
+		can_share_folder: { type: 'number' },
+		can_share_note: { type: 'number' },
+		max_total_item_size: { type: 'string' },
+		total_item_size: { type: 'number' },
+	},
 };
 // AUTO-GENERATED-TYPES

packages/server/src/migrations/20210702182439_account_max_total_size.ts

@@ -0,0 +1,18 @@
+import { Knex } from 'knex';
+import { DbConnection } from '../db';
+
+// Due to a bug in Knex.js, it is not possible to drop a column and
+// recreate it in the same migration. So this is split into two migrations.
+// https://github.com/knex/knex/issues/2581
+
+export async function up(db: DbConnection): Promise<any> {
+	await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) {
+		table.dropColumn('max_item_size');
+		table.dropColumn('can_share_folder');
+		table.dropColumn('can_share_note');
+	});
+}
+
+export async function down(_db: DbConnection): Promise<any> {
+
+}

packages/server/src/migrations/20210702182440_account_max_total_size_2.ts

@@ -0,0 +1,19 @@
+import { Knex } from 'knex';
+import { DbConnection } from '../db';
+
+export async function up(db: DbConnection): Promise<any> {
+	await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) {
+		table.integer('max_item_size').defaultTo(null).nullable();
+		table.specificType('can_share_folder', 'smallint').defaultTo(null).nullable();
+		table.specificType('can_share_note', 'smallint').defaultTo(null).nullable();
+		table.bigInteger('max_total_item_size').defaultTo(null).nullable();
+	});
+
+	await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) {
+		table.integer('total_item_size').defaultTo(0).notNullable();
+	});
+}
+
+export async function down(_db: DbConnection): Promise<any> {
+
+}

packages/server/src/models/ItemModel.test.ts

@@ -1,4 +1,4 @@
-import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder } from '../utils/testing/testUtils';
+import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItem, createItemTree, createResource, createNote, createFolder, createItemTree3 } from '../utils/testing/testUtils';
 import { shareFolderWithUser } from '../utils/testing/shareApiUtils';
 import { resourceBlobPath } from '../utils/joplinUtils';
 
@@ -142,4 +142,99 @@ describe('ItemModel', function() {
 		expect(await models().item().childrenCount(user1.id)).toBe(2);
 	});
 
+	test('should calculate the total size', async function() {
+		const { user: user1 } = await createUserAndSession(1);
+		const { user: user2 } = await createUserAndSession(2);
+		const { user: user3 } = await createUserAndSession(3);
+
+		await createItemTree3(user1.id, '', '', [
+			{
+				id: '000000000000000000000000000000F1',
+				children: [
+					{
+						id: '00000000000000000000000000000001',
+					},
+				],
+			},
+		]);
+
+		await createItemTree3(user2.id, '', '', [
+			{
+				id: '000000000000000000000000000000F2',
+				children: [
+					{
+						id: '00000000000000000000000000000002',
+					},
+					{
+						id: '00000000000000000000000000000003',
+					},
+				],
+			},
+		]);
+
+		const folder1 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F1');
+		const folder2 = await models().item().loadByJopId(user2.id, '000000000000000000000000000000F2');
+		const note1 = await models().item().loadByJopId(user1.id, '00000000000000000000000000000001');
+		const note2 = await models().item().loadByJopId(user2.id, '00000000000000000000000000000002');
+		const note3 = await models().item().loadByJopId(user2.id, '00000000000000000000000000000003');
+
+		const totalSize1 = await models().item().calculateUserTotalSize(user1.id);
+		const totalSize2 = await models().item().calculateUserTotalSize(user2.id);
+		const totalSize3 = await models().item().calculateUserTotalSize(user3.id);
+
+		const expected1 = folder1.content_size + note1.content_size;
+		const expected2 = folder2.content_size + note2.content_size + note3.content_size;
+		const expected3 = 0;
+
+		expect(totalSize1).toBe(expected1);
+		expect(totalSize2).toBe(expected2);
+		expect(totalSize3).toBe(expected3);
+
+		await models().item().updateTotalSizes();
+		expect((await models().user().load(user1.id)).total_item_size).toBe(totalSize1);
+		expect((await models().user().load(user2.id)).total_item_size).toBe(totalSize2);
+		expect((await models().user().load(user3.id)).total_item_size).toBe(totalSize3);
+	});
+
+	test('should include shared items in total size calculation', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+		const { user: user2, session: session2 } = await createUserAndSession(2);
+		const { user: user3 } = await createUserAndSession(3);
+
+		await shareFolderWithUser(session1.id, session2.id, '000000000000000000000000000000F1', [
+			{
+				id: '000000000000000000000000000000F1',
+				children: [
+					{
+						id: '00000000000000000000000000000001',
+					},
+				],
+			},
+			{
+				id: '000000000000000000000000000000F2',
+			},
+		]);
+
+		const folder1 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F1');
+		const folder2 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F2');
+		const note1 = await models().item().loadByJopId(user1.id, '00000000000000000000000000000001');
+
+		const totalSize1 = await models().item().calculateUserTotalSize(user1.id);
+		const totalSize2 = await models().item().calculateUserTotalSize(user2.id);
+		const totalSize3 = await models().item().calculateUserTotalSize(user3.id);
+
+		const expected1 = folder1.content_size + folder2.content_size + note1.content_size;
+		const expected2 = folder1.content_size + note1.content_size;
+		const expected3 = 0;
+
+		expect(totalSize1).toBe(expected1);
+		expect(totalSize2).toBe(expected2);
+		expect(totalSize3).toBe(expected3);
+
+		await models().item().updateTotalSizes();
+		expect((await models().user().load(user1.id)).total_item_size).toBe(expected1);
+		expect((await models().user().load(user2.id)).total_item_size).toBe(expected2);
+		expect((await models().user().load(user3.id)).total_item_size).toBe(expected3);
+	});
+
 });

packages/server/src/models/ItemModel.ts

@@ -1,11 +1,12 @@
 import BaseModel, { SaveOptions, LoadOptions, DeleteOptions, ValidateOptions, AclAction } from './BaseModel';
-import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User } from '../db';
+import { ItemType, databaseSchema, Uuid, Item, ShareType, Share, ChangeType, User, UserItem } from '../db';
 import { defaultPagination, paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination';
 import { isJoplinItemName, isJoplinResourceBlobPath, linkedResourceIds, serializeJoplinItem, unserializeJoplinItem } from '../utils/joplinUtils';
 import { ModelType } from '@joplin/lib/BaseModel';
 import { ApiError, ErrorForbidden, ErrorUnprocessableEntity } from '../utils/errors';
 import { Knex } from 'knex';
 import { ChangePreviousItem } from './ChangeModel';
+import { unique } from '../utils/array';
 
 const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
 
@@ -39,6 +40,8 @@ export interface ItemSaveOption extends SaveOptions {
 
 export default class ItemModel extends BaseModel<Item> {
 
+	private updatingTotalSizes_: boolean = false;
+
 	protected get tableName(): string {
 		return 'items';
 	}
@@ -591,6 +594,69 @@ export default class ItemModel extends BaseModel<Item> {
 		}, 'ItemModel::saveForUser');
 	}
 
+	public async updateTotalSizes(): Promise<void> {
+		interface TotalSizeRow {
+			userId: Uuid;
+			totalSize: number;
+		}
+
+		// Total sizes are updated once an hour, so unless there's something
+		// very wrong this error shouldn't happen.
+		if (this.updatingTotalSizes_) throw new Error('Already updating total sizes');
+
+		this.updatingTotalSizes_ = true;
+
+		const doneUserIds: Record<Uuid, boolean> = {};
+
+		try {
+			while (true) {
+				const latestProcessedChange = await this.models().keyValue().value<string>('ItemModel::updateTotalSizes::latestProcessedChange');
+
+				const changes = await this.models().change().allFromId(latestProcessedChange || '');
+				if (!changes.length) break;
+
+				const itemIds: Uuid[] = unique(changes.map(c => c.item_id));
+				const userItems: UserItem[] = await this.db('user_items').select('user_id').whereIn('item_id', itemIds);
+				const userIds: Uuid[] = unique(userItems.map(u => u.user_id));
+
+				const totalSizes: TotalSizeRow[] = [];
+				for (const userId of userIds) {
+					if (doneUserIds[userId]) continue;
+
+					totalSizes.push({
+						userId,
+						totalSize: await this.calculateUserTotalSize(userId),
+					});
+
+					doneUserIds[userId] = true;
+				}
+
+				await this.withTransaction(async () => {
+					for (const row of totalSizes) {
+						await this.models().user().save({
+							id: row.userId,
+							total_item_size: row.totalSize,
+						});
+					}
+
+					await this.models().keyValue().setValue('ItemModel::updateTotalSizes::latestProcessedChange', changes[changes.length - 1].id);
+				}, 'ItemModel::updateTotalSizes');
+			}
+		} finally {
+			this.updatingTotalSizes_ = false;
+		}
+	}
+
+	public async calculateUserTotalSize(userId: Uuid): Promise<number> {
+		const result = await this.db('items')
+			.sum('items.content_size', { as: 'total' })
+			.leftJoin('user_items', 'items.id', 'user_items.item_id')
+			.where('user_items.user_id', userId)
+			.first();
+
+		return result && result.total ? result.total : 0;
+	}
+
 	public async save(_item: Item, _options: SaveOptions = {}): Promise<Item> {
 		throw new Error('Use saveForUser()');
 		// return this.saveForUser('', item, options);

packages/server/src/models/ShareModel.ts

@@ -6,6 +6,7 @@ import { ErrorBadRequest, ErrorForbidden, ErrorNotFound } from '../utils/errors'
 import { setQueryParameters } from '../utils/urlUtils';
 import BaseModel, { AclAction, DeleteOptions, ValidateOptions } from './BaseModel';
 import { userIdFromUserContentUrl } from '../utils/routeUtils';
+import { getCanShareFolder } from './utils/user';
 
 export default class ShareModel extends BaseModel<Share> {
 
@@ -15,7 +16,7 @@ export default class ShareModel extends BaseModel<Share> {
 
 	public async checkIfAllowed(user: User, action: AclAction, resource: Share = null): Promise<void> {
 		if (action === AclAction.Create) {
-			if (resource.type === ShareType.Folder && !user.can_share_folder) throw new ErrorForbidden('The sharing feature is not enabled for this account');
+			if (resource.type === ShareType.Folder && !getCanShareFolder(user)) throw new ErrorForbidden('The sharing feature is not enabled for this account');
 
 			// Note that currently all users can always share notes by URL so
 			// there's no check on the permission

packages/server/src/models/ShareUserModel.ts

@@ -1,6 +1,7 @@
 import { Item, Share, ShareType, ShareUser, ShareUserStatus, User, Uuid } from '../db';
 import { ErrorForbidden, ErrorNotFound } from '../utils/errors';
 import BaseModel, { AclAction, DeleteOptions } from './BaseModel';
+import { getCanShareFolder } from './utils/user';
 
 export default class ShareUserModel extends BaseModel<ShareUser> {
 
@@ -10,8 +11,8 @@ export default class ShareUserModel extends BaseModel<ShareUser> {
 
 	public async checkIfAllowed(user: User, action: AclAction, resource: ShareUser = null): Promise<void> {
 		if (action === AclAction.Create) {
-			const recipient = await this.models().user().load(resource.user_id, { fields: ['can_share_folder'] });
-			if (!recipient.can_share_folder) throw new ErrorForbidden('The sharing feature is not enabled for the recipient account');
+			const recipient = await this.models().user().load(resource.user_id, { fields: ['account_type', 'can_share_folder'] });
+			if (!getCanShareFolder(recipient)) throw new ErrorForbidden('The sharing feature is not enabled for the recipient account');
 
 			const share = await this.models().share().load(resource.share_id);
 			if (share.owner_id !== user.id) throw new ErrorForbidden('no access to the share object');

packages/server/src/models/SubscriptionModel.test.ts

@@ -1,6 +1,7 @@
 import { beforeAllDb, afterAllTests, beforeEachDb, models } from '../utils/testing/testUtils';
 import { AccountType } from './UserModel';
 import { MB } from '../utils/bytes';
+import { getCanShareFolder, getMaxItemSize } from './utils/user';
 
 describe('SubscriptionModel', function() {
 
@@ -29,8 +30,8 @@ describe('SubscriptionModel', function() {
 
 		expect(user.account_type).toBe(AccountType.Pro);
 		expect(user.email).toBe('toto@example.com');
-		expect(user.can_share_folder).toBe(1);
-		expect(user.max_item_size).toBe(200 * MB);
+		expect(getCanShareFolder(user)).toBe(1);
+		expect(getMaxItemSize(user)).toBe(200 * MB);
 
 		expect(sub.stripe_subscription_id).toBe('STRIPE_SUB_ID');
 		expect(sub.stripe_user_id).toBe('STRIPE_USER_ID');

packages/server/src/models/SubscriptionModel.ts

@@ -2,7 +2,7 @@ import { EmailSender, Subscription, Uuid } from '../db';
 import { ErrorNotFound } from '../utils/errors';
 import uuidgen from '../utils/uuidgen';
 import BaseModel from './BaseModel';
-import { AccountType, accountTypeProperties } from './UserModel';
+import { AccountType } from './UserModel';
 
 export default class SubscriptionModel extends BaseModel<Subscription> {
 
@@ -51,7 +51,7 @@ export default class SubscriptionModel extends BaseModel<Subscription> {
 	public async saveUserAndSubscription(email: string, accountType: AccountType, stripeUserId: string, stripeSubscriptionId: string) {
 		return this.withTransaction(async () => {
 			const user = await this.models().user().save({
-				...accountTypeProperties(accountType),
+				account_type: accountType,
 				email,
 				email_confirmed: 1,
 				password: uuidgen(),

packages/server/src/models/UserModel.ts

@@ -4,8 +4,9 @@ import * as auth from '../utils/auth';
 import { ErrorUnprocessableEntity, ErrorForbidden, ErrorPayloadTooLarge, ErrorNotFound } from '../utils/errors';
 import { ModelType } from '@joplin/lib/BaseModel';
 import { _ } from '@joplin/lib/locale';
-import { formatBytes, MB } from '../utils/bytes';
+import { formatBytes, GB, MB } from '../utils/bytes';
 import { itemIsEncrypted } from '../utils/joplinUtils';
+import { getMaxItemSize, getMaxTotalItemSize } from './utils/user';
 
 export enum AccountType {
 	Default = 0,
@@ -13,10 +14,11 @@ export enum AccountType {
 	Pro = 2,
 }
 
-interface AccountTypeProperties {
+export interface Account {
 	account_type: number;
 	can_share_folder: number;
 	max_item_size: number;
+	max_total_item_size: number;
 }
 
 interface AccountTypeSelectOptions {
@@ -24,22 +26,25 @@ interface AccountTypeSelectOptions {
 	label: string;
 }
 
-export function accountTypeProperties(accountType: AccountType): AccountTypeProperties {
-	const types: AccountTypeProperties[] = [
+export function accountByType(accountType: AccountType): Account {
+	const types: Account[] = [
 		{
 			account_type: AccountType.Default,
 			can_share_folder: 1,
 			max_item_size: 0,
+			max_total_item_size: 0,
 		},
 		{
 			account_type: AccountType.Basic,
 			can_share_folder: 0,
 			max_item_size: 10 * MB,
+			max_total_item_size: 1 * GB,
 		},
 		{
 			account_type: AccountType.Pro,
 			can_share_folder: 1,
 			max_item_size: 200 * MB,
+			max_total_item_size: 10 * GB,
 		},
 	];
 
@@ -99,6 +104,7 @@ export default class UserModel extends BaseModel<User> {
 		if ('is_admin' in object) user.is_admin = object.is_admin;
 		if ('full_name' in object) user.full_name = object.full_name;
 		if ('max_item_size' in object) user.max_item_size = object.max_item_size;
+		if ('max_total_item_size' in object) user.max_total_item_size = object.max_total_item_size;
 		if ('can_share_folder' in object) user.can_share_folder = object.can_share_folder;
 		if ('account_type' in object) user.account_type = object.account_type;
 		if ('must_set_password' in object) user.must_set_password = object.must_set_password;
@@ -128,7 +134,10 @@ export default class UserModel extends BaseModel<User> {
 			if (!user.is_admin && resource.id !== user.id) throw new ErrorForbidden('non-admin user cannot modify another user');
 			if (!user.is_admin && 'is_admin' in resource) throw new ErrorForbidden('non-admin user cannot make themselves an admin');
 			if (user.is_admin && user.id === resource.id && 'is_admin' in resource && !resource.is_admin) throw new ErrorForbidden('admin user cannot make themselves a non-admin');
+
+			// TODO: Maybe define a whitelist of properties that can be changed
 			if ('max_item_size' in resource && !user.is_admin && resource.max_item_size !== previousResource.max_item_size) throw new ErrorForbidden('non-admin user cannot change max_item_size');
+			if ('max_total_item_size' in resource && !user.is_admin && resource.max_total_item_size !== previousResource.max_total_item_size) throw new ErrorForbidden('non-admin user cannot change max_total_item_size');
 			if ('can_share_folder' in resource && !user.is_admin && resource.can_share_folder !== previousResource.can_share_folder) throw new ErrorForbidden('non-admin user cannot change can_share_folder');
 			if ('account_type' in resource && !user.is_admin && resource.account_type !== previousResource.account_type) throw new ErrorForbidden('non-admin user cannot change account_type');
 			if ('must_set_password' in resource && !user.is_admin && resource.must_set_password !== previousResource.must_set_password) throw new ErrorForbidden('non-admin user cannot change must_set_password');
@@ -149,37 +158,31 @@ export default class UserModel extends BaseModel<User> {
 		// items can be much larger (seems to be up to twice the size but for
 		// safety let's go with 2.2).
 
-		const maxSize = user.max_item_size * (itemIsEncrypted(item) ? 2.2 : 1);
-		if (maxSize && buffer.byteLength > maxSize) {
-			const itemTitle = joplinItem ? joplinItem.title || '' : '';
-			const isNote = joplinItem && joplinItem.type_ === ModelType.Note;
+		const itemSize = buffer.byteLength;
+		const itemTitle = joplinItem ? joplinItem.title || '' : '';
+		const isNote = joplinItem && joplinItem.type_ === ModelType.Note;
 
+		const maxItemSize = getMaxItemSize(user);
+		const maxSize = maxItemSize * (itemIsEncrypted(item) ? 2.2 : 1);
+		if (maxSize && itemSize > maxSize) {
 			throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it is larger than than the allowed limit (%s)',
 				isNote ? _('note') : _('attachment'),
 				itemTitle ? itemTitle : item.name,
-				formatBytes(user.max_item_size)
+				formatBytes(maxItemSize)
+			));
+		}
+
+		// Also apply a multiplier to take into account E2EE overhead
+		const maxTotalItemSize = getMaxTotalItemSize(user) * 1.5;
+		if (maxTotalItemSize && user.total_item_size + itemSize >= maxTotalItemSize) {
+			throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it would go over the total allowed size (%s) for this account',
+				isNote ? _('note') : _('attachment'),
+				itemTitle ? itemTitle : item.name,
+				formatBytes(maxTotalItemSize)
 			));
 		}
 	}
 
-	// public async checkCanShare(share:Share) {
-
-	// 	// const itemTitle = joplinItem ? joplinItem.title || '' : '';
-	// 	// const isNote = joplinItem && joplinItem.type_ === ModelType.Note;
-
-	// 	// // If the item is encrypted, we apply a multipler because encrypted
-	// 	// // items can be much larger (seems to be up to twice the size but for
-	// 	// // safety let's go with 2.2).
-	// 	// const maxSize = user.max_item_size * (item.jop_encryption_applied ? 2.2 : 1);
-	// 	// if (maxSize && buffer.byteLength > maxSize) {
-	// 	// 	throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it is larger than than the allowed limit (%s)',
-	// 	// 		isNote ? _('note') : _('attachment'),
-	// 	// 		itemTitle ? itemTitle : name,
-	// 	// 		prettyBytes(user.max_item_size)
-	// 	// 	));
-	// 	// }
-	// }
-
 	protected async validate(object: User, options: ValidateOptions = {}): Promise<User> {
 		const user: User = await super.validate(object, options);
 

packages/server/src/models/utils/user.ts

@@ -0,0 +1,20 @@
+import { User } from '../../db';
+import { accountByType } from '../UserModel';
+
+export function getCanShareFolder(user: User): number {
+	if (!('account_type' in user) || !('can_share_folder' in user)) throw new Error('Missing account_type or can_share_folder property');
+	const account = accountByType(user.account_type);
+	return user.can_share_folder !== null ? user.can_share_folder : account.can_share_folder;
+}
+
+export function getMaxItemSize(user: User): number {
+	if (!('account_type' in user) || !('max_item_size' in user)) throw new Error('Missing account_type or max_item_size property');
+	const account = accountByType(user.account_type);
+	return user.max_item_size !== null ? user.max_item_size : account.max_item_size;
+}
+
+export function getMaxTotalItemSize(user: User): number {
+	if (!('account_type' in user) || !('max_total_item_size' in user)) throw new Error('Missing account_type or max_total_item_size property');
+	const account = accountByType(user.account_type);
+	return user.max_total_item_size !== null ? user.max_total_item_size : account.max_total_item_size;
+}

packages/server/src/routes/api/items.test.ts

@@ -327,6 +327,44 @@ describe('api_items', function() {
 		}
 	});
 
+	test('should check permissions - uploaded item should not make the account go over the allowed max limit', async function() {
+		const { user: user1, session: session1 } = await createUserAndSession(1);
+
+		{
+			await models().user().save({ id: user1.id, max_total_item_size: 4 });
+
+			await expectHttpError(
+				async () => createNote(session1.id, {
+					id: '00000000000000000000000000000001',
+					body: '12345',
+				}),
+				ErrorPayloadTooLarge.httpCode
+			);
+		}
+
+		{
+			await models().user().save({ id: user1.id, max_total_item_size: 1000 });
+
+			await expectNoHttpError(
+				async () => createNote(session1.id, {
+					id: '00000000000000000000000000000002',
+					body: '12345',
+				})
+			);
+		}
+
+		{
+			await models().user().save({ id: user1.id, max_total_item_size: 0 });
+
+			await expectNoHttpError(
+				async () => createNote(session1.id, {
+					id: '00000000000000000000000000000003',
+					body: '12345',
+				})
+			);
+		}
+	});
+
 	test('should check permissions - should not allow uploading items if disabled', async function() {
 		const { user: user1, session: session1 } = await createUserAndSession(1);
 

packages/server/src/routes/index/home.ts

@@ -5,9 +5,8 @@ import { AppContext } from '../../utils/types';
 import { contextSessionId } from '../../utils/requestUtils';
 import { ErrorMethodNotAllowed } from '../../utils/errors';
 import defaultView from '../../utils/defaultView';
-import { accountTypeProperties, accountTypeToString } from '../../models/UserModel';
-import { formatBytes } from '../../utils/bytes';
-import { yesOrNo } from '../../utils/strings';
+import { accountByType, accountTypeToString } from '../../models/UserModel';
+import { formatMaxItemSize, yesOrNo } from '../../utils/strings';
 
 const router: Router = new Router(RouteType.Web);
 
@@ -15,7 +14,7 @@ router.get('home', async (_path: SubPath, ctx: AppContext) => {
 	contextSessionId(ctx);
 
 	if (ctx.method === 'GET') {
-		const accountProps = accountTypeProperties(ctx.joplin.owner.account_type);
+		const accountProps = accountByType(ctx.joplin.owner.account_type);
 
 		const view = defaultView('home', 'Home');
 		view.content = {
@@ -30,7 +29,7 @@ router.get('home', async (_path: SubPath, ctx: AppContext) => {
 				},
 				{
 					label: 'Max Item Size',
-					value: accountProps.max_item_size ? formatBytes(accountProps.max_item_size) : '∞',
+					value: formatMaxItemSize(ctx.joplin.owner),
 				},
 				{
 					label: 'Can Share Note',

packages/server/src/routes/index/signup.test.ts

@@ -1,6 +1,7 @@
 import config from '../../config';
 import { NotificationKey } from '../../models/NotificationModel';
 import { AccountType } from '../../models/UserModel';
+import { getCanShareFolder, getMaxItemSize } from '../../models/utils/user';
 import { MB } from '../../utils/bytes';
 import { execRequestC } from '../../utils/testing/apiUtils';
 import { beforeAllDb, afterAllTests, beforeEachDb, models } from '../../utils/testing/testUtils';
@@ -43,8 +44,8 @@ describe('index_signup', function() {
 		expect(user).toBeTruthy();
 		expect(user.account_type).toBe(AccountType.Basic);
 		expect(user.email_confirmed).toBe(0);
-		expect(user.can_share_folder).toBe(0);
-		expect(user.max_item_size).toBe(10 * MB);
+		expect(getCanShareFolder(user)).toBe(0);
+		expect(getMaxItemSize(user)).toBe(10 * MB);
 
 		// Check that the user is logged in
 		const session = await models().session().load(context.cookies.get('sessionId'));

packages/server/src/routes/index/signup.ts

@@ -8,7 +8,7 @@ import defaultView from '../../utils/defaultView';
 import { View } from '../../services/MustacheService';
 import { checkPassword } from './users';
 import { NotificationKey } from '../../models/NotificationModel';
-import { AccountType, accountTypeProperties } from '../../models/UserModel';
+import { AccountType } from '../../models/UserModel';
 import { ErrorForbidden } from '../../utils/errors';
 
 function makeView(error: Error = null): View {
@@ -44,7 +44,7 @@ router.post('signup', async (_path: SubPath, ctx: AppContext) => {
 		const password = checkPassword(formUser, true);
 
 		const user = await ctx.joplin.models.user().save({
-			...accountTypeProperties(AccountType.Basic),
+			account_type: AccountType.Basic,
 			email: formUser.email,
 			full_name: formUser.full_name,
 			password,

packages/server/src/routes/index/users.test.ts

@@ -58,7 +58,7 @@ export async function getUserHtml(sessionId: string, userId: string): Promise<st
 	return context.response.body;
 }
 
-describe('index_users', function() {
+describe('index/users', function() {
 
 	beforeAll(async () => {
 		await beforeAllDb('index_users');
@@ -84,7 +84,7 @@ describe('index_users', function() {
 		expect(!!newUser.id).toBe(true);
 		expect(!!newUser.is_admin).toBe(false);
 		expect(!!newUser.email).toBe(true);
-		expect(newUser.max_item_size).toBe(0);
+		expect(newUser.max_item_size).toBe(null);
 		expect(newUser.must_set_password).toBe(0);
 
 		const userModel = models().user();
@@ -94,6 +94,18 @@ describe('index_users', function() {
 		expect(userFromModel.password === '123456').toBe(false); // Password has been hashed
 	});
 
+	test('should create a user with null properties if they are not explicitly set', async function() {
+		const { session } = await createUserAndSession(1, true);
+
+		await postUser(session.id, 'test@example.com', '123456');
+		const newUser = await models().user().loadByEmail('test@example.com');
+
+		expect(newUser.max_item_size).toBe(null);
+		expect(newUser.can_share_folder).toBe(null);
+		expect(newUser.can_share_note).toBe(null);
+		expect(newUser.max_total_item_size).toBe(null);
+	});
+
 	test('should ask user to set password if not set on creation', async function() {
 		const { session } = await createUserAndSession(1, true);
 

packages/server/src/routes/index/users.ts

@@ -10,9 +10,11 @@ import { View } from '../../services/MustacheService';
 import defaultView from '../../utils/defaultView';
 import { AclAction } from '../../models/BaseModel';
 import { NotificationKey } from '../../models/NotificationModel';
-import { formatBytes } from '../../utils/bytes';
-import { accountTypeOptions, accountTypeProperties } from '../../models/UserModel';
+import { accountTypeOptions, accountTypeToString } from '../../models/UserModel';
 import uuidgen from '../../utils/uuidgen';
+import { formatMaxItemSize, formatMaxTotalSize, formatTotalSize, yesOrNo } from '../../utils/strings';
+import { getCanShareFolder, getMaxTotalItemSize } from '../../models/utils/user';
+import { yesNoDefaultOptions } from '../../utils/views/select';
 
 interface CheckPasswordInput {
 	password: string;
@@ -30,22 +32,30 @@ export function checkPassword(fields: CheckPasswordInput, required: boolean): st
 	return '';
 }
 
+function boolOrDefaultToValue(fields: any, fieldName: string): number | null {
+	if (fields[fieldName] === '') return null;
+	const output = Number(fields[fieldName]);
+	if (isNaN(output) || (output !== 0 && output !== 1)) throw new Error(`Invalid value for ${fieldName}`);
+	return output;
+}
+
+function intOrDefaultToValue(fields: any, fieldName: string): number | null {
+	if (fields[fieldName] === '') return null;
+	const output = Number(fields[fieldName]);
+	if (isNaN(output)) throw new Error(`Invalid value for ${fieldName}`);
+	return output;
+}
+
 function makeUser(isNew: boolean, fields: any): User {
-	let user: User = {};
+	const user: User = {};
 
 	if ('email' in fields) user.email = fields.email;
 	if ('full_name' in fields) user.full_name = fields.full_name;
 	if ('is_admin' in fields) user.is_admin = fields.is_admin;
-	if ('max_item_size' in fields) user.max_item_size = fields.max_item_size || 0;
-	if ('can_share_folder' in fields) user.can_share_folder = fields.can_share_folder ? 1 : 0;
-
-	if ('account_type' in fields) {
-		user.account_type = Number(fields.account_type);
-		user = {
-			...user,
-			...accountTypeProperties(user.account_type),
-		};
-	}
+	if ('max_item_size' in fields) user.max_item_size = intOrDefaultToValue(fields, 'max_item_size');
+	if ('max_total_item_size' in fields) user.max_total_item_size = intOrDefaultToValue(fields, 'max_total_item_size');
+	if ('can_share_folder' in fields) user.can_share_folder = boolOrDefaultToValue(fields, 'can_share_folder');
+	if ('account_type' in fields) user.account_type = Number(fields.account_type);
 
 	const password = checkPassword(fields, false);
 	if (password) user.password = password;
@@ -61,10 +71,7 @@ function makeUser(isNew: boolean, fields: any): User {
 }
 
 function defaultUser(): User {
-	return {
-		can_share_folder: 1,
-		max_item_size: 0,
-	};
+	return {};
 }
 
 function userIsNew(path: SubPath): boolean {
@@ -77,6 +84,19 @@ function userIsMe(path: SubPath): boolean {
 
 const router = new Router(RouteType.Web);
 
+function totalSizePercent(user: User): number {
+	const maxTotalSize = getMaxTotalItemSize(user);
+	if (!maxTotalSize) return 0;
+	return user.total_item_size / maxTotalSize;
+}
+
+function totalSizeClass(user: User) {
+	const d = totalSizePercent(user);
+	if (d >= 1) return 'is-danger';
+	if (d >= .7) return 'is-warning';
+	return '';
+}
+
 router.get('users', async (_path: SubPath, ctx: AppContext) => {
 	const userModel = ctx.joplin.models.user();
 	await userModel.checkIfAllowed(ctx.joplin.owner, AclAction.List);
@@ -94,7 +114,13 @@ router.get('users', async (_path: SubPath, ctx: AppContext) => {
 	view.content.users = users.map(user => {
 		return {
 			...user,
-			formattedItemMaxSize: user.max_item_size ? formatBytes(user.max_item_size) : '∞',
+			formattedItemMaxSize: formatMaxItemSize(user),
+			formattedTotalSize: formatTotalSize(user),
+			formattedMaxTotalSize: formatMaxTotalSize(user),
+			formattedTotalSizePercent: `${Math.round(totalSizePercent(user) * 100)}%`,
+			totalSizeClass: totalSizeClass(user),
+			formattedAccountType: accountTypeToString(user.account_type),
+			formattedCanShareFolder: yesOrNo(getCanShareFolder(user)),
 		};
 	});
 	return view;
@@ -131,6 +157,8 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null
 	view.content.showDeleteButton = !isNew && !!owner.is_admin && owner.id !== user.id;
 	view.content.showResetPasswordButton = !isNew && owner.is_admin;
 
+	view.content.canShareFolderOptions = yesNoDefaultOptions(user, 'can_share_folder');
+
 	if (config().accountTypesEnabled) {
 		view.content.showAccountTypes = true;
 		view.content.accountTypes = accountTypeOptions().map((o: any) => {

packages/server/src/services/CronService.ts

@@ -1,11 +1,26 @@
+import Logger from '@joplin/lib/Logger';
 import BaseService from './BaseService';
 const cron = require('node-cron');
 
+const logger = Logger.create('cron');
+
 export default class CronService extends BaseService {
 
 	public async runInBackground() {
+		await this.models.item().updateTotalSizes();
+
 		cron.schedule('0 */6 * * *', async () => {
+			const startTime = Date.now();
+			logger.info('Deleting expired tokens...');
 			await this.models.token().deleteExpiredTokens();
+			logger.info(`Expired tokens deleted in ${Date.now() - startTime}ms`);
+		});
+
+		cron.schedule('0 * * * *', async () => {
+			const startTime = Date.now();
+			logger.info('Updating total sizes...');
+			await this.models.item().updateTotalSizes();
+			logger.info(`Total sizes updated in ${Date.now() - startTime}ms`);
 		});
 	}
 

packages/server/src/tools/generateTypes.ts

@@ -48,6 +48,10 @@ const propertyTypes: Record<string, string> = {
 	'emails.sent_time': 'number',
 	'subscriptions.last_payment_time': 'number',
 	'subscriptions.last_payment_failed_time': 'number',
+	'users.can_share_folder': 'number | null',
+	'users.can_share_note': 'number | null',
+	'users.max_total_item_size': 'number | null',
+	'users.max_item_size': 'number | null',
 };
 
 function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void {

packages/server/src/utils/strings.ts

@@ -1,3 +1,7 @@
+import { User } from '../db';
+import { getMaxItemSize, getMaxTotalItemSize } from '../models/utils/user';
+import { formatBytes } from './bytes';
+
 export function yesOrNo(value: any): string {
 	return value ? 'yes' : 'no';
 }
@@ -5,3 +9,17 @@ export function yesOrNo(value: any): string {
 export function nothing() {
 	return '';
 }
+
+export function formatMaxItemSize(user: User): string {
+	const size = getMaxItemSize(user);
+	return size ? formatBytes(size) : '∞';
+}
+
+export function formatMaxTotalSize(user: User): string {
+	const size = getMaxTotalItemSize(user);
+	return size ? formatBytes(size) : '∞';
+}
+
+export function formatTotalSize(user: User): string {
+	return formatBytes(user.total_item_size);
+}

packages/server/src/utils/types.ts

@@ -2,6 +2,7 @@ import { LoggerWrapper } from '@joplin/lib/Logger';
 import * as Koa from 'koa';
 import { DbConnection, User, Uuid } from '../db';
 import { Models } from '../models/factory';
+import { Account } from '../models/UserModel';
 import { Services } from '../services/types';
 import { Routers } from './routeUtils';
 
@@ -25,6 +26,7 @@ interface AppContextJoplin {
 	appLogger(): LoggerWrapper;
 	notifications: NotificationView[];
 	owner: User;
+	account: Account;
 	routes: Routers;
 	services: Services;
 }

packages/server/src/utils/views/select.ts

@@ -0,0 +1,36 @@
+interface Option {
+	value: any;
+	label: string;
+	selected: boolean;
+}
+
+type LabelFn = (key: string, value: any)=> string;
+
+export function yesNoDefaultLabel(_key: string, value: any): string {
+	if (value === '') return 'Default';
+	return value ? 'Yes' : 'No';
+}
+
+export function objectToSelectOptions(object: any, selectedValue: any, labelFn: LabelFn): Option[] {
+	const output: Option[] = [];
+	for (const [key, value] of Object.entries(object)) {
+		output.push({
+			label: labelFn(key, value),
+			selected: value === selectedValue,
+			value: value,
+		});
+	}
+	return output;
+}
+
+export function selectOption(label: string, value: any, selected: boolean): Option {
+	return { label, value, selected };
+}
+
+export function yesNoDefaultOptions(object: any, key: string): Option[] {
+	return [
+		selectOption('Default', '', object[key] === null),
+		selectOption('Yes', '1', object[key] === 1),
+		selectOption('No', '0', object[key] === 0),
+	];
+}

packages/server/src/views/index/user.mustache

@@ -14,6 +14,7 @@
 			<input class="input" type="email" name="email" value="{{user.email}}"/>
 		</div>
 	</div>
+
 	{{#global.owner.is_admin}}
 		{{#showAccountTypes}}
 			<div class="field">
@@ -25,23 +26,36 @@
 						{{/accountTypes}}
 					</select>
 				</div>
-				<p class="help">If the account type is anything other than Default, the account-specific properties will be applied.</p>
+				<p class="help">If the below properties are left to their default (empty) values, the account-specific properties will apply.</p>
 			</div>
 		{{/showAccountTypes}}
 
 		<div class="field">
 			<label class="label">Max item size</label>
 			<div class="control">
-				<input class="input" type="number" name="max_item_size" value="{{user.max_item_size}}"/>
+				<input class="input" type="number" placeholder="Default" name="max_item_size" value="{{user.max_item_size}}"/>
 			</div>
 		</div>
 
 		<div class="field">
-			<label class="checkbox">
-				<input type="checkbox" name="can_share_folder" {{#user.can_share_folder}}checked{{/user.can_share_folder}} value="1"> Can share notebook
-			</label>
+			<label class="label">Max total size</label>
+			<div class="control">
+				<input class="input" type="number" placeholder="Default" name="max_total_item_size" value="{{user.max_total_item_size}}"/>
+			</div>
+		</div>
+
+		<div class="field">
+			<label class="label">Can share notebook</label>
+			<div class="select">
+				<select name="can_share_folder">
+					{{#canShareFolderOptions}}
+						<option value="{{value}}" {{#selected}}selected{{/selected}}>{{label}}</option>
+					{{/canShareFolderOptions}}
+				</select>
+			</div>
 		</div>
 	{{/global.owner.is_admin}}
+
 	<div class="field">
 		<label class="label">Password</label>
 		<div class="control">

packages/server/src/views/index/users.mustache

@@ -1,9 +1,16 @@
+<div class="block">
+	<a class="button is-primary" href="{{{global.baseUrl}}}/users/new">Add user</a>
+</div>
+
 <table class="table">
 	<thead>
 		<tr>
 			<th>Full name</th>
 			<th>Email</th>
+			<th>Account</th>
 			<th>Max Item Size</th>
+			<th>Total Size</th>
+			<th>Max Total Size</th>
 			<th>Can share</th>
 			<th>Is admin?</th>
 			<th>Actions</th>
@@ -14,8 +21,11 @@
 			<tr>
 				<td>{{full_name}}</td>
 				<td>{{email}}</td>
+				<td>{{formattedAccountType}}</td>
 				<td>{{formattedItemMaxSize}}</td>
-				<td>{{can_share_folder}}</td>
+				<td class="{{totalSizeClass}}">{{formattedTotalSize}} ({{formattedTotalSizePercent}})</td>
+				<td>{{formattedMaxTotalSize}}</td>
+				<td>{{formattedCanShareFolder}}</td>
 				<td>{{is_admin}}</td>
 				<td><a href="{{{global.baseUrl}}}/users/{{id}}" class="button is-primary is-small">Edit</a></td>
 			</tr>