diff --git a/packages/server/src/commands/StorageCommand.ts b/packages/server/src/commands/StorageCommand.ts index 4fca4f054..a46b22dd8 100644 --- a/packages/server/src/commands/StorageCommand.ts +++ b/packages/server/src/commands/StorageCommand.ts @@ -9,6 +9,7 @@ const logger = Logger.create('ImportContentCommand'); enum ArgvCommand { Import = 'import', CheckConnection = 'check-connection', + DeleteDatabaseContentColumn = 'delete-database-content-col', } interface Argv { @@ -35,6 +36,7 @@ export default class StorageCommand extends BaseCommand { choices: [ ArgvCommand.Import, ArgvCommand.CheckConnection, + ArgvCommand.DeleteDatabaseContentColumn, ], }, }; @@ -58,12 +60,13 @@ export default class StorageCommand extends BaseCommand { } public async run(argv: Argv, runContext: RunContext): Promise { + const batchSize = argv.batchSize || 1000; + const commands: Record = { [ArgvCommand.Import]: async () => { if (!argv.connection) throw new Error('--connection option is required'); const toStorageConfig = parseStorageConnectionString(argv.connection); - const batchSize = argv.batchSize || 1000; const maxContentSize = argv.maxContentSize || 200000000; logger.info('Importing to storage:', toStorageConfig); @@ -80,6 +83,15 @@ export default class StorageCommand extends BaseCommand { [ArgvCommand.CheckConnection]: async () => { logger.info(await storageConnectionCheck(argv.connection, runContext.db, runContext.models)); }, + + [ArgvCommand.DeleteDatabaseContentColumn]: async () => { + logger.info(`Batch size: ${batchSize}`); + + await runContext.models.item().deleteDatabaseContentColumn({ + batchSize, + logger, + }); + }, }; await commands[argv.command](); diff --git a/packages/server/src/models/ItemModel.ts b/packages/server/src/models/ItemModel.ts index b131ee3bc..5b4052e7b 100644 --- a/packages/server/src/models/ItemModel.ts +++ b/packages/server/src/models/ItemModel.ts @@ -27,6 +27,11 @@ export interface ImportContentToStorageOptions { logger?: Logger | LoggerWrapper; } +export interface DeleteDatabaseContentOptions { + batchSize?: number; + logger?: Logger | LoggerWrapper; +} + export interface SaveFromRawContentItem { name: string; body: Buffer; @@ -394,6 +399,45 @@ export default class ItemModel extends BaseModel { } } + public async deleteDatabaseContentColumn(options: DeleteDatabaseContentOptions) { + options = { + batchSize: 1000, + logger: new Logger(), + ...options, + }; + + const itemCount = (await this.db(this.tableName) + .count('id', { as: 'total' }) + .where('content', '!=', Buffer.from('')) + .first())['total']; + + let totalDone = 0; + + // UPDATE items SET content = '\x' WHERE id IN (SELECT id FROM items WHERE content != '\x' LIMIT 5000); + + while (true) { + options.logger.info(`Processing items ${totalDone} / ${itemCount}`); + + const updatedRows = await this + .db(this.tableName) + .update({ content: Buffer.from('') }, ['id']) + .whereIn('id', this.db(this.tableName) + .select(['id']) + .where('content', '!=', Buffer.from('')) + .limit(options.batchSize) + ); + + totalDone += updatedRows.length; + + if (!updatedRows.length) { + options.logger.info(`All items have been processed. Total: ${totalDone}`); + return; + } + + await msleep(1000); + } + } + public async sharedFolderChildrenItems(shareUserIds: Uuid[], folderId: string, includeResources: boolean = true): Promise { if (!shareUserIds.length) throw new Error('User IDs must be specified');