From 2c79ce25faa77d8a2258beffddb63142aa8ca3cd Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sat, 14 Aug 2021 17:49:01 +0100 Subject: [PATCH] Server: Added commands to control db migrations - list, down, up --- .dockerignore | 4 +- packages/server/package.json | 3 +- packages/server/src/app.ts | 25 +++++- packages/server/src/db.ts | 77 ++++++++++++++++++- .../20210809222118_email_key_fix.ts | 18 ++--- packages/server/src/tools/dbTools.ts | 4 +- packages/server/src/tools/debugTools.ts | 4 +- .../server/src/utils/testing/testRouters.ts | 4 +- 8 files changed, 115 insertions(+), 24 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6845b54a0..39f251f40 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,6 @@ packages/app-cli packages/app-mobile packages/app-clipper packages/generator-joplin -packages/plugin-repo-cli \ No newline at end of file +packages/plugin-repo-cli +packages/server/db-*.sqlite +packages/server/temp diff --git a/packages/server/package.json b/packages/server/package.json index 1c64c6023..ab2906d3c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -4,11 +4,12 @@ "private": true, "scripts": { "start-dev": "nodemon --config nodemon.json --ext ts,js,mustache,css,tsx dist/app.js --env dev", + "start-dev-no-watch": "node dist/app.js --env dev", "devCreateDb": "node dist/app.js --env dev --create-db", "devDropTables": "node dist/app.js --env dev --drop-tables", "devDropDb": "node dist/app.js --env dev --drop-db", "start": "node dist/app.js", - "generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-db --env buildTypes && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite", + "generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-latest --env buildTypes && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite", "tsc": "tsc --project tsconfig.json", "test": "jest --verbose=false", "test-ci": "npm run test", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 42a676c95..d072d8dd7 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -7,7 +7,7 @@ import { argv } from 'yargs'; import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger'; import config, { initConfig, runningInDocker, EnvVariables } from './config'; import { createDb, dropDb } from './tools/dbTools'; -import { dropTables, connectDb, disconnectDb, migrateDb, waitForConnection, sqliteDefaultDir } from './db'; +import { dropTables, connectDb, disconnectDb, migrateLatest, waitForConnection, sqliteDefaultDir, migrateList, migrateUp, migrateDown } from './db'; import { AppContext, Env, KoaNext } from './utils/types'; import FsDriverNode from '@joplin/lib/fs-driver-node'; import routeHandler from './middleware/routeHandler'; @@ -205,10 +205,23 @@ async function main() { fs.writeFileSync(pidFile, `${process.pid}`); } - if (argv.migrateDb) { + let runCommandAndExitApp = true; + + if (argv.migrateLatest) { const db = await connectDb(config().database); - await migrateDb(db); + await migrateLatest(db); await disconnectDb(db); + } else if (argv.migrateUp) { + const db = await connectDb(config().database); + await migrateUp(db); + await disconnectDb(db); + } else if (argv.migrateDown) { + const db = await connectDb(config().database); + await migrateDown(db); + await disconnectDb(db); + } else if (argv.migrateList) { + const db = await connectDb(config().database); + console.info(await migrateList(db)); } else if (argv.dropDb) { await dropDb(config().database, { ignoreIfNotExists: true }); } else if (argv.dropTables) { @@ -218,6 +231,8 @@ async function main() { } else if (argv.createDb) { await createDb(config().database); } else { + runCommandAndExitApp = false; + appLogger().info(`Starting server v${config().appVersion} (${env}) on port ${config().port} and PID ${process.pid}...`); appLogger().info('Running in Docker:', runningInDocker()); appLogger().info('Public base URL:', config().baseUrl); @@ -239,7 +254,7 @@ async function main() { await initializeJoplinUtils(config(), ctx.joplinBase.models, ctx.joplinBase.services.mustache); appLogger().info('Migrating database...'); - await migrateDb(ctx.joplinBase.db); + await migrateLatest(ctx.joplinBase.db); appLogger().info('Starting services...'); await startServices(ctx.joplinBase.services); @@ -248,6 +263,8 @@ async function main() { app.listen(config().port); } + + if (runCommandAndExitApp) process.exit(0); } main().catch((error: any) => { diff --git a/packages/server/src/db.ts b/packages/server/src/db.ts index 2839ae12a..06de61027 100644 --- a/packages/server/src/db.ts +++ b/packages/server/src/db.ts @@ -121,14 +121,85 @@ export async function disconnectDb(db: DbConnection) { await db.destroy(); } -export async function migrateDb(db: DbConnection) { +export async function migrateLatest(db: DbConnection) { await db.migrate.latest({ directory: migrationDir, - // Disable transactions because the models might open one too - disableTransactions: true, }); } +export async function migrateUp(db: DbConnection) { + await db.migrate.up({ + directory: migrationDir, + }); +} + +export async function migrateDown(db: DbConnection) { + await db.migrate.down({ + directory: migrationDir, + }); +} + +export async function migrateList(db: DbConnection, asString: boolean = true) { + const migrations: any = await db.migrate.list({ + directory: migrationDir, + }); + + // The migration array has a rather inconsistent format: + // + // [ + // // Done migrations + // [ + // '20210809222118_email_key_fix.js', + // '20210814123815_testing.js', + // '20210814123816_testing.js' + // ], + // // Not done migrations + // [ + // { + // file: '20210814123817_testing.js', + // directory: '/path/to/packages/server/dist/migrations' + // } + // ] + // ] + + if (!asString) return migrations; + + const formatName = (migrationInfo: any) => { + const name = migrationInfo.file ? migrationInfo.file : migrationInfo; + + const s = name.split('.'); + s.pop(); + return s.join('.'); + }; + + interface Line { + text: string; + done: boolean; + } + + const output: Line[] = []; + + for (const s of migrations[0]) { + output.push({ + text: formatName(s), + done: true, + }); + } + + for (const s of migrations[1]) { + output.push({ + text: formatName(s), + done: false, + }); + } + + output.sort((a, b) => { + return a.text < b.text ? -1 : +1; + }); + + return output.map(l => `${l.done ? '✓' : '✗'} ${l.text}`).join('\n'); +} + function allTableNames(): string[] { const tableNames = Object.keys(databaseSchema); tableNames.push('knex_migrations'); diff --git a/packages/server/src/migrations/20210809222118_email_key_fix.ts b/packages/server/src/migrations/20210809222118_email_key_fix.ts index 2ca9d10d7..a4e16a43f 100644 --- a/packages/server/src/migrations/20210809222118_email_key_fix.ts +++ b/packages/server/src/migrations/20210809222118_email_key_fix.ts @@ -1,14 +1,14 @@ -import { Knex } from 'knex'; +// import { Knex } from 'knex'; import { DbConnection } from '../db'; -export async function up(db: DbConnection): Promise { - try { - await db.schema.alterTable('emails', function(table: Knex.CreateTableBuilder) { - table.dropUnique(['recipient_email', 'key']); - }); - } catch (error) { - console.warn('Could not drop unique constraint - this is not an error.', error); - } +export async function up(_db: DbConnection): Promise { + // try { + // await db.schema.alterTable('emails', function(table: Knex.CreateTableBuilder) { + // table.dropUnique(['recipient_email', 'key']); + // }); + // } catch (error) { + // // console.warn('Could not drop unique constraint - this is not an error.', error); + // } } export async function down(_db: DbConnection): Promise { diff --git a/packages/server/src/tools/dbTools.ts b/packages/server/src/tools/dbTools.ts index 8b3a9f4e2..57694fee0 100644 --- a/packages/server/src/tools/dbTools.ts +++ b/packages/server/src/tools/dbTools.ts @@ -1,4 +1,4 @@ -import { connectDb, disconnectDb, migrateDb } from '../db'; +import { connectDb, disconnectDb, migrateLatest } from '../db'; import * as fs from 'fs-extra'; import { DatabaseConfig } from '../utils/types'; @@ -46,7 +46,7 @@ export async function createDb(config: DatabaseConfig, options: CreateDbOptions try { const db = await connectDb(config); - await migrateDb(db); + await migrateLatest(db); await disconnectDb(db); } catch (error) { error.message += `: ${config.name}`; diff --git a/packages/server/src/tools/debugTools.ts b/packages/server/src/tools/debugTools.ts index 766451802..02fb5f1a5 100644 --- a/packages/server/src/tools/debugTools.ts +++ b/packages/server/src/tools/debugTools.ts @@ -1,4 +1,4 @@ -import { DbConnection, dropTables, migrateDb } from '../db'; +import { DbConnection, dropTables, migrateLatest } from '../db'; import newModelFactory from '../models/factory'; import { AccountType } from '../models/UserModel'; import { Config } from '../utils/types'; @@ -15,7 +15,7 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C export async function createTestUsers(db: DbConnection, config: Config) { await dropTables(db); - await migrateDb(db); + await migrateLatest(db); const password = 'hunter1hunter2hunter3'; const models = newModelFactory(db, config); diff --git a/packages/server/src/utils/testing/testRouters.ts b/packages/server/src/utils/testing/testRouters.ts index 58e961bb5..5df32165b 100644 --- a/packages/server/src/utils/testing/testRouters.ts +++ b/packages/server/src/utils/testing/testRouters.ts @@ -106,9 +106,9 @@ async function main() { fs.removeSync(`${serverRoot}/db-testing.sqlite`); - // const migrateCommand = 'NODE_ENV=testing node dist/app.js --migrate-db --env dev'; + // const migrateCommand = 'NODE_ENV=testing node dist/app.js --migrate-latest --env dev'; const clearCommand = 'node dist/app.js --env dev --drop-tables'; - const migrateCommand = 'node dist/app.js --env dev --migrate-db'; + const migrateCommand = 'node dist/app.js --env dev --migrate-latest'; await execCommand(clearCommand); await execCommand(migrateCommand);