Server: Added commands to control db migrations - list, down, up

.dockerignore

@@ -8,3 +8,5 @@ packages/app-mobile
 packages/app-clipper
 packages/generator-joplin
 packages/plugin-repo-cli
+packages/server/db-*.sqlite
+packages/server/temp

packages/server/package.json

@@ -4,11 +4,12 @@
   "private": true,
   "scripts": {
     "start-dev": "nodemon --config nodemon.json --ext ts,js,mustache,css,tsx dist/app.js --env dev",
+    "start-dev-no-watch": "node dist/app.js --env dev",
     "devCreateDb": "node dist/app.js --env dev --create-db",
     "devDropTables": "node dist/app.js --env dev --drop-tables",
     "devDropDb": "node dist/app.js --env dev --drop-db",
     "start": "node dist/app.js",
-    "generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-db --env buildTypes && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite",
+    "generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-latest --env buildTypes && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite",
     "tsc": "tsc --project tsconfig.json",
     "test": "jest --verbose=false",
     "test-ci": "npm run test",

packages/server/src/app.ts

@@ -7,7 +7,7 @@ import { argv } from 'yargs';
 import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger';
 import config, { initConfig, runningInDocker, EnvVariables } from './config';
 import { createDb, dropDb } from './tools/dbTools';
-import { dropTables, connectDb, disconnectDb, migrateDb, waitForConnection, sqliteDefaultDir } from './db';
+import { dropTables, connectDb, disconnectDb, migrateLatest, waitForConnection, sqliteDefaultDir, migrateList, migrateUp, migrateDown } from './db';
 import { AppContext, Env, KoaNext } from './utils/types';
 import FsDriverNode from '@joplin/lib/fs-driver-node';
 import routeHandler from './middleware/routeHandler';
@@ -205,10 +205,23 @@ async function main() {
 		fs.writeFileSync(pidFile, `${process.pid}`);
 	}
 
-	if (argv.migrateDb) {
+	let runCommandAndExitApp = true;
+
+	if (argv.migrateLatest) {
 		const db = await connectDb(config().database);
-		await migrateDb(db);
+		await migrateLatest(db);
 		await disconnectDb(db);
+	} else if (argv.migrateUp) {
+		const db = await connectDb(config().database);
+		await migrateUp(db);
+		await disconnectDb(db);
+	} else if (argv.migrateDown) {
+		const db = await connectDb(config().database);
+		await migrateDown(db);
+		await disconnectDb(db);
+	} else if (argv.migrateList) {
+		const db = await connectDb(config().database);
+		console.info(await migrateList(db));
 	} else if (argv.dropDb) {
 		await dropDb(config().database, { ignoreIfNotExists: true });
 	} else if (argv.dropTables) {
@@ -218,6 +231,8 @@ async function main() {
 	} else if (argv.createDb) {
 		await createDb(config().database);
 	} else {
+		runCommandAndExitApp = false;
+
 		appLogger().info(`Starting server v${config().appVersion} (${env}) on port ${config().port} and PID ${process.pid}...`);
 		appLogger().info('Running in Docker:', runningInDocker());
 		appLogger().info('Public base URL:', config().baseUrl);
@@ -239,7 +254,7 @@ async function main() {
 		await initializeJoplinUtils(config(), ctx.joplinBase.models, ctx.joplinBase.services.mustache);
 
 		appLogger().info('Migrating database...');
-		await migrateDb(ctx.joplinBase.db);
+		await migrateLatest(ctx.joplinBase.db);
 
 		appLogger().info('Starting services...');
 		await startServices(ctx.joplinBase.services);
@@ -248,6 +263,8 @@ async function main() {
 
 		app.listen(config().port);
 	}
+
+	if (runCommandAndExitApp) process.exit(0);
 }
 
 main().catch((error: any) => {

packages/server/src/db.ts

@@ -121,14 +121,85 @@ export async function disconnectDb(db: DbConnection) {
 	await db.destroy();
 }
 
-export async function migrateDb(db: DbConnection) {
+export async function migrateLatest(db: DbConnection) {
 	await db.migrate.latest({
 		directory: migrationDir,
-		// Disable transactions because the models might open one too
-		disableTransactions: true,
 	});
 }
 
+export async function migrateUp(db: DbConnection) {
+	await db.migrate.up({
+		directory: migrationDir,
+	});
+}
+
+export async function migrateDown(db: DbConnection) {
+	await db.migrate.down({
+		directory: migrationDir,
+	});
+}
+
+export async function migrateList(db: DbConnection, asString: boolean = true) {
+	const migrations: any = await db.migrate.list({
+		directory: migrationDir,
+	});
+
+	// The migration array has a rather inconsistent format:
+	//
+	// [
+	//   // Done migrations
+	//   [
+	//     '20210809222118_email_key_fix.js',
+	//     '20210814123815_testing.js',
+	//     '20210814123816_testing.js'
+	//   ],
+	//   // Not done migrations
+	//   [
+	//     {
+	//       file: '20210814123817_testing.js',
+	//       directory: '/path/to/packages/server/dist/migrations'
+	//     }
+	//   ]
+	// ]
+
+	if (!asString) return migrations;
+
+	const formatName = (migrationInfo: any) => {
+		const name = migrationInfo.file ? migrationInfo.file : migrationInfo;
+
+		const s = name.split('.');
+		s.pop();
+		return s.join('.');
+	};
+
+	interface Line {
+		text: string;
+		done: boolean;
+	}
+
+	const output: Line[] = [];
+
+	for (const s of migrations[0]) {
+		output.push({
+			text: formatName(s),
+			done: true,
+		});
+	}
+
+	for (const s of migrations[1]) {
+		output.push({
+			text: formatName(s),
+			done: false,
+		});
+	}
+
+	output.sort((a, b) => {
+		return a.text < b.text ? -1 : +1;
+	});
+
+	return output.map(l => `${l.done ? '✓' : '✗'} ${l.text}`).join('\n');
+}
+
 function allTableNames(): string[] {
 	const tableNames = Object.keys(databaseSchema);
 	tableNames.push('knex_migrations');

packages/server/src/migrations/20210809222118_email_key_fix.ts

@@ -1,14 +1,14 @@
-import { Knex } from 'knex';
+// import { Knex } from 'knex';
 import { DbConnection } from '../db';
 
-export async function up(db: DbConnection): Promise<any> {
-	try {
-		await db.schema.alterTable('emails', function(table: Knex.CreateTableBuilder) {
-			table.dropUnique(['recipient_email', 'key']);
-		});
-	} catch (error) {
-		console.warn('Could not drop unique constraint - this is not an error.', error);
-	}
+export async function up(_db: DbConnection): Promise<any> {
+	// try {
+	// 	await db.schema.alterTable('emails', function(table: Knex.CreateTableBuilder) {
+	// 		table.dropUnique(['recipient_email', 'key']);
+	// 	});
+	// } catch (error) {
+	// 	// console.warn('Could not drop unique constraint - this is not an error.', error);
+	// }
 }
 
 export async function down(_db: DbConnection): Promise<any> {

packages/server/src/tools/dbTools.ts

@@ -1,4 +1,4 @@
-import { connectDb, disconnectDb, migrateDb } from '../db';
+import { connectDb, disconnectDb, migrateLatest } from '../db';
 import * as fs from 'fs-extra';
 import { DatabaseConfig } from '../utils/types';
 
@@ -46,7 +46,7 @@ export async function createDb(config: DatabaseConfig, options: CreateDbOptions
 
 	try {
 		const db = await connectDb(config);
-		await migrateDb(db);
+		await migrateLatest(db);
 		await disconnectDb(db);
 	} catch (error) {
 		error.message += `: ${config.name}`;

packages/server/src/tools/debugTools.ts

@@ -1,4 +1,4 @@
-import { DbConnection, dropTables, migrateDb } from '../db';
+import { DbConnection, dropTables, migrateLatest } from '../db';
 import newModelFactory from '../models/factory';
 import { AccountType } from '../models/UserModel';
 import { Config } from '../utils/types';
@@ -15,7 +15,7 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C
 
 export async function createTestUsers(db: DbConnection, config: Config) {
 	await dropTables(db);
-	await migrateDb(db);
+	await migrateLatest(db);
 
 	const password = 'hunter1hunter2hunter3';
 	const models = newModelFactory(db, config);

packages/server/src/utils/testing/testRouters.ts

@@ -106,9 +106,9 @@ async function main() {
 
 	fs.removeSync(`${serverRoot}/db-testing.sqlite`);
 
-	// const migrateCommand = 'NODE_ENV=testing node dist/app.js --migrate-db --env dev';
+	// const migrateCommand = 'NODE_ENV=testing node dist/app.js --migrate-latest --env dev';
 	const clearCommand = 'node dist/app.js --env dev --drop-tables';
-	const migrateCommand = 'node dist/app.js --env dev --migrate-db';
+	const migrateCommand = 'node dist/app.js --env dev --migrate-latest';
 
 	await execCommand(clearCommand);
 	await execCommand(migrateCommand);