Testing Postgres collation

packages/server/src/db.perf.ts

@@ -0,0 +1,125 @@
+import { afterAllTests, beforeAllDb, beforeEachDb, createUserAndSession, db, models } from "./utils/testing/testUtils";
+import sqlts from '@rmp135/sql-ts';
+import config from "./config";
+import { connectDb, DbConnection, disconnectDb, migrateDown, migrateList, migrateUp, nextMigration, Uuid } from "./services/database/types";
+
+const { shimInit } = require('@joplin/lib/shim-init-node.js');
+const nodeSqlite = require('sqlite3');
+
+shimInit({ nodeSqlite });
+process.env.JOPLIN_IS_TESTING = '1';
+
+// async function makeTestItem(userId: Uuid, num: number): Promise<Item> {
+// 	return models().item().saveForUser(userId, {
+// 		name: `${num.toString().padStart(32, '0')}.md`,
+// 	});
+// }
+
+const main = async() => {
+	await beforeAllDb('db.perf');
+
+	const { user } = await createUserAndSession(1, true);
+
+	await models().item().makeTestItems(user.id, 10000);
+
+	{
+		const startTime = Date.now();
+		await db().raw('SELECT id FROM items ORDER BY name DESC');
+		console.info('Time:', Date.now() - startTime);
+
+		// With collate C:
+		//
+		// ASC: 23ms
+		// DESC: 114
+
+
+	}
+
+	// const durations:number[] = [];
+
+	// for (let i = 0; i < 1000; i++) {
+	// 	const id = 1 + Math.floor(Math.random() * 10000);
+	// 	const item = await models().item().loadByName(user.id, `${id.toString().padStart(32, '0')}.md`);
+	// 	const startTime = Date.now();
+	// 	await models().item().load(item.id);
+	// 	durations.push(Date.now() - startTime);
+	// }
+	
+
+	// let sum = 0;
+	// for (const d of durations) sum += d;
+
+	// console.info('Time per query: ' + (sum / durations.length));
+
+	await afterAllTests();
+}
+
+main().catch(error => {
+	console.error(error);
+});
+
+// async function dbSchemaSnapshot(db:DbConnection):Promise<any> {
+// 	return sqlts.toObject({
+// 		client: 'sqlite',
+// 		knex: db,
+// 		// 'connection': {
+// 		// 	'filename': config().database.name,
+// 		// },
+// 		useNullAsDefault: true,
+// 	} as any)
+
+// 	// return JSON.stringify(definitions);
+// }
+
+// describe('db', function() {
+
+// 	beforeAll(async () => {
+// 		await beforeAllDb('db', { autoMigrate: false });
+// 	});
+
+// 	afterAll(async () => {
+// 		await afterAllTests();
+// 	});
+
+// 	beforeEach(async () => {
+// 		await beforeEachDb();
+// 	});
+
+// 	it('should allow downgrading schema', async function() {
+// 		const ignoreAllBefore = '20210819165350_user_flags';
+// 		let startProcessing = false;
+
+// 		//console.info(await dbSchemaSnapshot());
+
+// 		while (true) {
+// 			await migrateUp(db());
+
+// 			if (!startProcessing) {
+// 				const next = await nextMigration(db());
+// 				if (next === ignoreAllBefore) {
+// 					startProcessing = true;
+// 				} else {
+// 					continue;
+// 				}
+// 			}
+
+// 			if (!(await nextMigration(db()))) break;
+
+// 			// await disconnectDb(db());
+// 			// const beforeSchema = await dbSchemaSnapshot(db());
+// 			// console.info(beforeSchema);
+// 			// await connectDb(db());
+
+// 			// await migrateUp(db());
+// 			// await migrateDown(db());
+
+// 			// const afterSchema = await dbSchemaSnapshot(db());
+
+// 			// // console.info(beforeSchema);
+// 			// // console.info(afterSchema);
+
+// 			// expect(beforeSchema).toEqual(afterSchema);
+// 		}
+// 	});
+
+// });

packages/server/src/db.ts

@@ -1,5 +1,5 @@
 import { knex, Knex } from 'knex';
-import { DatabaseConfig } from './utils/types';
+import { DatabaseConfig, DatabaseConfigClient } from './utils/types';
 import * as pathUtils from 'path';
 import time from '@joplin/lib/time';
 import Logger from '@joplin/lib/Logger';
@@ -112,6 +112,23 @@ export async function waitForConnection(dbConfig: DatabaseConfig): Promise<Conne
 	}
 }
 
+export const clientType = (db: DbConnection): DatabaseConfigClient => {
+	return db.client.config.client;
+};
+
+export const isPostgres = (db: DbConnection) => {
+	return clientType(db) === DatabaseConfigClient.PostgreSQL;
+};
+
+export const isSqlite = (db: DbConnection) => {
+	return clientType(db) === DatabaseConfigClient.SQLite;
+};
+
+export const setCollateC = async (db: DbConnection, tableName: string, columnName: string): Promise<void> => {
+	if (!isPostgres(db)) return;
+	await db.raw(`ALTER TABLE ${tableName} ALTER COLUMN ${columnName} SET DATA TYPE character varying(32) COLLATE "C"`);
+};
+
 function makeSlowQueryHandler(duration: number, connection: any, sql: string, bindings: any[]) {
 	return setTimeout(() => {
 		try {

packages/server/src/migrations/20211010124650_postgres_collate.ts

@@ -0,0 +1,39 @@
+import { DbConnection, setCollateC } from '../db';
+
+export async function up(db: DbConnection): Promise<any> {
+	// await setCollateC(db, 'api_clients', 'id');
+	// await setCollateC(db, 'changes', 'id');
+	// await setCollateC(db, 'changes', 'item_id');
+	// await setCollateC(db, 'changes', 'user_id');
+	// await setCollateC(db, 'emails', 'recipient_id');
+	// await setCollateC(db, 'item_resources', 'item_id');
+	// await setCollateC(db, 'item_resources', 'resource_id');
+	// await setCollateC(db, 'items', 'id');
+	// await setCollateC(db, 'items', 'jop_id');
+	// await setCollateC(db, 'items', 'jop_parent_id');
+	// await setCollateC(db, 'items', 'jop_share_id');
+	// await setCollateC(db, 'notifications', 'id');
+	// await setCollateC(db, 'notifications', 'owner_id');
+	// await setCollateC(db, 'sessions', 'id');
+	// await setCollateC(db, 'sessions', 'user_id');
+	// await setCollateC(db, 'share_users', 'id');
+	// await setCollateC(db, 'share_users', 'share_id');
+	// await setCollateC(db, 'share_users', 'user_id');
+	// await setCollateC(db, 'shares', 'folder_id');
+	// await setCollateC(db, 'shares', 'id');
+	// await setCollateC(db, 'shares', 'item_id');
+	// await setCollateC(db, 'shares', 'note_id');
+	// await setCollateC(db, 'shares', 'owner_id');
+	// await setCollateC(db, 'subscriptions', 'stripe_subscription_id');
+	// await setCollateC(db, 'subscriptions', 'stripe_user_id');
+	// await setCollateC(db, 'subscriptions', 'user_id');
+	// await setCollateC(db, 'tokens', 'user_id');
+	// await setCollateC(db, 'user_flags', 'user_id');
+	// await setCollateC(db, 'user_items', 'item_id');
+	// await setCollateC(db, 'user_items', 'user_id');
+	// await setCollateC(db, 'users', 'id');
+}
+
+export async function down(_db: DbConnection): Promise<any> {
+
+}

packages/server/src/models/ChangeModel.test.ts

@@ -1,15 +1,9 @@
 import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, expectThrow, createFolder, createItemTree3, expectNotThrow } from '../utils/testing/testUtils';
-import { ChangeType, Item, Uuid } from '../services/database/types';
+import { ChangeType } from '../services/database/types';
 import { msleep } from '../utils/time';
 import { ChangePagination } from './ChangeModel';
 import { SqliteMaxVariableNum } from '../db';
 
-async function makeTestItem(userId: Uuid, num: number): Promise<Item> {
-	return models().item().saveForUser(userId, {
-		name: `${num.toString().padStart(32, '0')}.md`,
-	});
-}
-
 describe('ChangeModel', function() {
 
 	beforeAll(async () => {
@@ -43,14 +37,14 @@ describe('ChangeModel', function() {
 		const itemModel = models().item();
 		const changeModel = models().change();
 
-		await msleep(1); const item1 = await makeTestItem(user.id, 1); // [1] CREATE 1
+		await msleep(1); const item1 = await models().item().makeTestItem(user.id, 1); // [1] CREATE 1
 		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001A.md' }); // [2] UPDATE 1a
 		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001B.md' }); // [3] UPDATE 1b
-		await msleep(1); const item2 = await makeTestItem(user.id, 2); // [4] CREATE 2
+		await msleep(1); const item2 = await models().item().makeTestItem(user.id, 2); // [4] CREATE 2
 		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002A.md' }); // [5] UPDATE 2a
 		await msleep(1); await itemModel.delete(item1.id); // [6] DELETE 1
 		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002B.md' }); // [7] UPDATE 2b
-		await msleep(1); const item3 = await makeTestItem(user.id, 3); // [8] CREATE 3
+		await msleep(1); const item3 = await models().item().makeTestItem(user.id, 3); // [8] CREATE 3
 
 		// Check that the 8 changes were created
 		const allUncompressedChanges = await changeModel.all();
@@ -125,7 +119,7 @@ describe('ChangeModel', function() {
 		const changeModel = models().change();
 
 		let i = 1;
-		await msleep(1); const item1 = await makeTestItem(user.id, 1); // CREATE 1
+		await msleep(1); const item1 = await models().item().makeTestItem(user.id, 1); // CREATE 1
 		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: `test_mod${i++}` }); // UPDATE 1
 
 		await expectThrow(async () => changeModel.delta(user.id, { limit: 1, cursor: 'invalid' }), 'resyncRequired');
@@ -173,9 +167,7 @@ describe('ChangeModel', function() {
 
 		const { user } = await createUserAndSession(1, true);
 
-		for (let i = 0; i < 1010; i++) {
+		await models().item().makeTestItems(user.id, 1010);
-			await makeTestItem(user.id, i);
-		}
 
 		let changeCount = 0;
 		await expectNotThrow(async () => {

packages/server/src/models/ItemModel.ts

@@ -549,6 +549,22 @@ export default class ItemModel extends BaseModel<Item> {
 		}
 	}
 
+	public async makeTestItem(userId: Uuid, num: number) {
+		return this.saveForUser(userId, {
+			name: `${num.toString().padStart(32, '0')}.md`,
+		});
+	}
+
+	public async makeTestItems(userId: Uuid, count: number) {
+		await this.withTransaction(async () => {
+			for (let i = 1; i <= count; i++) {
+				await this.saveForUser(userId, {
+					name: `${i.toString().padStart(32, '0')}.md`,
+				});
+			}
+		}, 'ItemModel::makeTestItems');
+	}
+
 	public async saveForUser(userId: Uuid, item: Item, options: SaveOptions = {}): Promise<Item> {
 		if (!userId) throw new Error('userId is required');
 

packages/server/src/utils/testing/testUtils.ts

@@ -73,23 +73,23 @@ export async function beforeAllDb(unitName: string, createDbOptions: CreateDbOpt
 	//
 	// sudo docker compose -f docker-compose.db-dev.yml up
 
-	// await initConfig(Env.Dev, {
-	// 	DB_CLIENT: 'pg',
-	// 	POSTGRES_DATABASE: unitName,
-	// 	POSTGRES_USER: 'joplin',
-	// 	POSTGRES_PASSWORD: 'joplin',
-	// 	SUPPORT_EMAIL: 'testing@localhost',
-	// }, {
-	// 	tempDir: tempDir,
-	// });
-
 	await initConfig(Env.Dev, {
-		SQLITE_DATABASE: createdDbPath_,
+		DB_CLIENT: 'pg',
+		POSTGRES_DATABASE: unitName,
+		POSTGRES_USER: 'joplin',
+		POSTGRES_PASSWORD: 'joplin',
 		SUPPORT_EMAIL: 'testing@localhost',
 	}, {
 		tempDir: tempDir,
 	});
 
+	// await initConfig(Env.Dev, {
+	// 	SQLITE_DATABASE: createdDbPath_,
+	// 	SUPPORT_EMAIL: 'testing@localhost',
+	// }, {
+	// 	tempDir: tempDir,
+	// });
+
 	initGlobalLogger();
 
 	await createDb(config().database, { dropIfExists: true, ...createDbOptions });