mirror of
https://github.com/laurent22/joplin.git
synced 2025-01-14 18:27:44 +02:00
Server: Lazy-load storage drivers
This commit is contained in:
Laurent Cozic
parent
4deeed0d5c
commit
7431da9f3a
Binary file not shown.
@ -5,7 +5,7 @@ import * as Koa from 'koa';
|
||||
import * as fs from 'fs-extra';
|
||||
import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger';
|
||||
import config, { initConfig, runningInDocker } from './config';
|
||||
import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration, DbConnection } from './db';
|
||||
import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration } from './db';
|
||||
import { AppContext, Env, KoaNext } from './utils/types';
|
||||
import FsDriverNode from '@joplin/lib/fs-driver-node';
|
||||
import routeHandler from './middleware/routeHandler';
|
||||
@ -17,11 +17,10 @@ import startServices from './utils/startServices';
|
||||
import { credentialFile } from './utils/testing/testUtils';
|
||||
import apiVersionHandler from './middleware/apiVersionHandler';
|
||||
import clickJackingHandler from './middleware/clickJackingHandler';
|
||||
import newModelFactory, { Options } from './models/factory';
|
||||
import newModelFactory from './models/factory';
|
||||
import setupCommands from './utils/setupCommands';
|
||||
import { RouteResponseFormat, routeResponseFormat } from './utils/routeUtils';
|
||||
import { parseEnv } from './env';
|
||||
import storageDriverFromConfig from './models/items/storage/storageDriverFromConfig';
|
||||
|
||||
interface Argv {
|
||||
env?: Env;
|
||||
@ -222,13 +221,6 @@ async function main() {
|
||||
fs.writeFileSync(pidFile, `${process.pid}`);
|
||||
}
|
||||
|
||||
const newModelFactoryOptions = async (db: DbConnection): Promise<Options> => {
|
||||
return {
|
||||
storageDriver: await storageDriverFromConfig(config().storageDriver, db, { assignDriverId: env !== 'buildTypes' }),
|
||||
storageDriverFallback: await storageDriverFromConfig(config().storageDriverFallback, db, { assignDriverId: env !== 'buildTypes' }),
|
||||
};
|
||||
};
|
||||
|
||||
let runCommandAndExitApp = true;
|
||||
|
||||
if (selectedCommand) {
|
||||
@ -245,7 +237,7 @@ async function main() {
|
||||
});
|
||||
} else {
|
||||
const connectionCheck = await waitForConnection(config().database);
|
||||
const models = newModelFactory(connectionCheck.connection, config(), await newModelFactoryOptions(connectionCheck.connection));
|
||||
const models = newModelFactory(connectionCheck.connection, config());
|
||||
|
||||
await selectedCommand.run(commandArgv, {
|
||||
db: connectionCheck.connection,
|
||||
@ -275,7 +267,7 @@ async function main() {
|
||||
appLogger().info('Connection check:', connectionCheckLogInfo);
|
||||
const ctx = app.context as AppContext;
|
||||
|
||||
await setupAppContext(ctx, env, connectionCheck.connection, appLogger, await newModelFactoryOptions(connectionCheck.connection));
|
||||
await setupAppContext(ctx, env, connectionCheck.connection, appLogger);
|
||||
|
||||
await initializeJoplinUtils(config(), ctx.joplinBase.models, ctx.joplinBase.services.mustache);
|
||||
|
||||
|
||||
@ -5,10 +5,16 @@ export async function up(db: DbConnection): Promise<any> {
|
||||
await db.schema.createTable('storages', (table: Knex.CreateTableBuilder) => {
|
||||
table.increments('id').unique().primary().notNullable();
|
||||
table.text('connection_string').notNullable();
|
||||
table.bigInteger('updated_time').notNullable();
|
||||
table.bigInteger('created_time').notNullable();
|
||||
});
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
await db('storages').insert({
|
||||
connection_string: 'Type=Database',
|
||||
updated_time: now,
|
||||
created_time: now,
|
||||
});
|
||||
|
||||
// First we create the column and set a default so as to populate the
|
||||
@ -21,6 +27,10 @@ export async function up(db: DbConnection): Promise<any> {
|
||||
await db.schema.alterTable('items', (table: Knex.CreateTableBuilder) => {
|
||||
table.integer('content_storage_id').notNullable().alter();
|
||||
});
|
||||
|
||||
await db.schema.alterTable('storages', (table: Knex.CreateTableBuilder) => {
|
||||
table.unique(['connection_string']);
|
||||
});
|
||||
}
|
||||
|
||||
export async function down(db: DbConnection): Promise<any> {
|
||||
|
||||
@ -9,8 +9,9 @@ import { ChangePreviousItem } from './ChangeModel';
|
||||
import { unique } from '../utils/array';
|
||||
import StorageDriverBase, { Context } from './items/storage/StorageDriverBase';
|
||||
import { DbConnection } from '../db';
|
||||
import { Config, StorageDriverMode } from '../utils/types';
|
||||
import { NewModelFactoryHandler, Options } from './factory';
|
||||
import { Config, StorageDriverConfig, StorageDriverMode } from '../utils/types';
|
||||
import { NewModelFactoryHandler } from './factory';
|
||||
import storageDriverFromConfig from './items/storage/storageDriverFromConfig';
|
||||
|
||||
const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
|
||||
|
||||
@ -49,14 +50,16 @@ export interface ItemLoadOptions extends LoadOptions {
|
||||
export default class ItemModel extends BaseModel<Item> {
|
||||
|
||||
private updatingTotalSizes_: boolean = false;
|
||||
private storageDriver_: StorageDriverBase = null;
|
||||
private storageDriverFallback_: StorageDriverBase = null;
|
||||
private storageDriverConfig_: StorageDriverConfig;
|
||||
private storageDriverConfigFallback_: StorageDriverConfig;
|
||||
|
||||
public constructor(db: DbConnection, modelFactory: NewModelFactoryHandler, config: Config, options: Options) {
|
||||
private static storageDrivers_: Map<StorageDriverConfig, StorageDriverBase> = new Map();
|
||||
|
||||
public constructor(db: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
|
||||
super(db, modelFactory, config);
|
||||
|
||||
this.storageDriver_ = options.storageDriver;
|
||||
this.storageDriverFallback_ = options.storageDriverFallback;
|
||||
this.storageDriverConfig_ = config.storageDriver;
|
||||
this.storageDriverConfigFallback_ = config.storageDriverFallback;
|
||||
}
|
||||
|
||||
protected get tableName(): string {
|
||||
@ -75,6 +78,26 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
return Object.keys(databaseSchema[this.tableName]).filter(f => f !== 'content');
|
||||
}
|
||||
|
||||
private async storageDriverFromConfig(config: StorageDriverConfig): Promise<StorageDriverBase> {
|
||||
let driver = ItemModel.storageDrivers_.get(config);
|
||||
|
||||
if (!driver) {
|
||||
driver = await storageDriverFromConfig(config, this.db);
|
||||
ItemModel.storageDrivers_.set(config, driver);
|
||||
}
|
||||
|
||||
return driver;
|
||||
}
|
||||
|
||||
public async storageDriver(): Promise<StorageDriverBase> {
|
||||
return this.storageDriverFromConfig(this.storageDriverConfig_);
|
||||
}
|
||||
|
||||
public async storageDriverFallback(): Promise<StorageDriverBase> {
|
||||
if (!this.storageDriverConfigFallback_) return null;
|
||||
return this.storageDriverFromConfig(this.storageDriverConfigFallback_);
|
||||
}
|
||||
|
||||
public async checkIfAllowed(user: User, action: AclAction, resource: Item = null): Promise<void> {
|
||||
if (action === AclAction.Create) {
|
||||
if (!(await this.models().shareUser().isShareParticipant(resource.jop_share_id, user.id))) throw new ErrorForbidden('user has no access to this share');
|
||||
@ -136,25 +159,31 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
}
|
||||
|
||||
private async storageDriverWrite(itemId: Uuid, content: Buffer, context: Context) {
|
||||
await this.storageDriver_.write(itemId, content, context);
|
||||
const storageDriver = await this.storageDriver();
|
||||
const storageDriverFallback = await this.storageDriverFallback();
|
||||
|
||||
if (this.storageDriverFallback_) {
|
||||
if (this.storageDriverFallback_.mode === StorageDriverMode.ReadWrite) {
|
||||
await this.storageDriverFallback_.write(itemId, content, context);
|
||||
} else if (this.storageDriverFallback_.mode === StorageDriverMode.ReadOnly) {
|
||||
await this.storageDriverFallback_.write(itemId, Buffer.from(''), context);
|
||||
await storageDriver.write(itemId, content, context);
|
||||
|
||||
if (storageDriverFallback) {
|
||||
if (storageDriverFallback.mode === StorageDriverMode.ReadWrite) {
|
||||
await storageDriverFallback.write(itemId, content, context);
|
||||
} else if (storageDriverFallback.mode === StorageDriverMode.ReadOnly) {
|
||||
await storageDriverFallback.write(itemId, Buffer.from(''), context);
|
||||
} else {
|
||||
throw new Error(`Unsupported fallback mode: ${this.storageDriverFallback_.mode}`);
|
||||
throw new Error(`Unsupported fallback mode: ${storageDriverFallback.mode}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async storageDriverRead(itemId: Uuid, context: Context) {
|
||||
if (await this.storageDriver_.exists(itemId, context)) {
|
||||
return this.storageDriver_.read(itemId, context);
|
||||
const storageDriver = await this.storageDriver();
|
||||
const storageDriverFallback = await this.storageDriverFallback();
|
||||
|
||||
if (await storageDriver.exists(itemId, context)) {
|
||||
return storageDriver.read(itemId, context);
|
||||
} else {
|
||||
if (!this.storageDriverFallback_) throw new Error(`Content does not exist but fallback content driver is not defined: ${itemId}`);
|
||||
return this.storageDriverFallback_.read(itemId, context);
|
||||
if (!storageDriverFallback) throw new Error(`Content does not exist but fallback content driver is not defined: ${itemId}`);
|
||||
return storageDriverFallback.read(itemId, context);
|
||||
}
|
||||
}
|
||||
|
||||
@ -417,7 +446,8 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
try {
|
||||
const content = itemToSave.content;
|
||||
delete itemToSave.content;
|
||||
itemToSave.content_storage_id = this.storageDriver_.storageId;
|
||||
|
||||
itemToSave.content_storage_id = (await this.storageDriver()).storageId;
|
||||
|
||||
itemToSave.content_size = content ? content.byteLength : 0;
|
||||
|
||||
@ -624,14 +654,17 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
const ids = typeof id === 'string' ? [id] : id;
|
||||
if (!ids.length) return;
|
||||
|
||||
const storageDriver = await this.storageDriver();
|
||||
const storageDriverFallback = await this.storageDriverFallback();
|
||||
|
||||
const shares = await this.models().share().byItemIds(ids);
|
||||
|
||||
await this.withTransaction(async () => {
|
||||
await this.models().share().delete(shares.map(s => s.id));
|
||||
await this.models().userItem().deleteByItemIds(ids);
|
||||
await this.models().itemResource().deleteByItemIds(ids);
|
||||
await this.storageDriver_.delete(ids, { models: this.models() });
|
||||
if (this.storageDriverFallback_) await this.storageDriverFallback_.delete(ids, { models: this.models() });
|
||||
await storageDriver.delete(ids, { models: this.models() });
|
||||
if (storageDriverFallback) await storageDriverFallback.delete(ids, { models: this.models() });
|
||||
|
||||
await super.delete(ids, options);
|
||||
}, 'ItemModel::delete');
|
||||
@ -679,7 +712,7 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
let previousItem: ChangePreviousItem = null;
|
||||
|
||||
if (item.content && !item.content_storage_id) {
|
||||
item.content_storage_id = this.storageDriver_.storageId;
|
||||
item.content_storage_id = (await this.storageDriver()).storageId;
|
||||
}
|
||||
|
||||
if (isNew) {
|
||||
|
||||
@ -72,39 +72,29 @@ import SubscriptionModel from './SubscriptionModel';
|
||||
import UserFlagModel from './UserFlagModel';
|
||||
import EventModel from './EventModel';
|
||||
import { Config } from '../utils/types';
|
||||
import StorageDriverBase from './items/storage/StorageDriverBase';
|
||||
import LockModel from './LockModel';
|
||||
import StorageModel from './StorageModel';
|
||||
|
||||
export interface Options {
|
||||
storageDriver: StorageDriverBase;
|
||||
storageDriverFallback?: StorageDriverBase;
|
||||
}
|
||||
|
||||
export type NewModelFactoryHandler = (db: DbConnection)=> Models;
|
||||
|
||||
export class Models {
|
||||
|
||||
private db_: DbConnection;
|
||||
private config_: Config;
|
||||
private options_: Options;
|
||||
|
||||
public constructor(db: DbConnection, config: Config, options: Options) {
|
||||
public constructor(db: DbConnection, config: Config) {
|
||||
this.db_ = db;
|
||||
this.config_ = config;
|
||||
this.options_ = options;
|
||||
|
||||
// if (!options.storageDriver) throw new Error('StorageDriver is required');
|
||||
|
||||
this.newModelFactory = this.newModelFactory.bind(this);
|
||||
}
|
||||
|
||||
private newModelFactory(db: DbConnection) {
|
||||
return new Models(db, this.config_, this.options_);
|
||||
return new Models(db, this.config_);
|
||||
}
|
||||
|
||||
public item() {
|
||||
return new ItemModel(this.db_, this.newModelFactory, this.config_, this.options_);
|
||||
return new ItemModel(this.db_, this.newModelFactory, this.config_);
|
||||
}
|
||||
|
||||
public user() {
|
||||
@ -177,6 +167,6 @@ export class Models {
|
||||
|
||||
}
|
||||
|
||||
export default function newModelFactory(db: DbConnection, config: Config, options: Options): Models {
|
||||
return new Models(db, config, options);
|
||||
export default function newModelFactory(db: DbConnection, config: Config): Models {
|
||||
return new Models(db, config);
|
||||
}
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
import { clientType } from '../../../db';
|
||||
import { afterAllTests, beforeAllDb, beforeEachDb, db, expectNotThrow, expectThrow, models } from '../../../utils/testing/testUtils';
|
||||
import { StorageDriverMode } from '../../../utils/types';
|
||||
import { StorageDriverConfig, StorageDriverMode, StorageDriverType } from '../../../utils/types';
|
||||
import StorageDriverDatabase from './StorageDriverDatabase';
|
||||
import StorageDriverMemory from './StorageDriverMemory';
|
||||
import { shouldDeleteContent, shouldNotCreateItemIfContentNotSaved, shouldNotUpdateItemIfContentNotSaved, shouldSupportFallbackDriver, shouldSupportFallbackDriverInReadWriteMode, shouldUpdateContentStorageIdAfterSwitchingDriver, shouldWriteToContentAndReadItBack } from './testUtils';
|
||||
|
||||
const newDriver = () => {
|
||||
@ -11,6 +10,12 @@ const newDriver = () => {
|
||||
});
|
||||
};
|
||||
|
||||
const newConfig = (): StorageDriverConfig => {
|
||||
return {
|
||||
type: StorageDriverType.Database,
|
||||
};
|
||||
};
|
||||
|
||||
describe('StorageDriverDatabase', function() {
|
||||
|
||||
beforeAll(async () => {
|
||||
@ -26,23 +31,19 @@ describe('StorageDriverDatabase', function() {
|
||||
});
|
||||
|
||||
test('should write to content and read it back', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldWriteToContentAndReadItBack(driver);
|
||||
await shouldWriteToContentAndReadItBack(newConfig());
|
||||
});
|
||||
|
||||
test('should delete the content', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldDeleteContent(driver);
|
||||
await shouldDeleteContent(newConfig());
|
||||
});
|
||||
|
||||
test('should not create the item if the content cannot be saved', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldNotCreateItemIfContentNotSaved(driver);
|
||||
await shouldNotCreateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
test('should not update the item if the content cannot be saved', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldNotUpdateItemIfContentNotSaved(driver);
|
||||
await shouldNotUpdateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
test('should fail if the item row does not exist', async function() {
|
||||
@ -56,15 +57,15 @@ describe('StorageDriverDatabase', function() {
|
||||
});
|
||||
|
||||
test('should support fallback content drivers', async function() {
|
||||
await shouldSupportFallbackDriver(newDriver(), new StorageDriverMemory(2));
|
||||
await shouldSupportFallbackDriver(newConfig(), { type: StorageDriverType.Memory });
|
||||
});
|
||||
|
||||
test('should support fallback content drivers in rw mode', async function() {
|
||||
await shouldSupportFallbackDriverInReadWriteMode(newDriver(), new StorageDriverMemory(2, { mode: StorageDriverMode.ReadWrite }));
|
||||
await shouldSupportFallbackDriverInReadWriteMode(newConfig(), { type: StorageDriverType.Memory, mode: StorageDriverMode.ReadWrite });
|
||||
});
|
||||
|
||||
test('should update content storage ID after switching driver', async function() {
|
||||
await shouldUpdateContentStorageIdAfterSwitchingDriver(newDriver(), new StorageDriverMemory(2));
|
||||
await shouldUpdateContentStorageIdAfterSwitchingDriver(newConfig(), { type: StorageDriverType.Memory });
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { pathExists, remove } from 'fs-extra';
|
||||
import { afterAllTests, beforeAllDb, beforeEachDb, expectNotThrow, expectThrow, tempDirPath } from '../../../utils/testing/testUtils';
|
||||
import { StorageDriverConfig, StorageDriverType } from '../../../utils/types';
|
||||
import StorageDriverFs from './StorageDriverFs';
|
||||
import { shouldDeleteContent, shouldNotCreateItemIfContentNotSaved, shouldNotUpdateItemIfContentNotSaved, shouldWriteToContentAndReadItBack } from './testUtils';
|
||||
|
||||
@ -9,6 +10,13 @@ const newDriver = () => {
|
||||
return new StorageDriverFs(1, { path: basePath_ });
|
||||
};
|
||||
|
||||
const newConfig = (): StorageDriverConfig => {
|
||||
return {
|
||||
type: StorageDriverType.Filesystem,
|
||||
path: basePath_,
|
||||
};
|
||||
};
|
||||
|
||||
describe('StorageDriverFs', function() {
|
||||
|
||||
beforeAll(async () => {
|
||||
@ -30,23 +38,19 @@ describe('StorageDriverFs', function() {
|
||||
});
|
||||
|
||||
test('should write to content and read it back', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldWriteToContentAndReadItBack(driver);
|
||||
await shouldWriteToContentAndReadItBack(newConfig());
|
||||
});
|
||||
|
||||
test('should delete the content', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldDeleteContent(driver);
|
||||
await shouldDeleteContent(newConfig());
|
||||
});
|
||||
|
||||
test('should not create the item if the content cannot be saved', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldNotCreateItemIfContentNotSaved(driver);
|
||||
await shouldNotCreateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
test('should not update the item if the content cannot be saved', async function() {
|
||||
const driver = newDriver();
|
||||
await shouldNotUpdateItemIfContentNotSaved(driver);
|
||||
await shouldNotUpdateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
test('should write to a file and read it back', async function() {
|
||||
|
||||
@ -1,7 +1,13 @@
|
||||
import { afterAllTests, beforeAllDb, beforeEachDb } from '../../../utils/testing/testUtils';
|
||||
import StorageDriverMemory from './StorageDriverMemory';
|
||||
import { StorageDriverConfig, StorageDriverType } from '../../../utils/types';
|
||||
import { shouldDeleteContent, shouldNotCreateItemIfContentNotSaved, shouldNotUpdateItemIfContentNotSaved, shouldWriteToContentAndReadItBack } from './testUtils';
|
||||
|
||||
const newConfig = (): StorageDriverConfig => {
|
||||
return {
|
||||
type: StorageDriverType.Memory,
|
||||
};
|
||||
};
|
||||
|
||||
describe('StorageDriverMemory', function() {
|
||||
|
||||
beforeAll(async () => {
|
||||
@ -17,23 +23,19 @@ describe('StorageDriverMemory', function() {
|
||||
});
|
||||
|
||||
test('should write to content and read it back', async function() {
|
||||
const driver = new StorageDriverMemory(1);
|
||||
await shouldWriteToContentAndReadItBack(driver);
|
||||
await shouldWriteToContentAndReadItBack(newConfig());
|
||||
});
|
||||
|
||||
test('should delete the content', async function() {
|
||||
const driver = new StorageDriverMemory(1);
|
||||
await shouldDeleteContent(driver);
|
||||
await shouldDeleteContent(newConfig());
|
||||
});
|
||||
|
||||
test('should not create the item if the content cannot be saved', async function() {
|
||||
const driver = new StorageDriverMemory(1);
|
||||
await shouldNotCreateItemIfContentNotSaved(driver);
|
||||
await shouldNotCreateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
test('should not update the item if the content cannot be saved', async function() {
|
||||
const driver = new StorageDriverMemory(1);
|
||||
await shouldNotUpdateItemIfContentNotSaved(driver);
|
||||
await shouldNotUpdateItemIfContentNotSaved(newConfig());
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
@ -23,19 +23,19 @@ export default async function(config: StorageDriverConfig, db: DbConnection, opt
|
||||
let storageId: number = 0;
|
||||
|
||||
if (options.assignDriverId) {
|
||||
const models = newModelFactory(db, globalConfig(), { storageDriver: null });
|
||||
const models = newModelFactory(db, globalConfig());
|
||||
|
||||
const connectionString = serializeStorageConfig(config);
|
||||
const existingStorage = await models.storage().byConnectionString(connectionString);
|
||||
let storage = await models.storage().byConnectionString(connectionString);
|
||||
|
||||
if (existingStorage) {
|
||||
storageId = existingStorage.id;
|
||||
} else {
|
||||
const storage = await models.storage().save({
|
||||
if (!storage) {
|
||||
await models.storage().save({
|
||||
connection_string: connectionString,
|
||||
});
|
||||
storageId = storage.id;
|
||||
storage = await models.storage().byConnectionString(connectionString);
|
||||
}
|
||||
|
||||
storageId = storage.id;
|
||||
}
|
||||
|
||||
if (config.type === StorageDriverType.Database) {
|
||||
|
||||
@ -1,20 +1,30 @@
|
||||
import config from '../../../config';
|
||||
import { Item } from '../../../services/database/types';
|
||||
import { createUserAndSession, makeNoteSerializedBody, models } from '../../../utils/testing/testUtils';
|
||||
import { StorageDriverMode } from '../../../utils/types';
|
||||
import StorageDriverBase, { Context } from './StorageDriverBase';
|
||||
import { createUserAndSession, db, makeNoteSerializedBody, models } from '../../../utils/testing/testUtils';
|
||||
import { Config, StorageDriverConfig, StorageDriverMode } from '../../../utils/types';
|
||||
import newModelFactory from '../../factory';
|
||||
import { Context } from './StorageDriverBase';
|
||||
|
||||
const testModels = (driver: StorageDriverBase) => {
|
||||
return models({ storageDriver: driver });
|
||||
const newTestModels = (driverConfig: StorageDriverConfig, driverConfigFallback: StorageDriverConfig = null) => {
|
||||
const newConfig: Config = {
|
||||
...config(),
|
||||
storageDriver: driverConfig,
|
||||
storageDriverFallback: driverConfigFallback,
|
||||
};
|
||||
return newModelFactory(db(), newConfig);
|
||||
};
|
||||
|
||||
export async function shouldWriteToContentAndReadItBack(driver: StorageDriverBase) {
|
||||
export async function shouldWriteToContentAndReadItBack(driverConfig: StorageDriverConfig) {
|
||||
const { user } = await createUserAndSession(1);
|
||||
const noteBody = makeNoteSerializedBody({
|
||||
id: '00000000000000000000000000000001',
|
||||
title: 'testing driver',
|
||||
});
|
||||
|
||||
const output = await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const testModels = newTestModels(driverConfig);
|
||||
const driver = await testModels.item().storageDriver();
|
||||
|
||||
const output = await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBody),
|
||||
}]);
|
||||
@ -22,38 +32,43 @@ export async function shouldWriteToContentAndReadItBack(driver: StorageDriverBas
|
||||
const result = output['00000000000000000000000000000001.md'];
|
||||
expect(result.error).toBeFalsy();
|
||||
|
||||
const item = await testModels(driver).item().loadWithContent(result.item.id);
|
||||
const item = await testModels.item().loadWithContent(result.item.id);
|
||||
expect(item.content.byteLength).toBe(item.content_size);
|
||||
expect(item.content_storage_id).toBe(driver.storageId);
|
||||
|
||||
const rawContent = await driver.read(item.id, { models: models() });
|
||||
expect(rawContent.byteLength).toBe(item.content_size);
|
||||
|
||||
const jopItem = testModels(driver).item().itemToJoplinItem(item);
|
||||
const jopItem = testModels.item().itemToJoplinItem(item);
|
||||
expect(jopItem.id).toBe('00000000000000000000000000000001');
|
||||
expect(jopItem.title).toBe('testing driver');
|
||||
}
|
||||
|
||||
export async function shouldDeleteContent(driver: StorageDriverBase) {
|
||||
export async function shouldDeleteContent(driverConfig: StorageDriverConfig) {
|
||||
const { user } = await createUserAndSession(1);
|
||||
const noteBody = makeNoteSerializedBody({
|
||||
id: '00000000000000000000000000000001',
|
||||
title: 'testing driver',
|
||||
});
|
||||
|
||||
const output = await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const testModels = newTestModels(driverConfig);
|
||||
|
||||
const output = await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBody),
|
||||
}]);
|
||||
|
||||
const item: Item = output['00000000000000000000000000000001.md'].item;
|
||||
|
||||
expect((await testModels(driver).item().all()).length).toBe(1);
|
||||
await testModels(driver).item().delete(item.id);
|
||||
expect((await testModels(driver).item().all()).length).toBe(0);
|
||||
expect((await testModels.item().all()).length).toBe(1);
|
||||
await testModels.item().delete(item.id);
|
||||
expect((await testModels.item().all()).length).toBe(0);
|
||||
}
|
||||
|
||||
export async function shouldNotCreateItemIfContentNotSaved(driver: StorageDriverBase) {
|
||||
export async function shouldNotCreateItemIfContentNotSaved(driverConfig: StorageDriverConfig) {
|
||||
const testModels = newTestModels(driverConfig);
|
||||
const driver = await testModels.item().storageDriver();
|
||||
|
||||
const previousWrite = driver.write;
|
||||
driver.write = () => { throw new Error('not working!'); };
|
||||
|
||||
@ -64,26 +79,29 @@ export async function shouldNotCreateItemIfContentNotSaved(driver: StorageDriver
|
||||
title: 'testing driver',
|
||||
});
|
||||
|
||||
const output = await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const output = await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBody),
|
||||
}]);
|
||||
|
||||
expect(output['00000000000000000000000000000001.md'].error.message).toBe('not working!');
|
||||
expect((await testModels(driver).item().all()).length).toBe(0);
|
||||
expect((await testModels.item().all()).length).toBe(0);
|
||||
} finally {
|
||||
driver.write = previousWrite;
|
||||
}
|
||||
}
|
||||
|
||||
export async function shouldNotUpdateItemIfContentNotSaved(driver: StorageDriverBase) {
|
||||
export async function shouldNotUpdateItemIfContentNotSaved(driverConfig: StorageDriverConfig) {
|
||||
const { user } = await createUserAndSession(1);
|
||||
const noteBody = makeNoteSerializedBody({
|
||||
id: '00000000000000000000000000000001',
|
||||
title: 'testing driver',
|
||||
});
|
||||
|
||||
await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const testModels = newTestModels(driverConfig);
|
||||
const driver = await testModels.item().storageDriver();
|
||||
|
||||
await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBody),
|
||||
}]);
|
||||
@ -93,12 +111,12 @@ export async function shouldNotUpdateItemIfContentNotSaved(driver: StorageDriver
|
||||
title: 'updated 1',
|
||||
});
|
||||
|
||||
await testModels(driver).item().saveFromRawContent(user, [{
|
||||
await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBodyMod1),
|
||||
}]);
|
||||
|
||||
const itemMod1 = testModels(driver).item().itemToJoplinItem(await testModels(driver).item().loadByJopId(user.id, '00000000000000000000000000000001', { withContent: true }));
|
||||
const itemMod1 = testModels.item().itemToJoplinItem(await testModels.item().loadByJopId(user.id, '00000000000000000000000000000001', { withContent: true }));
|
||||
expect(itemMod1.title).toBe('updated 1');
|
||||
|
||||
const noteBodyMod2 = makeNoteSerializedBody({
|
||||
@ -110,23 +128,26 @@ export async function shouldNotUpdateItemIfContentNotSaved(driver: StorageDriver
|
||||
driver.write = () => { throw new Error('not working!'); };
|
||||
|
||||
try {
|
||||
const output = await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const output = await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(noteBodyMod2),
|
||||
}]);
|
||||
|
||||
expect(output['00000000000000000000000000000001.md'].error.message).toBe('not working!');
|
||||
const itemMod2 = testModels(driver).item().itemToJoplinItem(await testModels(driver).item().loadByJopId(user.id, '00000000000000000000000000000001', { withContent: true }));
|
||||
const itemMod2 = testModels.item().itemToJoplinItem(await testModels.item().loadByJopId(user.id, '00000000000000000000000000000001', { withContent: true }));
|
||||
expect(itemMod2.title).toBe('updated 1'); // Check it has not been updated
|
||||
} finally {
|
||||
driver.write = previousWrite;
|
||||
}
|
||||
}
|
||||
|
||||
export async function shouldSupportFallbackDriver(driver: StorageDriverBase, fallbackDriver: StorageDriverBase) {
|
||||
export async function shouldSupportFallbackDriver(driverConfig: StorageDriverConfig, fallbackDriverConfig: StorageDriverConfig) {
|
||||
const { user } = await createUserAndSession(1);
|
||||
|
||||
const output = await testModels(driver).item().saveFromRawContent(user, [{
|
||||
const testModels = newTestModels(driverConfig);
|
||||
const driver = await testModels.item().storageDriver();
|
||||
|
||||
const output = await testModels.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
body: Buffer.from(makeNoteSerializedBody({
|
||||
id: '00000000000000000000000000000001',
|
||||
@ -144,10 +165,7 @@ export async function shouldSupportFallbackDriver(driver: StorageDriverBase, fal
|
||||
previousByteLength = content.byteLength;
|
||||
}
|
||||
|
||||
const testModelWithFallback = models({
|
||||
storageDriver: driver,
|
||||
storageDriverFallback: fallbackDriver,
|
||||
});
|
||||
const testModelWithFallback = newTestModels(driverConfig, fallbackDriverConfig);
|
||||
|
||||
// If the item content is not on the main content driver, it should get
|
||||
// it from the fallback one.
|
||||
@ -165,6 +183,8 @@ export async function shouldSupportFallbackDriver(driver: StorageDriverBase, fal
|
||||
}]);
|
||||
|
||||
{
|
||||
const fallbackDriver = await testModelWithFallback.item().storageDriverFallback();
|
||||
|
||||
// Check that it has cleared the fallback driver content
|
||||
const context: Context = { models: models() };
|
||||
const fallbackContent = await fallbackDriver.read(itemId, context);
|
||||
@ -176,15 +196,12 @@ export async function shouldSupportFallbackDriver(driver: StorageDriverBase, fal
|
||||
}
|
||||
}
|
||||
|
||||
export async function shouldSupportFallbackDriverInReadWriteMode(driver: StorageDriverBase, fallbackDriver: StorageDriverBase) {
|
||||
if (fallbackDriver.mode !== StorageDriverMode.ReadWrite) throw new Error('Content driver must be configured in RW mode for this test');
|
||||
export async function shouldSupportFallbackDriverInReadWriteMode(driverConfig: StorageDriverConfig, fallbackDriverConfig: StorageDriverConfig) {
|
||||
if (fallbackDriverConfig.mode !== StorageDriverMode.ReadWrite) throw new Error('Content driver must be configured in RW mode for this test');
|
||||
|
||||
const { user } = await createUserAndSession(1);
|
||||
|
||||
const testModelWithFallback = models({
|
||||
storageDriver: driver,
|
||||
storageDriverFallback: fallbackDriver,
|
||||
});
|
||||
const testModelWithFallback = newTestModels(driverConfig, fallbackDriverConfig);
|
||||
|
||||
const output = await testModelWithFallback.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
@ -197,6 +214,9 @@ export async function shouldSupportFallbackDriverInReadWriteMode(driver: Storage
|
||||
const itemId = output['00000000000000000000000000000001.md'].item.id;
|
||||
|
||||
{
|
||||
const driver = await testModelWithFallback.item().storageDriver();
|
||||
const fallbackDriver = await testModelWithFallback.item().storageDriverFallback();
|
||||
|
||||
// Check that it has written the content to both drivers
|
||||
const context: Context = { models: models() };
|
||||
const fallbackContent = await fallbackDriver.read(itemId, context);
|
||||
@ -207,18 +227,15 @@ export async function shouldSupportFallbackDriverInReadWriteMode(driver: Storage
|
||||
}
|
||||
}
|
||||
|
||||
export async function shouldUpdateContentStorageIdAfterSwitchingDriver(oldDriver: StorageDriverBase, newDriver: StorageDriverBase) {
|
||||
if (oldDriver.storageId === newDriver.storageId) throw new Error('Drivers must be different for this test');
|
||||
export async function shouldUpdateContentStorageIdAfterSwitchingDriver(oldDriverConfig: StorageDriverConfig, newDriverConfig: StorageDriverConfig) {
|
||||
if (oldDriverConfig.type === newDriverConfig.type) throw new Error('Drivers must be different for this test');
|
||||
|
||||
const { user } = await createUserAndSession(1);
|
||||
|
||||
const oldDriverModel = models({
|
||||
storageDriver: oldDriver,
|
||||
});
|
||||
|
||||
const newDriverModel = models({
|
||||
storageDriver: newDriver,
|
||||
});
|
||||
const oldDriverModel = newTestModels(oldDriverConfig);
|
||||
const newDriverModel = newTestModels(newDriverConfig);
|
||||
const oldDriver = await oldDriverModel.item().storageDriver();
|
||||
const newDriver = await newDriverModel.item().storageDriver();
|
||||
|
||||
const output = await oldDriverModel.item().saveFromRawContent(user, [{
|
||||
name: '00000000000000000000000000000001.md',
|
||||
|
||||
@ -249,6 +249,8 @@ export interface Event extends WithUuid {
|
||||
export interface Storage {
|
||||
id?: number;
|
||||
connection_string?: string;
|
||||
updated_time?: string;
|
||||
created_time?: string;
|
||||
}
|
||||
|
||||
export interface Item extends WithDates, WithUuid {
|
||||
@ -427,6 +429,8 @@ export const databaseSchema: DatabaseTables = {
|
||||
storages: {
|
||||
id: { type: 'number' },
|
||||
connection_string: { type: 'string' },
|
||||
updated_time: { type: 'string' },
|
||||
created_time: { type: 'string' },
|
||||
},
|
||||
items: {
|
||||
id: { type: 'string' },
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import time from '@joplin/lib/time';
|
||||
import { DbConnection, dropTables, migrateLatest } from '../db';
|
||||
import newModelFactory from '../models/factory';
|
||||
import storageDriverFromConfig from '../models/items/storage/storageDriverFromConfig';
|
||||
import { AccountType } from '../models/UserModel';
|
||||
import { User, UserFlagType } from '../services/database/types';
|
||||
import { Config } from '../utils/types';
|
||||
@ -35,10 +34,7 @@ export async function createTestUsers(db: DbConnection, config: Config, options:
|
||||
|
||||
const password = 'hunter1hunter2hunter3';
|
||||
|
||||
const models = newModelFactory(db, config, {
|
||||
// storageDriver: new StorageDriverDatabase(1, { dbClientType: clientType(db) }),
|
||||
storageDriver: await storageDriverFromConfig(config.storageDriver, db), // new StorageDriverDatabase(1, { dbClientType: clientType(db) }),
|
||||
});
|
||||
const models = newModelFactory(db, config);
|
||||
|
||||
if (options.count) {
|
||||
const users: User[] = [];
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { LoggerWrapper } from '@joplin/lib/Logger';
|
||||
import config from '../config';
|
||||
import { DbConnection } from '../db';
|
||||
import newModelFactory, { Models, Options as ModelFactoryOptions } from '../models/factory';
|
||||
import newModelFactory, { Models } from '../models/factory';
|
||||
import { AppContext, Config, Env } from './types';
|
||||
import routes from '../routes/routes';
|
||||
import ShareService from '../services/ShareService';
|
||||
@ -23,8 +23,8 @@ async function setupServices(env: Env, models: Models, config: Config): Promise<
|
||||
return output;
|
||||
}
|
||||
|
||||
export default async function(appContext: AppContext, env: Env, dbConnection: DbConnection, appLogger: ()=> LoggerWrapper, options: ModelFactoryOptions): Promise<AppContext> {
|
||||
const models = newModelFactory(dbConnection, config(), options);
|
||||
export default async function(appContext: AppContext, env: Env, dbConnection: DbConnection, appLogger: ()=> LoggerWrapper): Promise<AppContext> {
|
||||
const models = newModelFactory(dbConnection, config());
|
||||
|
||||
// The joplinBase object is immutable because it is shared by all requests.
|
||||
// Then a "joplin" context property is created from it per request, which
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { DbConnection, connectDb, disconnectDb, truncateTables } from '../../db';
|
||||
import { User, Session, Item, Uuid } from '../../services/database/types';
|
||||
import { createDb, CreateDbOptions } from '../../tools/dbTools';
|
||||
import modelFactory, { Options as ModelFactoryOptions } from '../../models/factory';
|
||||
import modelFactory from '../../models/factory';
|
||||
import { AppContext, Env } from '../types';
|
||||
import config, { initConfig } from '../../config';
|
||||
import Logger from '@joplin/lib/Logger';
|
||||
@ -23,7 +23,6 @@ import MustacheService from '../../services/MustacheService';
|
||||
import uuidgen from '../uuidgen';
|
||||
import { createCsrfToken } from '../csrf';
|
||||
import { cookieSet } from '../cookies';
|
||||
import StorageDriverMemory from '../../models/items/storage/StorageDriverMemory';
|
||||
import { parseEnv } from '../../env';
|
||||
|
||||
// Takes into account the fact that this file will be inside the /dist directory
|
||||
@ -195,7 +194,7 @@ export async function koaAppContext(options: AppContextTestOptions = null): Prom
|
||||
|
||||
const appLogger = Logger.create('AppTest');
|
||||
|
||||
const baseAppContext = await setupAppContext({} as any, Env.Dev, db_, () => appLogger, { storageDriver: new StorageDriverMemory(1) });
|
||||
const baseAppContext = await setupAppContext({} as any, Env.Dev, db_, () => appLogger);
|
||||
|
||||
// Set type to "any" because the Koa context has many properties and we
|
||||
// don't need to mock all of them.
|
||||
@ -243,16 +242,8 @@ export function db() {
|
||||
return db_;
|
||||
}
|
||||
|
||||
const storageDriverMemory = new StorageDriverMemory(1);
|
||||
|
||||
export function models(options: ModelFactoryOptions = null) {
|
||||
options = {
|
||||
storageDriver: storageDriverMemory,
|
||||
storageDriverFallback: null,
|
||||
...options,
|
||||
};
|
||||
|
||||
return modelFactory(db(), config(), options);
|
||||
export function models() {
|
||||
return modelFactory(db(), config());
|
||||
}
|
||||
|
||||
export function parseHtml(html: string): Document {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user