self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

update

Browse Source
This commit is contained in:
Laurent Cozic 2024-05-07 23:12:50 +01:00
parent 6c99c1cae6
commit 3813448b1e
14 changed files with 99 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/server/src/config.ts
View File

@ -20,15 +20,17 @@ export function runningInDocker(): boolean {
return runningInDocker_;
}
function databaseHostFromEnv(runningInDocker: boolean, env: EnvVariables): string {
if (env.POSTGRES_HOST) {
function databaseHostFromEnv(runningInDocker: boolean, env: EnvVariables, replica: boolean): string {
const postgresHost = replica ? env.REPLICA_POSTGRES_HOST : env.POSTGRES_HOST;
if (postgresHost) {
// When running within Docker, the app localhost is different from the
// host's localhost. To access the latter, Docker defines a special host
// called "host.docker.internal", so here we swap the values if necessary.
if (runningInDocker && ['localhost', '127.0.0.1'].includes(env.POSTGRES_HOST)) {
if (runningInDocker && ['localhost', '127.0.0.1'].includes(postgresHost)) {
return 'host.docker.internal';
} else {
return env.POSTGRES_HOST;
return postgresHost;
}
}
@ -42,7 +44,7 @@ export const fullVersionString = (config: Config) => {
return output.join(' ');
};
function databaseConfigFromEnv(runningInDocker: boolean, env: EnvVariables): DatabaseConfig {
function databaseConfigFromEnv(runningInDocker: boolean, env: EnvVariables, replica: boolean): DatabaseConfig {
const baseConfig: DatabaseConfig = {
client: DatabaseConfigClient.Null,
name: '',
@ -59,16 +61,16 @@ function databaseConfigFromEnv(runningInDocker: boolean, env: EnvVariables): Dat
if (env.POSTGRES_CONNECTION_STRING) {
return {
...databaseConfig,
connectionString: env.POSTGRES_CONNECTION_STRING,
connectionString: replica ? env.REPLICA_POSTGRES_CONNECTION_STRING : env.POSTGRES_CONNECTION_STRING,
};
} else {
return {
...databaseConfig,
name: env.POSTGRES_DATABASE,
user: env.POSTGRES_USER,
password: env.POSTGRES_PASSWORD,
port: env.POSTGRES_PORT,
host: databaseHostFromEnv(runningInDocker, env) || 'localhost',
name: replica ? env.REPLICA_POSTGRES_DATABASE : env.POSTGRES_DATABASE,
user: replica ? env.REPLICA_POSTGRES_USER : env.POSTGRES_USER,
password: replica ? env.REPLICA_POSTGRES_PASSWORD : env.POSTGRES_PASSWORD,
port: replica ? env.REPLICA_POSTGRES_PORT : env.POSTGRES_PORT,
host: databaseHostFromEnv(runningInDocker, env, replica) || 'localhost',
};
}
}
@ -156,6 +158,7 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
const apiBaseUrl = env.API_BASE_URL ? env.API_BASE_URL : baseUrl;
const supportEmail = env.SUPPORT_EMAIL;
const forkVersion = packageJson.joplinServer?.forkVersion;
const dbConfig = databaseConfigFromEnv(runningInDocker_, env, false);
config_ = {
...env,
@ -169,7 +172,8 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
layoutDir: `${viewDir}/layouts`,
tempDir: `${rootDir}/temp`,
logDir: `${rootDir}/logs`,
database: databaseConfigFromEnv(runningInDocker_, env),
database: dbConfig,
databaseReplica: env.DB_USE_REPLICA ? databaseConfigFromEnv(runningInDocker_, env, true) : dbConfig,
mailer: mailerConfigFromEnv(env),
stripe: stripeConfigFromEnv(stripePublicConfig, env),
port: appPort,

16
packages/server/src/env.ts
View File

@ -59,6 +59,7 @@ const defaultEnvValues: EnvVariables = {
DB_SLOW_QUERY_LOG_MIN_DURATION: 1000,
DB_AUTO_MIGRATION: true,
DB_ALLOW_INCOMPLETE_MIGRATIONS: false,
DB_USE_REPLICA: false,
POSTGRES_PASSWORD: 'joplin',
POSTGRES_DATABASE: 'joplin',
@ -67,6 +68,13 @@ const defaultEnvValues: EnvVariables = {
POSTGRES_PORT: 5432,
POSTGRES_CONNECTION_STRING: '',
REPLICA_POSTGRES_PASSWORD: 'joplin',
REPLICA_POSTGRES_DATABASE: 'joplin',
REPLICA_POSTGRES_USER: 'joplin',
REPLICA_POSTGRES_HOST: '',
REPLICA_POSTGRES_PORT: 5432,
REPLICA_POSTGRES_CONNECTION_STRING: '',
// This must be the full path to the database file
SQLITE_DATABASE: '',
@ -157,6 +165,7 @@ export interface EnvVariables {
DB_SLOW_QUERY_LOG_MIN_DURATION: number;
DB_AUTO_MIGRATION: boolean;
DB_ALLOW_INCOMPLETE_MIGRATIONS: boolean;
DB_USE_REPLICA: boolean;
POSTGRES_PASSWORD: string;
POSTGRES_DATABASE: string;
@ -165,6 +174,13 @@ export interface EnvVariables {
POSTGRES_PORT: number;
POSTGRES_CONNECTION_STRING: string;
REPLICA_POSTGRES_PASSWORD: string;
REPLICA_POSTGRES_DATABASE: string;
REPLICA_POSTGRES_USER: string;
REPLICA_POSTGRES_HOST: string;
REPLICA_POSTGRES_PORT: number;
REPLICA_POSTGRES_CONNECTION_STRING: string;
SQLITE_DATABASE: string;
STORAGE_DRIVER: string;

10
packages/server/src/models/BaseModel.ts
View File

@ -64,15 +64,15 @@ export default abstract class BaseModel<T> {
private defaultFields_: string[] = [];
private db_: DbConnection;
private dbReader_: DbConnection;
private dbReplica_: DbConnection;
private transactionHandler_: TransactionHandler;
private modelFactory_: NewModelFactoryHandler;
private config_: Config;
private savePoints_: SavePoint[] = [];
public constructor(db: DbConnection, dbReader: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
public constructor(db: DbConnection, dbReplica: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
this.db_ = db;
this.dbReader_ = dbReader;
this.dbReplica_ = dbReplica;
this.modelFactory_ = modelFactory;
this.config_ = config;
@ -115,8 +115,8 @@ export default abstract class BaseModel<T> {
return this.db_;
}
public get dbReader(): DbConnection {
return this.dbReader_;
public get dbReplica(): DbConnection {
return this.dbReplica_;
}
protected get defaultFields(): string[] {

12
packages/server/src/models/ChangeModel.ts
View File

@ -57,8 +57,8 @@ export default class ChangeModel extends BaseModel<Change> {
public deltaIncludesItems_: boolean;
public constructor(db: DbConnection, dbReader: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReader, modelFactory, config);
public constructor(db: DbConnection, dbReplica: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReplica, modelFactory, config);
this.deltaIncludesItems_ = config.DELTA_INCLUDES_ITEMS;
}
@ -199,8 +199,8 @@ export default class ChangeModel extends BaseModel<Change> {
if (!doCountQuery) {
finalParams.push(limit);
if (isPostgres(this.dbReader)) {
query = this.dbReader.raw(`
if (isPostgres(this.dbReplica)) {
query = this.dbReplica.raw(`
WITH cte1 AS MATERIALIZED (
${subQuery1}
)
@ -214,7 +214,7 @@ export default class ChangeModel extends BaseModel<Change> {
LIMIT ?
`, finalParams);
} else {
query = this.dbReader.raw(`
query = this.dbReplica.raw(`
SELECT ${fieldsSql} FROM (${subQuery1}) as sub1
UNION ALL
SELECT ${fieldsSql} FROM (${subQuery2}) as sub2
@ -223,7 +223,7 @@ export default class ChangeModel extends BaseModel<Change> {
`, finalParams);
}
} else {
query = this.dbReader.raw(`
query = this.dbReplica.raw(`
SELECT count(*) as total
FROM (
(${subQuery1})

18
packages/server/src/models/ItemModel.test.ts
View File

@ -1,4 +1,4 @@
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItemTree, createResource, createNote, createItemTree3, db, tempDir, expectNotThrow, expectHttpError, dbReader } from '../utils/testing/testUtils';
import { createUserAndSession, beforeAllDb, afterAllTests, beforeEachDb, models, createItemTree, createResource, createNote, createItemTree3, db, tempDir, expectNotThrow, expectHttpError, dbReplica } from '../utils/testing/testUtils';
import { shareFolderWithUser } from '../utils/testing/shareApiUtils';
import { resourceBlobPath } from '../utils/joplinUtils';
import newModelFactory from './factory';
@ -275,7 +275,7 @@ describe('ItemModel', () => {
test('should respect the hard item size limit', async () => {
const { user: user1 } = await createUserAndSession(1);
let models = newModelFactory(db(), dbReader(), config());
let models = newModelFactory(db(), dbReplica(), config());
let result = await models.item().saveFromRawContent(user1, {
body: Buffer.from('1234'),
@ -285,7 +285,7 @@ describe('ItemModel', () => {
const item = result['test1.txt'].item;
config().itemSizeHardLimit = 3;
models = newModelFactory(db(), dbReader(), config());
models = newModelFactory(db(), dbReplica(), config());
result = await models.item().saveFromRawContent(user1, {
body: Buffer.from('1234'),
@ -297,7 +297,7 @@ describe('ItemModel', () => {
await expectHttpError(async () => models.item().loadWithContent(item.id), ErrorPayloadTooLarge.httpCode);
config().itemSizeHardLimit = 1000;
models = newModelFactory(db(), dbReader(), config());
models = newModelFactory(db(), dbReplica(), config());
await expectNotThrow(async () => models.item().loadWithContent(item.id));
});
@ -316,18 +316,18 @@ describe('ItemModel', () => {
path: tempDir2,
};
const fromModels = newModelFactory(db(), dbReader(), {
const fromModels = newModelFactory(db(), dbReplica(), {
...config(),
storageDriver: fromStorageConfig,
});
const toModels = newModelFactory(db(), dbReader(), {
const toModels = newModelFactory(db(), dbReplica(), {
...config(),
storageDriver: toStorageConfig,
});
const fromDriver = await loadStorageDriver(fromStorageConfig, db(), dbReader());
const toDriver = await loadStorageDriver(toStorageConfig, db(), dbReader());
const fromDriver = await loadStorageDriver(fromStorageConfig, db(), dbReplica());
const toDriver = await loadStorageDriver(toStorageConfig, db(), dbReplica());
return {
fromStorageConfig,
@ -364,7 +364,7 @@ describe('ItemModel', () => {
await msleep(2);
const toModels = newModelFactory(db(), dbReader(), {
const toModels = newModelFactory(db(), dbReplica(), {
...config(),
storageDriver: toStorageConfig,
});

8
packages/server/src/models/ItemModel.ts
View File

@ -75,8 +75,8 @@ export default class ItemModel extends BaseModel<Item> {
private static storageDrivers_: Map<StorageDriverConfig, StorageDriverBase> = new Map();
public constructor(db: DbConnection, dbReader: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReader, modelFactory, config);
public constructor(db: DbConnection, dbReplica: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReplica, modelFactory, config);
this.storageDriverConfig_ = config.storageDriver;
this.storageDriverConfigFallback_ = config.storageDriverFallback;
@ -102,7 +102,7 @@ export default class ItemModel extends BaseModel<Item> {
let driver = ItemModel.storageDrivers_.get(config);
if (!driver) {
driver = await loadStorageDriver(config, this.db, this.dbReader);
driver = await loadStorageDriver(config, this.db, this.dbReplica);
ItemModel.storageDrivers_.set(config, driver);
}
@ -331,7 +331,7 @@ export default class ItemModel extends BaseModel<Item> {
let fromDriver: StorageDriverBase = drivers[item.content_storage_id];
if (!fromDriver) {
fromDriver = await loadStorageDriver(item.content_storage_id, this.db, this.dbReader);
fromDriver = await loadStorageDriver(item.content_storage_id, this.db, this.dbReplica);
drivers[item.content_storage_id] = fromDriver;
}

4
packages/server/src/models/UserModel.ts
View File

@ -118,8 +118,8 @@ export default class UserModel extends BaseModel<User> {
private ldapConfig_: LdapConfig[];
public constructor(db: DbConnection, dbReader: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReader, modelFactory, config);
public constructor(db: DbConnection, dbReplica: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, dbReplica, modelFactory, config);
this.ldapConfig_ = config.ldap;
}

54
packages/server/src/models/factory.ts
View File

@ -83,107 +83,107 @@ export type NewModelFactoryHandler = (db: DbConnection)=> Models;
export class Models {
private db_: DbConnection;
private dbReader_: DbConnection;
private dbReplica_: DbConnection;
private config_: Config;
public constructor(db: DbConnection, dbReader_: DbConnection, config: Config) {
public constructor(db: DbConnection, dbReplica_: DbConnection, config: Config) {
this.db_ = db;
this.dbReader_ = dbReader_;
this.dbReplica_ = dbReplica_;
this.config_ = config;
this.newModelFactory = this.newModelFactory.bind(this);
}
private newModelFactory(db: DbConnection) {
return new Models(db, this.dbReader_, this.config_);
return new Models(db, this.dbReplica_, this.config_);
}
public item() {
return new ItemModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ItemModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public user() {
return new UserModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new UserModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public email() {
return new EmailModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new EmailModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public userItem() {
return new UserItemModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new UserItemModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public token() {
return new TokenModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new TokenModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public itemResource() {
return new ItemResourceModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ItemResourceModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public apiClient() {
return new ApiClientModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ApiClientModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public session() {
return new SessionModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new SessionModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public change() {
return new ChangeModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ChangeModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public notification() {
return new NotificationModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new NotificationModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public share() {
return new ShareModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ShareModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public shareUser() {
return new ShareUserModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new ShareUserModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public keyValue() {
return new KeyValueModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new KeyValueModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public subscription() {
return new SubscriptionModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new SubscriptionModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public userFlag() {
return new UserFlagModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new UserFlagModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public event() {
return new EventModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new EventModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public lock() {
return new LockModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new LockModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public storage() {
return new StorageModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new StorageModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public userDeletion() {
return new UserDeletionModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new UserDeletionModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public backupItem() {
return new BackupItemModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new BackupItemModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
public taskState() {
return new TaskStateModel(this.db_, this.dbReader_, this.newModelFactory, this.config_);
return new TaskStateModel(this.db_, this.dbReplica_, this.newModelFactory, this.config_);
}
}
export default function newModelFactory(db: DbConnection, dbReader: DbConnection, config: Config): Models {
return new Models(db, dbReader, config);
export default function newModelFactory(db: DbConnection, dbReplica: DbConnection, config: Config): Models {
return new Models(db, dbReplica, config);
}

6
packages/server/src/models/items/storage/loadStorageDriver.test.ts
View File

@ -1,4 +1,4 @@
import { afterAllTests, beforeAllDb, beforeEachDb, db, dbReader, expectThrow, models } from '../../../utils/testing/testUtils';
import { afterAllTests, beforeAllDb, beforeEachDb, db, dbReplica, expectThrow, models } from '../../../utils/testing/testUtils';
import { StorageDriverType } from '../../../utils/types';
import loadStorageDriver from './loadStorageDriver';
@ -18,13 +18,13 @@ describe('loadStorageDriver', () => {
test('should load a driver and assign an ID to it', async () => {
{
const newDriver = await loadStorageDriver({ type: StorageDriverType.Memory }, db(), dbReader());
const newDriver = await loadStorageDriver({ type: StorageDriverType.Memory }, db(), dbReplica());
expect(newDriver.storageId).toBe(1);
expect((await models().storage().count())).toBe(1);
}
{
const newDriver = await loadStorageDriver({ type: StorageDriverType.Filesystem, path: '/just/testing' }, db(), dbReader());
const newDriver = await loadStorageDriver({ type: StorageDriverType.Filesystem, path: '/just/testing' }, db(), dbReplica());
expect(newDriver.storageId).toBe(2);
expect((await models().storage().count())).toBe(2);
}

6
packages/server/src/models/items/storage/loadStorageDriver.ts
View File

@ -14,7 +14,7 @@ export interface Options {
assignDriverId?: boolean;
}
export default async function(config: StorageDriverConfig | number, db: DbConnection, dbReader: DbConnection, options: Options = null): Promise<StorageDriverBase | null> {
export default async function(config: StorageDriverConfig | number, db: DbConnection, dbReplica: DbConnection, options: Options = null): Promise<StorageDriverBase | null> {
if (!config) return null;
options = {
@ -27,14 +27,14 @@ export default async function(config: StorageDriverConfig | number, db: DbConnec
if (typeof config === 'number') {
storageId = config;
const models = newModelFactory(db, dbReader, globalConfig());
const models = newModelFactory(db, dbReplica, globalConfig());
const storage = await models.storage().byId(storageId);
if (!storage) throw new Error(`No such storage ID: ${storageId}`);
config = parseStorageDriverConnectionString(storage.connection_string);
} else {
if (options.assignDriverId) {
const models = newModelFactory(db, dbReader, globalConfig());
const models = newModelFactory(db, dbReplica, globalConfig());
const connectionString = serializeStorageConfig(config);
let storage = await models.storage().byConnectionString(connectionString);

6
packages/server/src/models/items/storage/testUtils.ts
View File

@ -3,7 +3,7 @@
import config from '../../../config';
import { Item } from '../../../services/database/types';
import { CustomErrorCode } from '../../../utils/errors';
import { createUserAndSession, db, dbReader, makeNoteSerializedBody, models } from '../../../utils/testing/testUtils';
import { createUserAndSession, db, dbReplica, makeNoteSerializedBody, models } from '../../../utils/testing/testUtils';
import { Config, StorageDriverConfig, StorageDriverMode } from '../../../utils/types';
import newModelFactory from '../../factory';
import loadStorageDriver from './loadStorageDriver';
@ -15,7 +15,7 @@ const newTestModels = (driverConfig: StorageDriverConfig, driverConfigFallback:
storageDriver: driverConfig,
storageDriverFallback: driverConfigFallback,
};
return newModelFactory(db(), dbReader(), newConfig);
return newModelFactory(db(), dbReplica(), newConfig);
};
export function shouldWriteToContentAndReadItBack(driverConfig: StorageDriverConfig) {
@ -281,7 +281,7 @@ export function shouldUpdateContentStorageIdAfterSwitchingDriver(oldDriverConfig
export function shouldThrowNotFoundIfNotExist(driverConfig: StorageDriverConfig) {
test('should throw not found if item does not exist', async () => {
const driver = await loadStorageDriver(driverConfig, db(), dbReader());
const driver = await loadStorageDriver(driverConfig, db(), dbReplica());
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
let error: any = null;

4
packages/server/src/utils/storageConnectionCheck.ts
View File

@ -6,12 +6,12 @@ import { Context } from '../models/items/storage/StorageDriverBase';
import { StorageDriverConfig, StorageDriverType } from './types';
import { uuidgen } from '@joplin/lib/uuid';
export default async function(connection: string | StorageDriverConfig, db: DbConnection, dbReader: DbConnection, models: Models): Promise<string> {
export default async function(connection: string | StorageDriverConfig, db: DbConnection, dbReplica: DbConnection, models: Models): Promise<string> {
const storageConfig = typeof connection === 'string' ? parseStorageConnectionString(connection) : connection;
if (storageConfig.type === StorageDriverType.Database) return 'Database storage is special and cannot be checked this way. If the connection to the database was successful then the storage driver should work too.';
const driver = await loadStorageDriver(storageConfig, db, dbReader, { assignDriverId: false });
const driver = await loadStorageDriver(storageConfig, db, dbReplica, { assignDriverId: false });
const itemId = `testingconnection${uuidgen(8)}`;
const itemContent = Buffer.from(uuidgen(8));
const context: Context = { models };

4
packages/server/src/utils/testing/testUtils.ts
View File

@ -257,12 +257,12 @@ export function db() {
return db_;
}
export function dbReader() {
export function dbReplica() {
return db_;
}
export function models() {
return modelFactory(db(), dbReader(), config());
return modelFactory(db(), dbReplica(), config());
}
export function parseHtml(html: string): Document {

1
packages/server/src/utils/types.ts
View File

@ -165,6 +165,7 @@ export interface Config extends EnvVariables {
accountTypesEnabled: boolean;
showErrorStackTraces: boolean;
database: DatabaseConfig;
databaseReplica: DatabaseConfig;
mailer: MailerConfig;
stripe: StripeConfig;
supportEmail: string;