From 0d2bf6d787c7466b44024ce236fbda68c784f0f3 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 18 Jan 2021 10:13:26 +0000 Subject: [PATCH] Server: Improved config and support for Docker --- .env-sample | 29 ++++- Dockerfile.db | 3 - Dockerfile.server | 35 +++--- docker-compose.db-dev.yml | 15 +++ docker-compose.server-dev.yml | 33 +++--- docker-compose.server.yml | 60 +++++----- package.json | 1 + packages/server/README.md | 72 +++++++----- packages/server/package-lock.json | 7 +- packages/server/package.json | 3 +- packages/server/src/app.ts | 65 +++++++---- packages/server/src/config-base.ts | 21 ---- packages/server/src/config-buildTypes.ts | 13 --- packages/server/src/config-dev.ts | 22 ---- packages/server/src/config-prod.ts | 20 ---- packages/server/src/config-tests.ts | 13 --- packages/server/src/config.ts | 103 ++++++++++++++---- packages/server/src/db.ts | 6 +- .../src/middleware/notificationHandler.ts | 97 +++++++++++------ .../server/src/middleware/routeHandler.ts | 1 + packages/server/src/routes/default.ts | 4 + packages/server/src/routes/index/files.ts | 6 +- packages/server/src/routes/index/login.ts | 4 +- packages/server/src/routes/index/logout.ts | 4 +- packages/server/src/routes/index/users.ts | 10 +- .../server/src/services/MustacheService.ts | 6 +- packages/server/src/tools/dbTools.ts | 4 +- .../server/src/utils/testing/testUtils.ts | 31 +++--- packages/server/src/utils/types.ts | 10 +- .../server/src/views/layouts/default.mustache | 2 +- .../src/views/partials/notifications.mustache | 14 ++- packages/tools/build-plugin-repository.ts | 2 - packages/tools/release-server.ts | 37 +++---- packages/tools/tool-utils.js | 24 ++++ 34 files changed, 448 insertions(+), 329 deletions(-) delete mode 100644 Dockerfile.db create mode 100644 docker-compose.db-dev.yml delete mode 100644 packages/server/src/config-base.ts delete mode 100644 packages/server/src/config-buildTypes.ts delete mode 100644 packages/server/src/config-dev.ts delete mode 100644 packages/server/src/config-prod.ts delete mode 100644 packages/server/src/config-tests.ts diff --git a/.env-sample b/.env-sample index 7c2021c6e9..9a79ceba6f 100644 --- a/.env-sample +++ b/.env-sample @@ -1,9 +1,26 @@ -# Example of local config, for development: +# ============================================================================= +# PRODUCTION CONFIG EXAMPLE +# ----------------------------------------------------------------------------- +# By default it will use SQLite, but that's mostly to test and evaluate the +# server. So you'll want to specify db connection settings to use Postgres. +# ============================================================================= # -# JOPLIN_BASE_URL=http://localhost:22300 -# JOPLIN_PORT=22300 +# APP_BASE_URL=https://example.com/joplin +# APP_PORT=22300 +# +# DB_CLIENT=pg +# POSTGRES_PASSWORD=joplin +# POSTGRES_DATABASE=joplin +# POSTGRES_USER=joplin +# POSTGRES_PORT=5432 +# POSTGRES_HOST=localhost -# Example of config for production: +# ============================================================================= +# DEV CONFIG EXAMPLE +# ----------------------------------------------------------------------------- +# Example of local config, for development. In dev mode, you would usually use +# SQLite so database settings are not needed. +# ============================================================================= # -# JOPLIN_BASE_URL=https://example.com/joplin -# JOPLIN_PORT=22300 \ No newline at end of file +# APP_BASE_URL=http://localhost:22300 +# APP_PORT=22300 diff --git a/Dockerfile.db b/Dockerfile.db deleted file mode 100644 index 4d9b4ab9e4..0000000000 --- a/Dockerfile.db +++ /dev/null @@ -1,3 +0,0 @@ -FROM postgres:13.1 - -EXPOSE 5432 \ No newline at end of file diff --git a/Dockerfile.server b/Dockerfile.server index 4aa601045e..878eb825ae 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -16,8 +16,15 @@ WORKDIR /home/$user RUN mkdir /home/$user/logs +# Install the root scripts but don't run postinstall (which would bootstrap +# and build TypeScript files, but we don't have the TypeScript files at +# this point) + +COPY --chown=$user:$user package*.json ./ +RUN npm install --ignore-scripts + # To take advantage of the Docker cache, we first copy all the package.json -# and package-lock.json files, as they rarely change? and then bootstrap +# and package-lock.json files, as they rarely change, and then bootstrap # all the packages. # # Note that bootstrapping the packages will run all the postinstall @@ -27,19 +34,10 @@ RUN mkdir /home/$user/logs # We can't run boostrap with "--ignore-scripts" because that would # prevent certain sub-packages, such as sqlite3, from being built -COPY --chown=$user:$user package*.json ./ - -# Install the root scripts but don't run postinstall (which would bootstrap -# and build TypeScript files, but we don't have the TypeScript files at -# this point) - -RUN npm install --ignore-scripts - COPY --chown=$user:$user packages/fork-sax/package*.json ./packages/fork-sax/ -COPY --chown=$user:$user packages/lib/package*.json ./packages/lib/ COPY --chown=$user:$user packages/renderer/package*.json ./packages/renderer/ COPY --chown=$user:$user packages/tools/package*.json ./packages/tools/ -COPY --chown=$user:$user packages/server/package*.json ./packages/server/ +COPY --chown=$user:$user packages/lib/package*.json ./packages/lib/ COPY --chown=$user:$user lerna.json . COPY --chown=$user:$user tsconfig.json . @@ -50,22 +48,29 @@ COPY --chown=$user:$user packages/turndown ./packages/turndown COPY --chown=$user:$user packages/turndown-plugin-gfm ./packages/turndown-plugin-gfm COPY --chown=$user:$user packages/fork-htmlparser2 ./packages/fork-htmlparser2 -RUN ls -la /home/$user - # Then bootstrap only, without compiling the TypeScript files RUN npm run bootstrap +# We have a separate step for the server files because they are more likely to +# change. + +COPY --chown=$user:$user packages/server/package*.json ./packages/server/ +RUN npm run bootstrapServerOnly + +# Now copy the source files. Put lib and server last as they are more likely to change. + COPY --chown=$user:$user packages/fork-sax ./packages/fork-sax -COPY --chown=$user:$user packages/lib ./packages/lib COPY --chown=$user:$user packages/renderer ./packages/renderer COPY --chown=$user:$user packages/tools ./packages/tools +COPY --chown=$user:$user packages/lib ./packages/lib COPY --chown=$user:$user packages/server ./packages/server # Finally build everything, in particular the TypeScript files. RUN npm run build -EXPOSE ${JOPLIN_PORT} +ENV RUNNING_IN_DOCKER=1 +EXPOSE ${APP_PORT} CMD [ "npm", "--prefix", "packages/server", "start" ] diff --git a/docker-compose.db-dev.yml b/docker-compose.db-dev.yml new file mode 100644 index 0000000000..f14b626d75 --- /dev/null +++ b/docker-compose.db-dev.yml @@ -0,0 +1,15 @@ +# For development this compose file starts the database only. The app can then +# be started using `npm run start-dev`, which is useful for development, because +# it means the app Docker file doesn't have to be rebuilt on each change. + +version: '3' + +services: + db: + image: postgres:13.1 + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=joplin + - POSTGRES_USER=joplin + - POSTGRES_DB=joplin diff --git a/docker-compose.server-dev.yml b/docker-compose.server-dev.yml index d1b30d8652..ccc9a47588 100644 --- a/docker-compose.server-dev.yml +++ b/docker-compose.server-dev.yml @@ -1,28 +1,27 @@ -# For development, the easiest might be to only start the Postgres container and -# run the app directly with `npm start`. Or use sqlite3. +# This compose file can be used in development to run both the database and app +# within Docker. version: '3' services: - # app: - # build: - # context: . - # dockerfile: Dockerfile.server-dev - # ports: - # - "22300:22300" - # # volumes: - # # - ./packages/server/:/var/www/joplin/packages/server/ - # # - /var/www/joplin/packages/server/node_modules/ - db: + app: build: context: . - dockerfile: Dockerfile.db + dockerfile: Dockerfile.server + ports: + - "22300:22300" + environment: + - DB_CLIENT=pg + - POSTGRES_PASSWORD=joplin + - POSTGRES_DATABASE=joplin + - POSTGRES_USER=joplin + - POSTGRES_PORT=5432 + - POSTGRES_HOST=localhost + db: + image: postgres:13.1 ports: - "5432:5432" environment: - # TODO: Considering the database is only exposed to the - # application, and not to the outside world, is there a need to - # pick a secure password? - POSTGRES_PASSWORD=joplin - POSTGRES_USER=joplin - - POSTGRES_DB=joplin \ No newline at end of file + - POSTGRES_DB=joplin diff --git a/docker-compose.server.yml b/docker-compose.server.yml index da978f2ef0..6858eca85e 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -1,40 +1,34 @@ +# This is a sample docker-compose file that can be used to run Joplin Server +# along with a PostgreSQL server. +# +# All environment variables are optional. If you don't set them, you will get a +# warning from docker-compose, however the app should use working defaults. + version: '3' services: - app: - environment: - - JOPLIN_BASE_URL=${JOPLIN_BASE_URL} - - JOPLIN_PORT=${JOPLIN_PORT} - restart: unless-stopped - build: - context: . - dockerfile: Dockerfile.server - ports: - - "${JOPLIN_PORT}:${JOPLIN_PORT}" - # volumes: - # # Mount the server directory so that it's possible to edit file - # # while the container is running. However don't mount the - # # node_modules directory which will be specific to the Docker - # # image (eg native modules will be built for Ubuntu, while the - # # container might be running in Windows) - # # https://stackoverflow.com/a/37898591/561309 - # - ./packages/server:/home/joplin/packages/server - # - /home/joplin/packages/server/node_modules/ db: - restart: unless-stopped - # By default, the Postgres image saves the data to a Docker volume, - # so it persists whenever the server is restarted using - # `docker-compose up`. Note that it would however be deleted when - # running `docker-compose down`. - build: - context: . - dockerfile: Dockerfile.db + image: postgres:13.1 ports: - "5432:5432" + restart: unless-stopped environment: - # TODO: Considering the database is only exposed to the - # application, and not to the outside world, is there a need to - # pick a secure password? - - POSTGRES_PASSWORD=joplin - - POSTGRES_USER=joplin - - POSTGRES_DB=joplin \ No newline at end of file + - APP_PORT=22300 + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_DB=${POSTGRES_DATABASE} + app: + image: joplin/server:latest + depends_on: + - db + ports: + - "22300:22300" + restart: unless-stopped + environment: + - APP_BASE_URL=${APP_BASE_URL} + - DB_CLIENT=pg + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DATABASE=${POSTGRES_DATABASE} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_HOST=db \ No newline at end of file diff --git a/package.json b/package.json index c96392de19..fb0c31702d 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "license": "MIT", "scripts": { "bootstrap": "lerna bootstrap --no-ci", + "bootstrapServerOnly": "lerna bootstrap --no-ci --include-dependents --include-dependencies --scope @joplin/server", "bootstrapIgnoreScripts": "lerna bootstrap --ignore-scripts --no-ci", "build": "lerna run build && npm run tsc", "buildApiDoc": "npm start --prefix=packages/app-cli -- apidoc ../../readme/api/references/rest_api.md", diff --git a/packages/server/README.md b/packages/server/README.md index c780c319f7..f4c5ff6a7a 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -4,64 +4,86 @@ First copy `.env-sample` to `.env` and edit the values in there: -- `JOPLIN_BASE_URL`: This is the base public URL where the service will be running. For example, if you want it to run from `https://example.com/joplin`, this is what you should set the URL to. The base URL can include the port. -- `JOPLIN_PORT`: The local port on which the Docker container will listen. You would typically map this port to 443 (TLS) with a reverse proxy. +- `APP_BASE_URL`: This is the base public URL where the service will be running. For example, if you want it to run from `https://example.com/joplin`, this is what you should set the URL to. The base URL can include the port. +- `APP_PORT`: The local port on which the Docker container will listen. You would typically map this port to 443 (TLS) with a reverse proxy. -## Install application +## Running the server + +To start the server with default configuration, run: ```shell -wget https://github.com/laurent22/joplin/archive/server-v1.6.4.tar.gz -tar xzvf server-v1.6.4.tar.gz -mv joplin-server-v1.6.4 joplin-server -cd joplin-server -docker-compose --file docker-compose.server.yml up --detach +docker run --env-file .env -p 22300:22300 joplin/server:latest ``` -This will start the server, which will listen on port **22300** on **localhost**. +This will start the server, which will listen on port **22300** on **localhost**. By default it will use SQLite, which allows you to test the app without setting up a database. To run it for production though, you'll want to connect the container to a database, as described below. -Due to the restart policy defined in the docker-compose file, the server will be restarted automatically whenever the host reboots. +## Setup the database + +You can setup the container to either use an existing PostgreSQL server, or connect it to a new one using docker-compose + +### Using an existing PostgreSQL server + +To use an existing PostgresSQL server, set the following environment variables in the .env file: + +```conf +DB_CLIENT=pg +POSTGRES_PASSWORD=joplin +POSTGRES_DATABASE=joplin +POSTGRES_USER=joplin +POSTGRES_PORT=5432 +POSTGRES_HOST=localhost +``` + +Make sure that the provided database and user exist as the server will not create them. + +### Using docker-compose + +A [sample docker-compose file](https://github.com/laurent22/joplin/blob/dev/docker-compose.server.yml + ) is available to show how to use Docker to install both the database and server and connect them: ## Setup reverse proxy -You will then need to expose this server to the internet by setting up a reverse proxy, and that will depend on how your server is currently configured, and whether you already have Nginx or Apache running: +Once Joplin Server is running, you will then need to expose it to the internet by setting up a reverse proxy, and that will depend on how your server is currently configured, and whether you already have Nginx or Apache running: - [Apache Reverse Proxy](https://httpd.apache.org/docs/current/mod/mod_proxy.html) - [Nginx Reverse Proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) -## Setup admin user +## Setup the website -For the following instructions, we'll assume that the Joplin server is running on `https://example.com/joplin`. +Once the server is exposed to the internet, you can open the admin UI and get it ready for synchronisation. For the following instructions, we'll assume that the Joplin server is running on `https://example.com/joplin`. -By default, the instance will be setup with an admin user with email **admin@localhost** and password **admin** and you should change this by opening the admin UI. To do so, open `https://example.com/joplin/login`. From there, go to Profile and change the admin password. +### Secure the admin user -## Setup a user for sync +By default, the instance will be setup with an admin user with email **admin@localhost** and password **admin** and you should change this. To do so, open `https://example.com/joplin/login` and login as admin. Then go to the Profile section and change the admin password. -While the admin user can be used for synchronisation, it is recommended to create a separate non-admin user for it. To do, open the admin UI and navigate to the Users page - from there you can create a new user. +### Create a user for sync -Once this is done, you can use the email and password you specified to sync this user account with your Joplin clients. +While the admin user can be used for synchronisation, it is recommended to create a separate non-admin user for it. To do so, navigate to the Users page - from there you can create a new user. Once this is done, you can use the email and password you specified to sync this user account with your Joplin clients. ## Checking the logs Checking the log can be done the standard Docker way: -```shell +```bash +# With Docker: +docker logs --follow CONTAINER + +# With docker-compose: docker-compose --file docker-compose.server.yml logs ``` -# Set up for development +# Setup for development -## Setting up the database +## Setup up the database ### SQLite -The server supports SQLite for development and test units. To use it, open `src/config-dev.ts` and uncomment the sqlite3 config. +By default the server supports SQLite for development, so nothing needs to be setup. ### PostgreSQL -It's best to use PostgreSQL as this is what is used in production, however it requires Docker. - -To use it, from the monorepo root, run `docker-compose --file docker-compose.server-dev.yml up`, which will start the PostgreSQL database. +To use Postgres, from the monorepo root, run `docker-compose --file docker-compose.server-dev.yml up`, which will start the PostgreSQL database. ## Starting the server -From `packages/server`, run `npm run start-dev` \ No newline at end of file +From `packages/server`, run `npm run start-dev` diff --git a/packages/server/package-lock.json b/packages/server/package-lock.json index fa2086edcc..152eef0b05 100644 --- a/packages/server/package-lock.json +++ b/packages/server/package-lock.json @@ -1,6 +1,6 @@ { "name": "@joplin/server", - "version": "1.7.0", + "version": "1.7.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -5938,6 +5938,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-env-file": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/node-env-file/-/node-env-file-0.1.8.tgz", + "integrity": "sha1-/Mt7BQ9zW1oz2p65N89vGrRX+2k=" + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", diff --git a/packages/server/package.json b/packages/server/package.json index b2251604a5..6dd5424b83 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@joplin/server", - "version": "1.7.0", + "version": "1.7.1", "private": true, "scripts": { "start-dev": "nodemon --config nodemon.json dist/app.js --env dev", @@ -26,6 +26,7 @@ "markdown-it": "^12.0.4", "mustache": "^3.1.0", "nanoid": "^2.1.1", + "node-env-file": "^0.1.8", "nodemon": "^2.0.6", "pg": "^8.5.1", "query-string": "^6.8.3", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 0241a368f2..9363b66a62 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -5,32 +5,30 @@ import * as Koa from 'koa'; import * as fs from 'fs-extra'; import { argv } from 'yargs'; import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger'; -import config, { initConfig, baseUrl } from './config'; -import configDev from './config-dev'; -import configProd from './config-prod'; -import configBuildTypes from './config-buildTypes'; +import config, { initConfig, runningInDocker, EnvVariables } from './config'; import { createDb, dropDb } from './tools/dbTools'; -import { dropTables, connectDb, disconnectDb, migrateDb, waitForConnection } from './db'; +import { dropTables, connectDb, disconnectDb, migrateDb, waitForConnection, sqliteFilePath } from './db'; import modelFactory from './models/factory'; -import { AppContext, Config, Env } from './utils/types'; +import { AppContext, Env } from './utils/types'; import FsDriverNode from '@joplin/lib/fs-driver-node'; import routeHandler from './middleware/routeHandler'; import notificationHandler from './middleware/notificationHandler'; import ownerHandler from './middleware/ownerHandler'; +const nodeEnvFile = require('node-env-file'); const { shimInit } = require('@joplin/lib/shim-init-node.js'); shimInit(); const env: Env = argv.env as Env || Env.Prod; -interface Configs { - [name: string]: Config; -} - -const configs: Configs = { - dev: configDev, - prod: configProd, - buildTypes: configBuildTypes, +const envVariables: Record = { + dev: { + SQLITE_DATABASE: 'dev', + }, + buildTypes: { + SQLITE_DATABASE: 'buildTypes', + }, + prod: {}, // Actually get the env variables from the environment }; let appLogger_: LoggerWrapper = null; @@ -52,11 +50,31 @@ app.use(ownerHandler); app.use(notificationHandler); app.use(routeHandler); -async function main() { - const configObject: Config = configs[env]; - if (!configObject) throw new Error(`Invalid env: ${env}`); +function markPasswords(o: Record): Record { + const output: Record = {}; - initConfig(configObject); + for (const k of Object.keys(o)) { + if (k.toLowerCase().includes('password')) { + output[k] = '********'; + } else { + output[k] = o[k]; + } + } + + return output; +} + +async function main() { + if (argv.envFile) { + nodeEnvFile(argv.envFile); + } + + if (!envVariables[env]) throw new Error(`Invalid env: ${env}`); + + initConfig({ + ...envVariables[env], + ...process.env, + }); await fs.mkdirp(config().logDir); Logger.fsDriver_ = new FsDriverNode(); @@ -90,8 +108,11 @@ async function main() { await createDb(config().database); } else { appLogger().info(`Starting server (${env}) on port ${config().port} and PID ${process.pid}...`); - appLogger().info('Public base URL:', baseUrl()); - appLogger().info('DB Config:', config().database); + appLogger().info('Running in Docker:', runningInDocker()); + appLogger().info('Public base URL:', config().baseUrl); + appLogger().info('Log dir:', config().logDir); + appLogger().info('DB Config:', markPasswords(config().database)); + if (config().database.client === 'sqlite3') appLogger().info('DB file:', sqliteFilePath(config().database.name)); const appContext = app.context as AppContext; @@ -104,13 +125,13 @@ async function main() { appLogger().info('Connection check:', connectionCheckLogInfo); appContext.env = env; appContext.db = connectionCheck.connection; - appContext.models = modelFactory(appContext.db, baseUrl()); + appContext.models = modelFactory(appContext.db, config().baseUrl); appContext.appLogger = appLogger; appLogger().info('Migrating database...'); await migrateDb(appContext.db); - appLogger().info(`Call this for testing: \`curl ${baseUrl()}/api/ping\``); + appLogger().info(`Call this for testing: \`curl ${config().baseUrl}/api/ping\``); app.listen(config().port); } diff --git a/packages/server/src/config-base.ts b/packages/server/src/config-base.ts deleted file mode 100644 index 95990ec31d..0000000000 --- a/packages/server/src/config-base.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Config } from './utils/types'; -import * as pathUtils from 'path'; - -const rootDir = pathUtils.dirname(__dirname); -const viewDir = `${pathUtils.dirname(__dirname)}/src/views`; - -const envPort = Number(process.env.JOPLIN_PORT); - -const config: Config = { - port: (envPort && !isNaN(envPort)) ? envPort : 22300, - viewDir: viewDir, - rootDir: rootDir, - layoutDir: `${viewDir}/layouts`, - logDir: `${rootDir}/logs`, - database: { - client: 'pg', - name: 'joplin', - }, -}; - -export default config; diff --git a/packages/server/src/config-buildTypes.ts b/packages/server/src/config-buildTypes.ts deleted file mode 100644 index 118338710b..0000000000 --- a/packages/server/src/config-buildTypes.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Config } from './utils/types'; -import configBase from './config-base'; - -const config: Config = { - ...configBase, - database: { - name: 'buildTypes', - client: 'sqlite3', - asyncStackTraces: true, - }, -}; - -export default config; diff --git a/packages/server/src/config-dev.ts b/packages/server/src/config-dev.ts deleted file mode 100644 index a9f430acb7..0000000000 --- a/packages/server/src/config-dev.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Config } from './utils/types'; -import configBase from './config-base'; - -const config: Config = { - ...configBase, - database: { - name: 'dev', - client: 'sqlite3', - asyncStackTraces: true, - }, - // database: { - // client: 'pg', - // name: 'joplin', - // user: 'joplin', - // host: 'localhost', - // port: 5432, - // password: 'joplin', - // asyncStackTraces: true, - // }, -}; - -export default config; diff --git a/packages/server/src/config-prod.ts b/packages/server/src/config-prod.ts deleted file mode 100644 index f32906480e..0000000000 --- a/packages/server/src/config-prod.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Config } from './utils/types'; -import configBase from './config-base'; - -const rootDir = '/home/joplin/'; - -const config: Config = { - ...configBase, - rootDir: rootDir, - logDir: `${rootDir}/logs`, - database: { - client: 'pg', - name: 'joplin', - user: 'joplin', - host: 'db', - port: 5432, - password: 'joplin', - }, -}; - -export default config; diff --git a/packages/server/src/config-tests.ts b/packages/server/src/config-tests.ts deleted file mode 100644 index 4041b61cee..0000000000 --- a/packages/server/src/config-tests.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Config } from './utils/types'; -import configBase from './config-base'; - -const config: Config = { - ...configBase, - database: { - name: 'DYNAMIC', - client: 'sqlite3', - asyncStackTraces: true, - }, -}; - -export default config; diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 3f57bda634..35a1fa8528 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -1,28 +1,93 @@ import { rtrimSlashes } from '@joplin/lib/path-utils'; -import { Config } from './utils/types'; +import { Config, DatabaseConfig, DatabaseConfigClient } from './utils/types'; +import * as pathUtils from 'path'; -let baseConfig_: Config = null; -let baseUrl_: string = null; +export interface EnvVariables { + APP_BASE_URL?: string; + APP_PORT?: string; + DB_CLIENT?: string; + RUNNING_IN_DOCKER?: string; -export function initConfig(baseConfig: Config) { - baseConfig_ = baseConfig; + POSTGRES_PASSWORD?: string; + POSTGRES_DATABASE?: string; + POSTGRES_USER?: string; + POSTGRES_HOST?: string; + POSTGRES_PORT?: string; + + SQLITE_DATABASE?: string; +} + +let runningInDocker_: boolean = false; + +export function runningInDocker(): boolean { + return runningInDocker_; +} + +function databaseHostFromEnv(runningInDocker: boolean, env: EnvVariables): string { + if (env.POSTGRES_HOST) { + // When running within Docker, the app localhost is different from the + // host's localhost. To access the latter, Docker defines a special host + // called "host.docker.internal", so here we swap the values if necessary. + if (runningInDocker && ['localhost', '127.0.0.1'].includes(env.POSTGRES_HOST)) { + return 'host.docker.internal'; + } else { + return env.POSTGRES_HOST; + } + } + + return null; +} + +function databaseConfigFromEnv(runningInDocker: boolean, env: EnvVariables): DatabaseConfig { + if (env.DB_CLIENT === 'pg') { + return { + client: DatabaseConfigClient.PostgreSQL, + name: env.POSTGRES_DATABASE || 'joplin', + user: env.POSTGRES_USER || 'joplin', + password: env.POSTGRES_PASSWORD || 'joplin', + port: env.POSTGRES_PORT ? Number(env.POSTGRES_PORT) : 5432, + host: databaseHostFromEnv(runningInDocker, env) || 'localhost', + }; + } + + return { + client: DatabaseConfigClient.SQLite, + name: env.SQLITE_DATABASE || 'prod', + asyncStackTraces: true, + }; +} + +function baseUrlFromEnv(env: any, appPort: number): string { + if (env.APP_BASE_URL) { + return rtrimSlashes(env.APP_BASE_URL); + } else { + return `http://localhost:${appPort}`; + } +} + +let config_: Config = null; + +export function initConfig(env: EnvVariables) { + runningInDocker_ = !!env.RUNNING_IN_DOCKER; + + const rootDir = pathUtils.dirname(__dirname); + const viewDir = `${pathUtils.dirname(__dirname)}/src/views`; + const appPort = env.APP_PORT ? Number(env.APP_PORT) : 22300; + + config_ = { + rootDir: rootDir, + viewDir: viewDir, + layoutDir: `${viewDir}/layouts`, + logDir: `${rootDir}/logs`, + database: databaseConfigFromEnv(runningInDocker_, env), + port: appPort, + baseUrl: baseUrlFromEnv(env, appPort), + }; } function config(): Config { - if (!baseConfig_) throw new Error('Config has not been initialized!'); - return baseConfig_; -} - -export function baseUrl() { - if (baseUrl_) return baseUrl_; - - if (process.env.JOPLIN_BASE_URL) { - baseUrl_ = rtrimSlashes(process.env.JOPLIN_BASE_URL); - } else { - baseUrl_ = `http://localhost:${config().port}`; - } - - return baseUrl_; + if (!config_) throw new Error('Config has not been initialized!'); + return config_; } export default config; diff --git a/packages/server/src/db.ts b/packages/server/src/db.ts index 8437631bcb..9fcc19ab40 100644 --- a/packages/server/src/db.ts +++ b/packages/server/src/db.ts @@ -47,15 +47,15 @@ export interface ConnectionCheckResult { connection: DbConnection; } -export function sqliteFilePath(dbConfig: DatabaseConfig): string { - return `${sqliteDbDir}/db-${dbConfig.name}.sqlite`; +export function sqliteFilePath(name: string): string { + return `${sqliteDbDir}/db-${name}.sqlite`; } export function makeKnexConfig(dbConfig: DatabaseConfig): KnexDatabaseConfig { const connection: DbConfigConnection = {}; if (dbConfig.client === 'sqlite3') { - connection.filename = sqliteFilePath(dbConfig); + connection.filename = sqliteFilePath(dbConfig.name); } else { connection.database = dbConfig.name; connection.host = dbConfig.host; diff --git a/packages/server/src/middleware/notificationHandler.ts b/packages/server/src/middleware/notificationHandler.ts index d5ec2e552b..dae4fef490 100644 --- a/packages/server/src/middleware/notificationHandler.ts +++ b/packages/server/src/middleware/notificationHandler.ts @@ -4,47 +4,80 @@ import { defaultAdminEmail, defaultAdminPassword, NotificationLevel } from '../d import { _ } from '@joplin/lib/locale'; import Logger from '@joplin/lib/Logger'; import * as MarkdownIt from 'markdown-it'; +import config from '../config'; const logger = Logger.create('notificationHandler'); +async function handleChangeAdminPasswordNotification(ctx: AppContext) { + if (!ctx.owner.is_admin) return; + + const defaultAdmin = await ctx.models.user().login(defaultAdminEmail, defaultAdminPassword); + const notificationModel = ctx.models.notification({ userId: ctx.owner.id }); + + if (defaultAdmin) { + await notificationModel.add( + 'change_admin_password', + NotificationLevel.Important, + _('The default admin password is insecure and has not been changed! [Change it now](%s)', await ctx.models.user().profileUrl()) + ); + } else { + await notificationModel.markAsRead('change_admin_password'); + } + + if (config().database.client === 'sqlite3' && ctx.env === 'prod') { + await notificationModel.add( + 'using_sqlite_in_prod', + NotificationLevel.Important, + 'The server is currently using SQLite3 as a database. It is not recommended in production as it is slow and can cause locking issues. Please see the README for information on how to change it.' + ); + } +} + +async function handleSqliteInProdNotification(ctx: AppContext) { + if (!ctx.owner.is_admin) return; + + const notificationModel = ctx.models.notification({ userId: ctx.owner.id }); + + if (config().database.client === 'sqlite3' && ctx.env === 'prod') { + await notificationModel.add( + 'using_sqlite_in_prod', + NotificationLevel.Important, + 'The server is currently using SQLite3 as a database. It is not recommended in production as it is slow and can cause locking issues. Please see the README for information on how to change it.' + ); + } +} + +async function makeNotificationViews(ctx: AppContext): Promise { + const markdownIt = new MarkdownIt(); + + const notificationModel = ctx.models.notification({ userId: ctx.owner.id }); + const notifications = await notificationModel.allUnreadByUserId(ctx.owner.id); + const views: NotificationView[] = []; + for (const n of notifications) { + views.push({ + id: n.id, + messageHtml: markdownIt.render(n.message), + level: n.level === NotificationLevel.Important ? 'warning' : 'info', + closeUrl: notificationModel.closeUrl(n.id), + }); + } + + return views; +} + +// The role of this middleware is to inspect the system and to generate +// notifications for any issue it finds. It is only active for logged in users +// on the website. It is inactive for API calls. export default async function(ctx: AppContext, next: KoaNext): Promise { ctx.notifications = []; try { if (isApiRequest(ctx)) return next(); + if (!ctx.owner) return next(); - const user = ctx.owner; - if (!user) return next(); - - const notificationModel = ctx.models.notification({ userId: user.id }); - - if (user.is_admin) { - const defaultAdmin = await ctx.models.user().login(defaultAdminEmail, defaultAdminPassword); - - if (defaultAdmin) { - await notificationModel.add( - 'change_admin_password', - NotificationLevel.Important, - _('The default admin password is insecure and has not been changed! [Change it now](%s)', await ctx.models.user().profileUrl()) - ); - } else { - await notificationModel.markAsRead('change_admin_password'); - } - } - - const markdownIt = new MarkdownIt(); - const notifications = await notificationModel.allUnreadByUserId(user.id); - const views: NotificationView[] = []; - for (const n of notifications) { - views.push({ - id: n.id, - messageHtml: markdownIt.render(n.message), - level: n.level === NotificationLevel.Important ? 'warning' : 'info', - closeUrl: notificationModel.closeUrl(n.id), - }); - } - - ctx.notifications = views; + await handleChangeAdminPasswordNotification(ctx); + await handleSqliteInProdNotification(ctx); + ctx.notifications = await makeNotificationViews(ctx); } catch (error) { logger.error(error); } diff --git a/packages/server/src/middleware/routeHandler.ts b/packages/server/src/middleware/routeHandler.ts index 117b0ed8df..83a0558cb0 100644 --- a/packages/server/src/middleware/routeHandler.ts +++ b/packages/server/src/middleware/routeHandler.ts @@ -26,6 +26,7 @@ export default async function(ctx: AppContext) { ctx.response.status = 200; ctx.response.body = await mustacheService.renderView(responseObject, { notifications: ctx.notifications || [], + hasNotifications: !!ctx.notifications && !!ctx.notifications.length, owner: ctx.owner, }); } else { diff --git a/packages/server/src/routes/default.ts b/packages/server/src/routes/default.ts index d0bb96a244..73908dde0e 100644 --- a/packages/server/src/routes/default.ts +++ b/packages/server/src/routes/default.ts @@ -39,6 +39,10 @@ async function findLocalFile(path: string): Promise { const router = new Router(); +router.public = true; + +// Used to serve static files, so it needs to be public because for example the +// login page, which is public, needs access to the CSS files. router.get('', async (path: SubPath, ctx: Koa.Context) => { const localPath = await findLocalFile(path.raw); diff --git a/packages/server/src/routes/index/files.ts b/packages/server/src/routes/index/files.ts index 330e2eb78c..f13c5829ce 100644 --- a/packages/server/src/routes/index/files.ts +++ b/packages/server/src/routes/index/files.ts @@ -6,7 +6,7 @@ import { ErrorNotFound } from '../../utils/errors'; import { File } from '../../db'; import { createPaginationLinks, pageMaxSize, Pagination, PaginationOrder, PaginationOrderDir, requestPaginationOrder, validatePagination } from '../../models/utils/pagination'; import { setQueryParameters } from '../../utils/urlUtils'; -import { baseUrl } from '../../config'; +import config from '../../config'; import { formatDateTime } from '../../utils/time'; import defaultView from '../../utils/defaultView'; import { View } from '../../services/MustacheService'; @@ -51,7 +51,7 @@ router.get('files/:id', async (path: SubPath, ctx: AppContext) => { async function fileToViewItem(file: File, fileFullPaths: Record): Promise { const filePath = fileFullPaths[file.id]; - let url = `${baseUrl()}/files/${filePath}`; + let url = `${config().baseUrl}/files/${filePath}`; if (!file.is_directory) { url += '/content'; } else { @@ -88,7 +88,7 @@ router.get('files/:id', async (path: SubPath, ctx: AppContext) => { const view: View = defaultView('files'); view.content.paginatedFiles = { ...paginatedFiles, items: files }; view.content.paginationLinks = paginationLinks; - view.content.postUrl = `${baseUrl()}/files`; + view.content.postUrl = `${config().baseUrl}/files`; view.content.parentId = parent.id; view.cssFiles = ['index/files']; view.partials.push('pagination'); diff --git a/packages/server/src/routes/index/login.ts b/packages/server/src/routes/index/login.ts index 58aab81623..1d0b9227e3 100644 --- a/packages/server/src/routes/index/login.ts +++ b/packages/server/src/routes/index/login.ts @@ -2,7 +2,7 @@ import { SubPath, redirect } from '../../utils/routeUtils'; import Router from '../../utils/Router'; import { AppContext } from '../../utils/types'; import { formParse } from '../../utils/requestUtils'; -import { baseUrl } from '../../config'; +import config from '../../config'; import defaultView from '../../utils/defaultView'; import { View } from '../../services/MustacheService'; @@ -27,7 +27,7 @@ router.post('login', async (_path: SubPath, ctx: AppContext) => { const session = await ctx.models.session().authenticate(body.fields.email, body.fields.password); ctx.cookies.set('sessionId', session.id); - return redirect(ctx, `${baseUrl()}/home`); + return redirect(ctx, `${config().baseUrl}/home`); } catch (error) { return makeView(error); } diff --git a/packages/server/src/routes/index/logout.ts b/packages/server/src/routes/index/logout.ts index 661ba433b5..7a2e9129c9 100644 --- a/packages/server/src/routes/index/logout.ts +++ b/packages/server/src/routes/index/logout.ts @@ -1,7 +1,7 @@ import { SubPath, redirect } from '../../utils/routeUtils'; import Router from '../../utils/Router'; import { AppContext } from '../../utils/types'; -import { baseUrl } from '../../config'; +import config from '../../config'; import { contextSessionId } from '../../utils/requestUtils'; const router = new Router(); @@ -10,7 +10,7 @@ router.post('logout', async (_path: SubPath, ctx: AppContext) => { const sessionId = contextSessionId(ctx, false); ctx.cookies.set('sessionId', ''); await ctx.models.session().logout(sessionId); - return redirect(ctx, `${baseUrl()}/login`); + return redirect(ctx, `${config().baseUrl}/login`); }); export default router; diff --git a/packages/server/src/routes/index/users.ts b/packages/server/src/routes/index/users.ts index 8fb2d8f55a..5703ffcc57 100644 --- a/packages/server/src/routes/index/users.ts +++ b/packages/server/src/routes/index/users.ts @@ -4,7 +4,7 @@ import { AppContext, HttpMethod } from '../../utils/types'; import { formParse } from '../../utils/requestUtils'; import { ErrorUnprocessableEntity } from '../../utils/errors'; import { User } from '../../db'; -import { baseUrl } from '../../config'; +import config from '../../config'; import { View } from '../../services/MustacheService'; import defaultView from '../../utils/defaultView'; @@ -55,11 +55,11 @@ router.get('users/:id', async (path: SubPath, ctx: AppContext, user: User = null let postUrl = ''; if (isNew) { - postUrl = `${baseUrl()}/users/new`; + postUrl = `${config().baseUrl}/users/new`; } else if (isMe) { - postUrl = `${baseUrl()}/users/me`; + postUrl = `${config().baseUrl}/users/me`; } else { - postUrl = `${baseUrl()}/users/${user.id}`; + postUrl = `${config().baseUrl}/users/${user.id}`; } const view: View = defaultView('user'); @@ -100,7 +100,7 @@ router.post('users', async (path: SubPath, ctx: AppContext) => { throw new Error('Invalid form button'); } - return redirect(ctx, `${baseUrl()}/users${userIsMe(path) ? '/me' : ''}`); + return redirect(ctx, `${config().baseUrl}/users${userIsMe(path) ? '/me' : ''}`); } catch (error) { const endPoint = router.findEndPoint(HttpMethod.GET, 'users/:id'); return endPoint(path, ctx, user, error); diff --git a/packages/server/src/services/MustacheService.ts b/packages/server/src/services/MustacheService.ts index d718af53ea..14e50f3619 100644 --- a/packages/server/src/services/MustacheService.ts +++ b/packages/server/src/services/MustacheService.ts @@ -1,6 +1,6 @@ import * as Mustache from 'mustache'; import * as fs from 'fs-extra'; -import config, { baseUrl } from '../config'; +import config from '../config'; export interface RenderOptions { partials?: any; @@ -30,7 +30,7 @@ class MustacheService { private get defaultLayoutOptions(): any { return { - baseUrl: baseUrl(), + baseUrl: config().baseUrl, }; } @@ -41,7 +41,7 @@ class MustacheService { private resolvesFilePaths(type: string, paths: string[]): string[] { const output: string[] = []; for (const path of paths) { - output.push(`${baseUrl()}/${type}/${path}.${type}`); + output.push(`${config().baseUrl}/${type}/${path}.${type}`); } return output; } diff --git a/packages/server/src/tools/dbTools.ts b/packages/server/src/tools/dbTools.ts index b8b16bd2b4..6b14b12224 100644 --- a/packages/server/src/tools/dbTools.ts +++ b/packages/server/src/tools/dbTools.ts @@ -33,7 +33,7 @@ export async function createDb(config: DatabaseConfig, options: CreateDbOptions await execCommand(cmd.join(' ')); } else if (config.client === 'sqlite3') { - const filePath = sqliteFilePath(config); + const filePath = sqliteFilePath(config.name); if (await fs.pathExists(filePath)) { if (options.dropIfExists) { @@ -71,6 +71,6 @@ export async function dropDb(config: DatabaseConfig, options: DropDbOptions = nu throw error; } } else if (config.client === 'sqlite3') { - await fs.remove(sqliteFilePath(config)); + await fs.remove(sqliteFilePath(config.name)); } } diff --git a/packages/server/src/utils/testing/testUtils.ts b/packages/server/src/utils/testing/testUtils.ts index 0446d3f58d..43d038f6f9 100644 --- a/packages/server/src/utils/testing/testUtils.ts +++ b/packages/server/src/utils/testing/testUtils.ts @@ -1,9 +1,8 @@ -import { User, Session, DbConnection, connectDb, disconnectDb, File, truncateTables } from '../../db'; +import { User, Session, DbConnection, connectDb, disconnectDb, File, truncateTables, sqliteFilePath } from '../../db'; import { createDb } from '../../tools/dbTools'; import modelFactory from '../../models/factory'; -import baseConfig from '../../config-tests'; -import { AppContext, Config, Env } from '../types'; -import { initConfig } from '../../config'; +import { AppContext, Env } from '../types'; +import config, { initConfig } from '../../config'; import FileModel from '../../models/FileModel'; import Logger from '@joplin/lib/Logger'; import FakeCookies from './koa/FakeCookies'; @@ -34,18 +33,16 @@ export async function tempDir(): Promise { return tempDir_; } +let createdDbName_: string = null; export async function beforeAllDb(unitName: string) { - const config: Config = { - ...baseConfig, - database: { - ...baseConfig.database, - name: unitName, - }, - }; + createdDbName_ = unitName; - initConfig(config); - await createDb(config.database, { dropIfExists: true }); - db_ = await connectDb(config.database); + initConfig({ + SQLITE_DATABASE: createdDbName_, + }); + + await createDb(config().database, { dropIfExists: true }); + db_ = await connectDb(config().database); } export async function afterAllTests() { @@ -58,6 +55,12 @@ export async function afterAllTests() { await fs.remove(tempDir_); tempDir_ = null; } + + if (createdDbName_) { + const filePath = sqliteFilePath(createdDbName_); + await fs.remove(filePath); + createdDbName_ = null; + } } export async function beforeEachDb() { diff --git a/packages/server/src/utils/types.ts b/packages/server/src/utils/types.ts index 3df9acbc6d..271eb018d6 100644 --- a/packages/server/src/utils/types.ts +++ b/packages/server/src/utils/types.ts @@ -25,8 +25,13 @@ export interface AppContext extends Koa.Context { owner: User; } +export enum DatabaseConfigClient { + PostgreSQL = 'pg', + SQLite = 'sqlite3', +} + export interface DatabaseConfig { - client: string; + client: DatabaseConfigClient; name: string; host?: string; port?: number; @@ -40,8 +45,11 @@ export interface Config { rootDir: string; viewDir: string; layoutDir: string; + // Not that, for now, nothing is being logged to file. Log is just printed + // to stdout, which is then handled by Docker own log mechanism logDir: string; database: DatabaseConfig; + baseUrl: string; } export enum HttpMethod { diff --git a/packages/server/src/views/layouts/default.mustache b/packages/server/src/views/layouts/default.mustache index ffa7e066d1..a7f8d489b3 100644 --- a/packages/server/src/views/layouts/default.mustache +++ b/packages/server/src/views/layouts/default.mustache @@ -13,8 +13,8 @@ {{> navbar}} - {{> notifications}}
+ {{> notifications}} {{{contentHtml}}}
diff --git a/packages/server/src/views/partials/notifications.mustache b/packages/server/src/views/partials/notifications.mustache index 034add3426..25dd91a37e 100644 --- a/packages/server/src/views/partials/notifications.mustache +++ b/packages/server/src/views/partials/notifications.mustache @@ -1,9 +1,11 @@ -{{#global.notifications}} -
- - {{{messageHtml}}} -
-{{/global.notifications}} +{{#global.hasNotifications}} + {{#global.notifications}} +
+ + {{{messageHtml}}} +
+ {{/global.notifications}} +{{/global.hasNotifications}}