self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

Server: Add Joplin Server package (#1872)

Browse Source
This commit is contained in:
Laurent 2020-12-28 11:48:47 +00:00 committed by GitHub
parent 2cd7839552
commit 41684a64ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
120 changed files with 14784 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.dockerignore Normal file
View File

@ -0,0 +1,9 @@
**/node_modules
Assets/
.git/
_releases/
packages/app-desktop
packages/app-cli
packages/app-mobile
packages/app-clipper
packages/generator-joplin

9
.env-sample Normal file
View File

@ -0,0 +1,9 @@
# Example of local config, for development:
#
# JOPLIN_BASE_URL=http://localhost:22300
# JOPLIN_PORT=22300
# Example of config for production:
#
# JOPLIN_BASE_URL=https://example.com/joplin
# JOPLIN_PORT=22300

208
.eslintignore
View File

@ -6,6 +6,7 @@ _releases/
**/node_modules/
Assets/
docs/
packages/server/dist/
highlight.pack.js
Modules/TinyMCE/IconPack/postinstall.js
Modules/TinyMCE/JoplinLists/
@ -889,12 +890,18 @@ packages/lib/InMemoryCache.js.map
packages/lib/JoplinServerApi.d.ts
packages/lib/JoplinServerApi.js
packages/lib/JoplinServerApi.js.map
packages/lib/JoplinServerApi2.d.ts
packages/lib/JoplinServerApi2.js
packages/lib/JoplinServerApi2.js.map
packages/lib/Logger.d.ts
packages/lib/Logger.js
packages/lib/Logger.js.map
packages/lib/PoorManIntervals.d.ts
packages/lib/PoorManIntervals.js
packages/lib/PoorManIntervals.js.map
packages/lib/SyncTargetJoplinServer.d.ts
packages/lib/SyncTargetJoplinServer.js
packages/lib/SyncTargetJoplinServer.js.map
packages/lib/Synchronizer.d.ts
packages/lib/Synchronizer.js
packages/lib/Synchronizer.js.map
@ -916,6 +923,9 @@ packages/lib/errorUtils.js.map
packages/lib/eventManager.d.ts
packages/lib/eventManager.js
packages/lib/eventManager.js.map
packages/lib/file-api-driver-joplinServer.d.ts
packages/lib/file-api-driver-joplinServer.js
packages/lib/file-api-driver-joplinServer.js.map
packages/lib/fs-driver-base.d.ts
packages/lib/fs-driver-base.js
packages/lib/fs-driver-base.js.map
@ -1387,4 +1397,202 @@ packages/renderer/pathUtils.js.map
packages/renderer/utils.d.ts
packages/renderer/utils.js
packages/renderer/utils.js.map
packages/server/src/app.d.ts
packages/server/src/app.js
packages/server/src/app.js.map
packages/server/src/config-base.d.ts
packages/server/src/config-base.js
packages/server/src/config-base.js.map
packages/server/src/config-buildTypes.d.ts
packages/server/src/config-buildTypes.js
packages/server/src/config-buildTypes.js.map
packages/server/src/config-dev.d.ts
packages/server/src/config-dev.js
packages/server/src/config-dev.js.map
packages/server/src/config-prod.d.ts
packages/server/src/config-prod.js
packages/server/src/config-prod.js.map
packages/server/src/config-tests.d.ts
packages/server/src/config-tests.js
packages/server/src/config-tests.js.map
packages/server/src/config.d.ts
packages/server/src/config.js
packages/server/src/config.js.map
packages/server/src/controllers/BaseController.d.ts
packages/server/src/controllers/BaseController.js
packages/server/src/controllers/BaseController.js.map
packages/server/src/controllers/api/FileController.d.ts
packages/server/src/controllers/api/FileController.js
packages/server/src/controllers/api/FileController.js.map
packages/server/src/controllers/api/FileController.test.d.ts
packages/server/src/controllers/api/FileController.test.js
packages/server/src/controllers/api/FileController.test.js.map
packages/server/src/controllers/api/OAuthController.d.ts
packages/server/src/controllers/api/OAuthController.js
packages/server/src/controllers/api/OAuthController.js.map
packages/server/src/controllers/api/SessionController.d.ts
packages/server/src/controllers/api/SessionController.js
packages/server/src/controllers/api/SessionController.js.map
packages/server/src/controllers/api/SessionController.test.d.ts
packages/server/src/controllers/api/SessionController.test.js
packages/server/src/controllers/api/SessionController.test.js.map
packages/server/src/controllers/api/UserController.d.ts
packages/server/src/controllers/api/UserController.js
packages/server/src/controllers/api/UserController.js.map
packages/server/src/controllers/api/UserController.test.d.ts
packages/server/src/controllers/api/UserController.test.js
packages/server/src/controllers/api/UserController.test.js.map
packages/server/src/controllers/factory.d.ts
packages/server/src/controllers/factory.js
packages/server/src/controllers/factory.js.map
packages/server/src/controllers/index/HomeController.d.ts
packages/server/src/controllers/index/HomeController.js
packages/server/src/controllers/index/HomeController.js.map
packages/server/src/controllers/index/LoginController.d.ts
packages/server/src/controllers/index/LoginController.js
packages/server/src/controllers/index/LoginController.js.map
packages/server/src/controllers/index/ProfileController.d.ts
packages/server/src/controllers/index/ProfileController.js
packages/server/src/controllers/index/ProfileController.js.map
packages/server/src/controllers/index/UserController.d.ts
packages/server/src/controllers/index/UserController.js
packages/server/src/controllers/index/UserController.js.map
packages/server/src/db.d.ts
packages/server/src/db.js
packages/server/src/db.js.map
packages/server/src/migrations/20190913171451_create.d.ts
packages/server/src/migrations/20190913171451_create.js
packages/server/src/migrations/20190913171451_create.js.map
packages/server/src/models/ApiClientModel.d.ts
packages/server/src/models/ApiClientModel.js
packages/server/src/models/ApiClientModel.js.map
packages/server/src/models/BaseModel.d.ts
packages/server/src/models/BaseModel.js
packages/server/src/models/BaseModel.js.map
packages/server/src/models/ChangeModel.d.ts
packages/server/src/models/ChangeModel.js
packages/server/src/models/ChangeModel.js.map
packages/server/src/models/ChangeModel.test.d.ts
packages/server/src/models/ChangeModel.test.js
packages/server/src/models/ChangeModel.test.js.map
packages/server/src/models/FileModel.d.ts
packages/server/src/models/FileModel.js
packages/server/src/models/FileModel.js.map
packages/server/src/models/PermissionModel.d.ts
packages/server/src/models/PermissionModel.js
packages/server/src/models/PermissionModel.js.map
packages/server/src/models/SessionModel.d.ts
packages/server/src/models/SessionModel.js
packages/server/src/models/SessionModel.js.map
packages/server/src/models/UserModel.d.ts
packages/server/src/models/UserModel.js
packages/server/src/models/UserModel.js.map
packages/server/src/models/factory.d.ts
packages/server/src/models/factory.js
packages/server/src/models/factory.js.map
packages/server/src/models/utils/pagination.d.ts
packages/server/src/models/utils/pagination.js
packages/server/src/models/utils/pagination.js.map
packages/server/src/models/utils/pagination.test.d.ts
packages/server/src/models/utils/pagination.test.js
packages/server/src/models/utils/pagination.test.js.map
packages/server/src/routes/api/files.d.ts
packages/server/src/routes/api/files.js
packages/server/src/routes/api/files.js.map
packages/server/src/routes/api/index.d.ts
packages/server/src/routes/api/index.js
packages/server/src/routes/api/index.js.map
packages/server/src/routes/api/ping.d.ts
packages/server/src/routes/api/ping.js
packages/server/src/routes/api/ping.js.map
packages/server/src/routes/api/sessions.d.ts
packages/server/src/routes/api/sessions.js
packages/server/src/routes/api/sessions.js.map
packages/server/src/routes/default.d.ts
packages/server/src/routes/default.js
packages/server/src/routes/default.js.map
packages/server/src/routes/index/home.d.ts
packages/server/src/routes/index/home.js
packages/server/src/routes/index/home.js.map
packages/server/src/routes/index/login.d.ts
packages/server/src/routes/index/login.js
packages/server/src/routes/index/login.js.map
packages/server/src/routes/index/logout.d.ts
packages/server/src/routes/index/logout.js
packages/server/src/routes/index/logout.js.map
packages/server/src/routes/index/profile.d.ts
packages/server/src/routes/index/profile.js
packages/server/src/routes/index/profile.js.map
packages/server/src/routes/index/user.d.ts
packages/server/src/routes/index/user.js
packages/server/src/routes/index/user.js.map
packages/server/src/routes/index/users.d.ts
packages/server/src/routes/index/users.js
packages/server/src/routes/index/users.js.map
packages/server/src/routes/oauth2/authorize.d.ts
packages/server/src/routes/oauth2/authorize.js
packages/server/src/routes/oauth2/authorize.js.map
packages/server/src/routes/routes.d.ts
packages/server/src/routes/routes.js
packages/server/src/routes/routes.js.map
packages/server/src/services/MustacheService.d.ts
packages/server/src/services/MustacheService.js
packages/server/src/services/MustacheService.js.map
packages/server/src/tools/db-migrate.d.ts
packages/server/src/tools/db-migrate.js
packages/server/src/tools/db-migrate.js.map
packages/server/src/tools/dbTools.d.ts
packages/server/src/tools/dbTools.js
packages/server/src/tools/dbTools.js.map
packages/server/src/tools/generate-types.d.ts
packages/server/src/tools/generate-types.js
packages/server/src/tools/generate-types.js.map
packages/server/src/utils/TransactionHandler.d.ts
packages/server/src/utils/TransactionHandler.js
packages/server/src/utils/TransactionHandler.js.map
packages/server/src/utils/auth.d.ts
packages/server/src/utils/auth.js
packages/server/src/utils/auth.js.map
packages/server/src/utils/base64.d.ts
packages/server/src/utils/base64.js
packages/server/src/utils/base64.js.map
packages/server/src/utils/cache.d.ts
packages/server/src/utils/cache.js
packages/server/src/utils/cache.js.map
packages/server/src/utils/defaultView.d.ts
packages/server/src/utils/defaultView.js
packages/server/src/utils/defaultView.js.map
packages/server/src/utils/errors.d.ts
packages/server/src/utils/errors.js
packages/server/src/utils/errors.js.map
packages/server/src/utils/htmlUtils.d.ts
packages/server/src/utils/htmlUtils.js
packages/server/src/utils/htmlUtils.js.map
packages/server/src/utils/koaIf.d.ts
packages/server/src/utils/koaIf.js
packages/server/src/utils/koaIf.js.map
packages/server/src/utils/requestUtils.d.ts
packages/server/src/utils/requestUtils.js
packages/server/src/utils/requestUtils.js.map
packages/server/src/utils/routeUtils.d.ts
packages/server/src/utils/routeUtils.js
packages/server/src/utils/routeUtils.js.map
packages/server/src/utils/routeUtils.test.d.ts
packages/server/src/utils/routeUtils.test.js
packages/server/src/utils/routeUtils.test.js.map
packages/server/src/utils/testUtils.d.ts
packages/server/src/utils/testUtils.js
packages/server/src/utils/testUtils.js.map
packages/server/src/utils/testing/testRouters.d.ts
packages/server/src/utils/testing/testRouters.js
packages/server/src/utils/testing/testRouters.js.map
packages/server/src/utils/time.d.ts
packages/server/src/utils/time.js
packages/server/src/utils/time.js.map
packages/server/src/utils/types.d.ts
packages/server/src/utils/types.js
packages/server/src/utils/types.js.map
packages/server/src/utils/uuidgen.d.ts
packages/server/src/utils/uuidgen.js
packages/server/src/utils/uuidgen.js.map
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

4
.eslintrc.js
View File

@ -1,4 +1,5 @@
module.exports = {
'root': true,
'env': {
'browser': true,
'es6': true,
@ -34,6 +35,9 @@ module.exports = {
'chrome': 'readonly',
'browser': 'readonly',
// Server admin UI global variables
'onDocumentReady': 'readonly',
'tinymce': 'readonly',
},
'parserOptions': {

208
.gitignore vendored
View File

@ -48,6 +48,7 @@ TODO.md
packages/tools/commit_hook.txt
packages/tools/github_oauth_token.txt
lerna-debug.log
.env
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
packages/app-cli/app/LinkSelector.d.ts
@ -878,12 +879,18 @@ packages/lib/InMemoryCache.js.map
packages/lib/JoplinServerApi.d.ts
packages/lib/JoplinServerApi.js
packages/lib/JoplinServerApi.js.map
packages/lib/JoplinServerApi2.d.ts
packages/lib/JoplinServerApi2.js
packages/lib/JoplinServerApi2.js.map
packages/lib/Logger.d.ts
packages/lib/Logger.js
packages/lib/Logger.js.map
packages/lib/PoorManIntervals.d.ts
packages/lib/PoorManIntervals.js
packages/lib/PoorManIntervals.js.map
packages/lib/SyncTargetJoplinServer.d.ts
packages/lib/SyncTargetJoplinServer.js
packages/lib/SyncTargetJoplinServer.js.map
packages/lib/Synchronizer.d.ts
packages/lib/Synchronizer.js
packages/lib/Synchronizer.js.map
@ -905,6 +912,9 @@ packages/lib/errorUtils.js.map
packages/lib/eventManager.d.ts
packages/lib/eventManager.js
packages/lib/eventManager.js.map
packages/lib/file-api-driver-joplinServer.d.ts
packages/lib/file-api-driver-joplinServer.js
packages/lib/file-api-driver-joplinServer.js.map
packages/lib/fs-driver-base.d.ts
packages/lib/fs-driver-base.js
packages/lib/fs-driver-base.js.map
@ -1376,4 +1386,202 @@ packages/renderer/pathUtils.js.map
packages/renderer/utils.d.ts
packages/renderer/utils.js
packages/renderer/utils.js.map
packages/server/src/app.d.ts
packages/server/src/app.js
packages/server/src/app.js.map
packages/server/src/config-base.d.ts
packages/server/src/config-base.js
packages/server/src/config-base.js.map
packages/server/src/config-buildTypes.d.ts
packages/server/src/config-buildTypes.js
packages/server/src/config-buildTypes.js.map
packages/server/src/config-dev.d.ts
packages/server/src/config-dev.js
packages/server/src/config-dev.js.map
packages/server/src/config-prod.d.ts
packages/server/src/config-prod.js
packages/server/src/config-prod.js.map
packages/server/src/config-tests.d.ts
packages/server/src/config-tests.js
packages/server/src/config-tests.js.map
packages/server/src/config.d.ts
packages/server/src/config.js
packages/server/src/config.js.map
packages/server/src/controllers/BaseController.d.ts
packages/server/src/controllers/BaseController.js
packages/server/src/controllers/BaseController.js.map
packages/server/src/controllers/api/FileController.d.ts
packages/server/src/controllers/api/FileController.js
packages/server/src/controllers/api/FileController.js.map
packages/server/src/controllers/api/FileController.test.d.ts
packages/server/src/controllers/api/FileController.test.js
packages/server/src/controllers/api/FileController.test.js.map
packages/server/src/controllers/api/OAuthController.d.ts
packages/server/src/controllers/api/OAuthController.js
packages/server/src/controllers/api/OAuthController.js.map
packages/server/src/controllers/api/SessionController.d.ts
packages/server/src/controllers/api/SessionController.js
packages/server/src/controllers/api/SessionController.js.map
packages/server/src/controllers/api/SessionController.test.d.ts
packages/server/src/controllers/api/SessionController.test.js
packages/server/src/controllers/api/SessionController.test.js.map
packages/server/src/controllers/api/UserController.d.ts
packages/server/src/controllers/api/UserController.js
packages/server/src/controllers/api/UserController.js.map
packages/server/src/controllers/api/UserController.test.d.ts
packages/server/src/controllers/api/UserController.test.js
packages/server/src/controllers/api/UserController.test.js.map
packages/server/src/controllers/factory.d.ts
packages/server/src/controllers/factory.js
packages/server/src/controllers/factory.js.map
packages/server/src/controllers/index/HomeController.d.ts
packages/server/src/controllers/index/HomeController.js
packages/server/src/controllers/index/HomeController.js.map
packages/server/src/controllers/index/LoginController.d.ts
packages/server/src/controllers/index/LoginController.js
packages/server/src/controllers/index/LoginController.js.map
packages/server/src/controllers/index/ProfileController.d.ts
packages/server/src/controllers/index/ProfileController.js
packages/server/src/controllers/index/ProfileController.js.map
packages/server/src/controllers/index/UserController.d.ts
packages/server/src/controllers/index/UserController.js
packages/server/src/controllers/index/UserController.js.map
packages/server/src/db.d.ts
packages/server/src/db.js
packages/server/src/db.js.map
packages/server/src/migrations/20190913171451_create.d.ts
packages/server/src/migrations/20190913171451_create.js
packages/server/src/migrations/20190913171451_create.js.map
packages/server/src/models/ApiClientModel.d.ts
packages/server/src/models/ApiClientModel.js
packages/server/src/models/ApiClientModel.js.map
packages/server/src/models/BaseModel.d.ts
packages/server/src/models/BaseModel.js
packages/server/src/models/BaseModel.js.map
packages/server/src/models/ChangeModel.d.ts
packages/server/src/models/ChangeModel.js
packages/server/src/models/ChangeModel.js.map
packages/server/src/models/ChangeModel.test.d.ts
packages/server/src/models/ChangeModel.test.js
packages/server/src/models/ChangeModel.test.js.map
packages/server/src/models/FileModel.d.ts
packages/server/src/models/FileModel.js
packages/server/src/models/FileModel.js.map
packages/server/src/models/PermissionModel.d.ts
packages/server/src/models/PermissionModel.js
packages/server/src/models/PermissionModel.js.map
packages/server/src/models/SessionModel.d.ts
packages/server/src/models/SessionModel.js
packages/server/src/models/SessionModel.js.map
packages/server/src/models/UserModel.d.ts
packages/server/src/models/UserModel.js
packages/server/src/models/UserModel.js.map
packages/server/src/models/factory.d.ts
packages/server/src/models/factory.js
packages/server/src/models/factory.js.map
packages/server/src/models/utils/pagination.d.ts
packages/server/src/models/utils/pagination.js
packages/server/src/models/utils/pagination.js.map
packages/server/src/models/utils/pagination.test.d.ts
packages/server/src/models/utils/pagination.test.js
packages/server/src/models/utils/pagination.test.js.map
packages/server/src/routes/api/files.d.ts
packages/server/src/routes/api/files.js
packages/server/src/routes/api/files.js.map
packages/server/src/routes/api/index.d.ts
packages/server/src/routes/api/index.js
packages/server/src/routes/api/index.js.map
packages/server/src/routes/api/ping.d.ts
packages/server/src/routes/api/ping.js
packages/server/src/routes/api/ping.js.map
packages/server/src/routes/api/sessions.d.ts
packages/server/src/routes/api/sessions.js
packages/server/src/routes/api/sessions.js.map
packages/server/src/routes/default.d.ts
packages/server/src/routes/default.js
packages/server/src/routes/default.js.map
packages/server/src/routes/index/home.d.ts
packages/server/src/routes/index/home.js
packages/server/src/routes/index/home.js.map
packages/server/src/routes/index/login.d.ts
packages/server/src/routes/index/login.js
packages/server/src/routes/index/login.js.map
packages/server/src/routes/index/logout.d.ts
packages/server/src/routes/index/logout.js
packages/server/src/routes/index/logout.js.map
packages/server/src/routes/index/profile.d.ts
packages/server/src/routes/index/profile.js
packages/server/src/routes/index/profile.js.map
packages/server/src/routes/index/user.d.ts
packages/server/src/routes/index/user.js
packages/server/src/routes/index/user.js.map
packages/server/src/routes/index/users.d.ts
packages/server/src/routes/index/users.js
packages/server/src/routes/index/users.js.map
packages/server/src/routes/oauth2/authorize.d.ts
packages/server/src/routes/oauth2/authorize.js
packages/server/src/routes/oauth2/authorize.js.map
packages/server/src/routes/routes.d.ts
packages/server/src/routes/routes.js
packages/server/src/routes/routes.js.map
packages/server/src/services/MustacheService.d.ts
packages/server/src/services/MustacheService.js
packages/server/src/services/MustacheService.js.map
packages/server/src/tools/db-migrate.d.ts
packages/server/src/tools/db-migrate.js
packages/server/src/tools/db-migrate.js.map
packages/server/src/tools/dbTools.d.ts
packages/server/src/tools/dbTools.js
packages/server/src/tools/dbTools.js.map
packages/server/src/tools/generate-types.d.ts
packages/server/src/tools/generate-types.js
packages/server/src/tools/generate-types.js.map
packages/server/src/utils/TransactionHandler.d.ts
packages/server/src/utils/TransactionHandler.js
packages/server/src/utils/TransactionHandler.js.map
packages/server/src/utils/auth.d.ts
packages/server/src/utils/auth.js
packages/server/src/utils/auth.js.map
packages/server/src/utils/base64.d.ts
packages/server/src/utils/base64.js
packages/server/src/utils/base64.js.map
packages/server/src/utils/cache.d.ts
packages/server/src/utils/cache.js
packages/server/src/utils/cache.js.map
packages/server/src/utils/defaultView.d.ts
packages/server/src/utils/defaultView.js
packages/server/src/utils/defaultView.js.map
packages/server/src/utils/errors.d.ts
packages/server/src/utils/errors.js
packages/server/src/utils/errors.js.map
packages/server/src/utils/htmlUtils.d.ts
packages/server/src/utils/htmlUtils.js
packages/server/src/utils/htmlUtils.js.map
packages/server/src/utils/koaIf.d.ts
packages/server/src/utils/koaIf.js
packages/server/src/utils/koaIf.js.map
packages/server/src/utils/requestUtils.d.ts
packages/server/src/utils/requestUtils.js
packages/server/src/utils/requestUtils.js.map
packages/server/src/utils/routeUtils.d.ts
packages/server/src/utils/routeUtils.js
packages/server/src/utils/routeUtils.js.map
packages/server/src/utils/routeUtils.test.d.ts
packages/server/src/utils/routeUtils.test.js
packages/server/src/utils/routeUtils.test.js.map
packages/server/src/utils/testUtils.d.ts
packages/server/src/utils/testUtils.js
packages/server/src/utils/testUtils.js.map
packages/server/src/utils/testing/testRouters.d.ts
packages/server/src/utils/testing/testRouters.js
packages/server/src/utils/testing/testRouters.js.map
packages/server/src/utils/time.d.ts
packages/server/src/utils/time.js
packages/server/src/utils/time.js.map
packages/server/src/utils/types.d.ts
packages/server/src/utils/types.js
packages/server/src/utils/types.js.map
packages/server/src/utils/uuidgen.d.ts
packages/server/src/utils/uuidgen.js
packages/server/src/utils/uuidgen.js.map
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

3
Dockerfile.db Normal file
View File

@ -0,0 +1,3 @@
FROM postgres:13.1
EXPOSE 5432

71
Dockerfile.server Normal file
View File

@ -0,0 +1,71 @@
# https://versatile.nl/blog/deploying-lerna-web-apps-with-docker
FROM node:12
RUN apt-get update
RUN apt-get --yes install vim
ARG user=joplin
RUN useradd --create-home --shell /bin/bash $user
USER $user
ENV NODE_ENV development
WORKDIR /home/$user
RUN mkdir /home/$user/logs
# To take advantage of the Docker cache, we first copy all the package.json
# and package-lock.json files, as they rarely change? and then bootstrap
# all the packages.
#
# Note that bootstrapping the packages will run all the postinstall
# scripts, which means that for packages that have such scripts, we need to
# copy all the files.
#
# We can't run boostrap with "--ignore-scripts" because that would
# prevent certain sub-packages, such as sqlite3, from being built
COPY --chown=$user:$user package*.json ./
# Install the root scripts but don't run postinstall (which would bootstrap
# and build TypeScript files, but we don't have the TypeScript files at
# this point)
RUN npm install --ignore-scripts
COPY --chown=$user:$user packages/fork-sax/package*.json ./packages/fork-sax/
COPY --chown=$user:$user packages/lib/package*.json ./packages/lib/
COPY --chown=$user:$user packages/renderer/package*.json ./packages/renderer/
COPY --chown=$user:$user packages/tools/package*.json ./packages/tools/
COPY --chown=$user:$user packages/server/package*.json ./packages/server/
COPY --chown=$user:$user lerna.json .
COPY --chown=$user:$user tsconfig.json .
# The following have postinstall scripts so we need to copy all the files.
# Since they should rarely change this is not an issue
COPY --chown=$user:$user packages/turndown ./packages/turndown
COPY --chown=$user:$user packages/turndown-plugin-gfm ./packages/turndown-plugin-gfm
COPY --chown=$user:$user packages/fork-htmlparser2 ./packages/fork-htmlparser2
RUN ls -la /home/$user
# Then bootstrap only, without compiling the TypeScript files
RUN npm run bootstrap
COPY --chown=$user:$user packages/fork-sax ./packages/fork-sax
COPY --chown=$user:$user packages/lib ./packages/lib
COPY --chown=$user:$user packages/renderer ./packages/renderer
COPY --chown=$user:$user packages/tools ./packages/tools
COPY --chown=$user:$user packages/server ./packages/server
# Finally build everything, in particular the TypeScript files.
RUN npm run build
EXPOSE ${JOPLIN_PORT}
CMD [ "npm", "--prefix", "packages/server", "start" ]

28
docker-compose.server-dev.yml Normal file
View File

@ -0,0 +1,28 @@
# For development, the easiest might be to only start the Postgres container and
# run the app directly with `npm start`. Or use sqlite3.
version: '3'
services:
# app:
# build:
# context: .
# dockerfile: Dockerfile.server-dev
# ports:
# - "22300:22300"
# # volumes:
# # - ./packages/server/:/var/www/joplin/packages/server/
# # - /var/www/joplin/packages/server/node_modules/
db:
build:
context: .
dockerfile: Dockerfile.db
ports:
- "5432:5432"
environment:
# TODO: Considering the database is only exposed to the
# application, and not to the outside world, is there a need to
# pick a secure password?
- POSTGRES_PASSWORD=joplin
- POSTGRES_USER=joplin
- POSTGRES_DB=joplin

40
docker-compose.server.yml Normal file
View File

@ -0,0 +1,40 @@
version: '3'
services:
app:
environment:
- JOPLIN_BASE_URL=${JOPLIN_BASE_URL}
- JOPLIN_PORT=${JOPLIN_PORT}
restart: unless-stopped
build:
context: .
dockerfile: Dockerfile.server
ports:
- "${JOPLIN_PORT}:${JOPLIN_PORT}"
# volumes:
# # Mount the server directory so that it's possible to edit file
# # while the container is running. However don't mount the
# # node_modules directory which will be specific to the Docker
# # image (eg native modules will be built for Ubuntu, while the
# # container might be running in Windows)
# # https://stackoverflow.com/a/37898591/561309
# - ./packages/server:/home/joplin/packages/server
# - /home/joplin/packages/server/node_modules/
db:
restart: unless-stopped
# By default, the Postgres image saves the data to a Docker volume,
# so it persists whenever the server is restarted using
# `docker-compose up`. Note that it would however be deleted when
# running `docker-compose down`.
build:
context: .
dockerfile: Dockerfile.db
ports:
- "5432:5432"
environment:
# TODO: Considering the database is only exposed to the
# application, and not to the outside world, is there a need to
# pick a secure password?
- POSTGRES_PASSWORD=joplin
- POSTGRES_USER=joplin
- POSTGRES_DB=joplin

5
joplin.code-workspace
View File

@ -116,6 +116,7 @@
"**/_vieux/": true,
"**/.DS_Store": true,
"**/*.base64": true,
"**/*~": true,
"**/*.bundle.js": true,
"**/*.eot": true,
"**/*.icns": true,
@ -309,6 +310,10 @@
"packages/renderer/MdToHtml/rules/sanitize_html.js": true,
"packages/app-mobile/lib/rnInjectedJs/": true,
"packages/app-mobile/lib/sql-extensions/spellfix.so": true,
"packages/server/dist/": true,
"packages/server/db-*.sqlite": true,
"packages/server/test.pid": true,
"packages/server/temp": true,
"packages/generator-joplin/generators/app/templates/api/": true,
"packages/app-mobile/node_modules/": true,
"phpunit.xml": true,

146
package-lock.json generated
View File

@ -289,6 +289,75 @@
"dedent": "^0.7.0",
"npmlog": "^4.1.2",
"yargs": "^14.2.2"
},
"dependencies": {
"ansi-regex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
"integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
"dev": true
},
"is-fullwidth-code-point": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
"integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
"dev": true
},
"string-width": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
"integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
"dev": true,
"requires": {
"emoji-regex": "^7.0.1",
"is-fullwidth-code-point": "^2.0.0",
"strip-ansi": "^5.1.0"
}
},
"strip-ansi": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
"integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
"dev": true,
"requires": {
"ansi-regex": "^4.1.0"
}
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"dev": true
},
"yargs": {
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
"integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
"dev": true,
"requires": {
"cliui": "^5.0.0",
"decamelize": "^1.2.0",
"find-up": "^3.0.0",
"get-caller-file": "^2.0.1",
"require-directory": "^2.1.1",
"require-main-filename": "^2.0.0",
"set-blocking": "^2.0.0",
"string-width": "^3.0.0",
"which-module": "^2.0.0",
"y18n": "^4.0.0",
"yargs-parser": "^15.0.1"
}
},
"yargs-parser": {
"version": "15.0.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
"integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
"dev": true,
"requires": {
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
}
}
}
},
"@lerna/collect-uncommitted": {
@ -2183,6 +2252,14 @@
"ssri": "^6.0.1",
"unique-filename": "^1.1.1",
"y18n": "^4.0.0"
},
"dependencies": {
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"dev": true
}
}
},
"cache-base": {
@ -10578,80 +10655,11 @@
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"dev": true
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"dev": true
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
"dev": true
},
"yargs": {
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
"integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
"dev": true,
"requires": {
"cliui": "^5.0.0",
"decamelize": "^1.2.0",
"find-up": "^3.0.0",
"get-caller-file": "^2.0.1",
"require-directory": "^2.1.1",
"require-main-filename": "^2.0.0",
"set-blocking": "^2.0.0",
"string-width": "^3.0.0",
"which-module": "^2.0.0",
"y18n": "^4.0.0",
"yargs-parser": "^15.0.1"
},
"dependencies": {
"ansi-regex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
"integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
"dev": true
},
"is-fullwidth-code-point": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
"integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
"dev": true
},
"string-width": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
"integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
"dev": true,
"requires": {
"emoji-regex": "^7.0.1",
"is-fullwidth-code-point": "^2.0.0",
"strip-ansi": "^5.1.0"
}
},
"strip-ansi": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
"integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
"dev": true,
"requires": {
"ansi-regex": "^4.1.0"
}
}
}
},
"yargs-parser": {
"version": "15.0.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
"integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
"dev": true,
"requires": {
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
}
}
}
}

9
package.json
View File

@ -20,12 +20,16 @@
"buildTranslationsNoTsc": "node packages/tools/build-translation.js",
"buildWebsite": "npm run buildApiDoc && node ./packages/tools/build-website.js && npm run buildPluginDoc",
"clean": "lerna clean -y && lerna run clean",
"circularDependencyCheck": "npx madge --warning --circular --extensions js ./",
"generateDatabaseTypes": "node packages/tools/generate-database-types",
"linkChecker": "linkchecker https://joplinapp.org",
"linter-ci": "./node_modules/.bin/eslint --resolve-plugins-relative-to . --quiet --ext .js --ext .jsx --ext .ts --ext .tsx",
"linter-precommit": "./node_modules/.bin/eslint --resolve-plugins-relative-to . --fix --ext .js --ext .jsx --ext .ts --ext .tsx",
"linter": "./node_modules/.bin/eslint --resolve-plugins-relative-to . --fix --quiet --ext .js --ext .jsx --ext .ts --ext .tsx",
"postinstall": "lerna bootstrap --no-ci && npm run tsc",
"bootstrap": "lerna bootstrap --no-ci",
"bootstrapIgnoreScripts": "lerna bootstrap --ignore-scripts --no-ci",
"postinstall": "npm run bootstrap --no-ci && npm run build",
"build": "lerna run build && npm run tsc",
"publishAll": "git pull && lerna version --yes --no-private --no-git-tag-version && gulp completePublishAll",
"releaseAndroid": "node packages/tools/release-android.js",
"releaseCli": "node packages/tools/release-cli.js",
@ -39,7 +43,8 @@
"updateIgnored": "gulp updateIgnoredTypeScriptBuild",
"updatePluginTypes": "./packages/generator-joplin/updateTypes.sh",
"watch": "lerna run watch --stream --parallel",
"i": "lerna add --no-bootstrap --scope"
"i": "lerna add --no-bootstrap --scope",
"server-start-dev": "docker-compose --file docker-compose.server-dev.yml up"
},
"husky": {
"hooks": {

10
packages/app-cli/tests/Synchronizer.basics.ts
View File

@ -1,7 +1,7 @@
import Setting from '@joplin/lib/models/Setting';
import { allNotesFolders, remoteNotesAndFolders, localNotesFoldersSameAsRemote } from './test-utils-synchronizer';
const { syncTargetName, synchronizerStart, setupDatabaseAndSynchronizer, synchronizer, sleep, switchClient, syncTargetId, fileApi } = require('./test-utils.js');
const { syncTargetName, afterAllCleanUp, synchronizerStart, setupDatabaseAndSynchronizer, synchronizer, sleep, switchClient, syncTargetId, fileApi } = require('./test-utils.js');
const Folder = require('@joplin/lib/models/Folder.js');
const Note = require('@joplin/lib/models/Note.js');
const BaseItem = require('@joplin/lib/models/BaseItem.js');
@ -16,6 +16,10 @@ describe('Synchronizer.basics', function() {
done();
});
afterAll(async () => {
await afterAllCleanUp();
});
it('should create remote items', (async () => {
const folder = await Folder.save({ title: 'folder1' });
await Note.save({ title: 'un', parent_id: folder.id });
@ -123,10 +127,6 @@ describe('Synchronizer.basics', function() {
}));
it('should delete local notes', (async () => {
// For these tests we pass the context around for each user. This is to make sure that the "deletedItemsProcessed"
// property of the basicDelta() function is cleared properly at the end of a sync operation. If it is not cleared
// it means items will no longer be deleted locally via sync.
const folder1 = await Folder.save({ title: 'folder1' });
const note1 = await Note.save({ title: 'un', parent_id: folder1.id });
const note2 = await Note.save({ title: 'deux', parent_id: folder1.id });

49
packages/app-cli/tests/test-utils.ts
View File

@ -15,6 +15,7 @@ import KeychainServiceDriver from '@joplin/lib/services/keychain/KeychainService
import KeychainServiceDriverDummy from '@joplin/lib/services/keychain/KeychainServiceDriver.dummy';
import PluginRunner from '../app/services/plugins/PluginRunner';
import PluginService from '@joplin/lib/services/plugins/PluginService';
import FileApiDriverJoplinServer from '@joplin/lib/file-api-driver-joplinServer';
const fs = require('fs-extra');
const { JoplinDatabase } = require('@joplin/lib/joplin-database.js');
@ -43,12 +44,14 @@ const SyncTargetOneDrive = require('@joplin/lib/SyncTargetOneDrive.js');
const SyncTargetNextcloud = require('@joplin/lib/SyncTargetNextcloud.js');
const SyncTargetDropbox = require('@joplin/lib/SyncTargetDropbox.js');
const SyncTargetAmazonS3 = require('@joplin/lib/SyncTargetAmazonS3.js');
const SyncTargetJoplinServer = require('@joplin/lib/SyncTargetJoplinServer').default;
const EncryptionService = require('@joplin/lib/services/EncryptionService.js');
const DecryptionWorker = require('@joplin/lib/services/DecryptionWorker.js');
const RevisionService = require('@joplin/lib/services/RevisionService.js');
const ResourceFetcher = require('@joplin/lib/services/ResourceFetcher.js');
const WebDavApi = require('@joplin/lib/WebDavApi');
const DropboxApi = require('@joplin/lib/DropboxApi');
const JoplinServerApi = require('@joplin/lib/JoplinServerApi2').default;
const { OneDriveApi } = require('@joplin/lib/onedrive-api');
const { loadKeychainServiceAndSettings } = require('@joplin/lib/services/SettingUtils');
const md5 = require('md5');
@ -116,6 +119,7 @@ SyncTargetRegistry.addClass(SyncTargetOneDrive);
SyncTargetRegistry.addClass(SyncTargetNextcloud);
SyncTargetRegistry.addClass(SyncTargetDropbox);
SyncTargetRegistry.addClass(SyncTargetAmazonS3);
SyncTargetRegistry.addClass(SyncTargetJoplinServer);
let syncTargetName_ = '';
let syncTargetId_: number = null;
@ -132,7 +136,7 @@ function setSyncTargetName(name: string) {
syncTargetName_ = name;
syncTargetId_ = SyncTargetRegistry.nameToId(syncTargetName_);
sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 100;// 400;
isNetworkSyncTarget_ = ['nextcloud', 'dropbox', 'onedrive', 'amazon_s3'].includes(syncTargetName_);
isNetworkSyncTarget_ = ['nextcloud', 'dropbox', 'onedrive', 'amazon_s3', 'joplinServer'].includes(syncTargetName_);
synchronizers_ = [];
return previousName;
}
@ -142,6 +146,7 @@ setSyncTargetName('memory');
// setSyncTargetName('dropbox');
// setSyncTargetName('onedrive');
// setSyncTargetName('amazon_s3');
// setSyncTargetName('joplinServer');
// console.info(`Testing with sync target: ${syncTargetName_}`);
@ -214,6 +219,16 @@ async function afterEachCleanUp() {
KeymapService.destroyInstance();
}
async function afterAllCleanUp() {
if (fileApi()) {
try {
await fileApi().clearRoot();
} catch (error) {
console.warn('Could not clear sync target root:', error);
}
}
}
async function switchClient(id: number, options: any = null) {
options = Object.assign({}, { keychainEnabled: false }, options);
@ -346,7 +361,7 @@ async function setupDatabaseAndSynchronizer(id: number, options: any = null) {
if (!synchronizers_[id]) {
const SyncTargetClass = SyncTargetRegistry.classById(syncTargetId_);
const syncTarget = new SyncTargetClass(db(id));
await initFileApi();
await initFileApi(suiteName_);
syncTarget.setFileApi(fileApi());
syncTarget.setLogger(logger);
synchronizers_[id] = await syncTarget.synchronizer();
@ -361,6 +376,7 @@ async function setupDatabaseAndSynchronizer(id: number, options: any = null) {
resourceFetchers_[id] = new ResourceFetcher(() => { return synchronizers_[id].api(); });
kvStores_[id] = new KvStore();
await fileApi().initialize();
await fileApi().clearRoot();
}
@ -440,7 +456,7 @@ async function loadEncryptionMasterKey(id: number = null, useExisting = false) {
return masterKey;
}
async function initFileApi() {
async function initFileApi(suiteName: string) {
if (fileApis_[syncTargetId_]) return;
let fileApi = null;
@ -482,7 +498,6 @@ async function initFileApi() {
if (!process.argv.includes('--runInBand')) {
throw new Error('OneDrive tests must be run sequentially, with the --runInBand arg. eg `npm test -- --runInBand`');
}
const { parameters, setEnvOverride } = require('@joplin/lib/parameters.js');
@ -506,6 +521,16 @@ async function initFileApi() {
if (!amazonS3Creds || !amazonS3Creds.accessKeyId) throw new Error(`AWS auth JSON missing in ${amazonS3CredsPath} format should be: { "accessKeyId": "", "secretAccessKey": "", "bucket": "mybucket"}`);
const api = new S3({ accessKeyId: amazonS3Creds.accessKeyId, secretAccessKey: amazonS3Creds.secretAccessKey, s3UseArnRegion: true });
fileApi = new FileApi('', new FileApiDriverAmazonS3(api, amazonS3Creds.bucket));
} else if (syncTargetId_ == SyncTargetRegistry.nameToId('joplinServer')) {
// Note that to test the API in parallel mode, you need to use Postgres
// as database, as the SQLite database is not reliable when being
// read/write from multiple processes at the same time.
const api = new JoplinServerApi({
baseUrl: () => 'http://localhost:22300',
username: () => 'admin@localhost',
password: () => 'admin',
});
fileApi = new FileApi(`root:/Apps/Joplin-${suiteName}`, new FileApiDriverJoplinServer(api));
}
fileApi.setLogger(logger);
@ -743,18 +768,18 @@ class TestApp extends BaseApplication {
private middlewareCalls_: any[];
private logger_: LoggerWrapper;
constructor(hasGui = true) {
public constructor(hasGui = true) {
super();
this.hasGui_ = hasGui;
this.middlewareCalls_ = [];
this.logger_ = super.logger();
}
hasGui() {
public hasGui() {
return this.hasGui_;
}
async start(argv: any[]) {
public async start(argv: any[]) {
this.logger_.info('Test app starting...');
if (!argv.includes('--profile')) {
@ -775,7 +800,7 @@ class TestApp extends BaseApplication {
this.logger_.info('Test app started...');
}
async generalMiddleware(store: any, next: any, action: any) {
public async generalMiddleware(store: any, next: any, action: any) {
this.middlewareCalls_.push(true);
try {
await super.generalMiddleware(store, next, action);
@ -784,7 +809,7 @@ class TestApp extends BaseApplication {
}
}
async wait() {
public async wait() {
return new Promise((resolve) => {
const iid = shim.setInterval(() => {
if (!this.middlewareCalls_.length) {
@ -795,11 +820,11 @@ class TestApp extends BaseApplication {
});
}
async profileDir() {
public async profileDir() {
return Setting.value('profileDir');
}
async destroy() {
public async destroy() {
this.logger_.info('Test app stopping...');
await this.wait();
await ItemChange.waitForAllSaved();
@ -809,4 +834,4 @@ class TestApp extends BaseApplication {
}
}
module.exports = { exportDir, newPluginService, newPluginScript, synchronizerStart, afterEachCleanUp, syncTargetName, setSyncTargetName, syncDir, createTempDir, isNetworkSyncTarget, kvStore, expectThrow, logger, expectNotThrow, resourceService, resourceFetcher, tempFilePath, allSyncTargetItemsEncrypted, msleep, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, checkThrow, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, currentClientId, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };
module.exports = { afterAllCleanUp, exportDir, newPluginService, newPluginScript, synchronizerStart, afterEachCleanUp, syncTargetName, setSyncTargetName, syncDir, createTempDir, isNetworkSyncTarget, kvStore, expectThrow, logger, expectNotThrow, resourceService, resourceFetcher, tempFilePath, allSyncTargetItemsEncrypted, msleep, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, checkThrow, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, currentClientId, id, ids, sortedIds, at, createNTestNotes, createNTestFolders, createNTestTags, TestApp };

2
packages/lib/BaseApplication.ts
View File

@ -6,6 +6,7 @@ import reducer from './reducer';
import KeychainServiceDriver from './services/keychain/KeychainServiceDriver.node';
import { _, setLocale } from './locale';
import KvStore from './services/KvStore';
import SyncTargetJoplinServer from './SyncTargetJoplinServer';
const { createStore, applyMiddleware } = require('redux');
const { defaultState, stateUtils } = require('./reducer');
@ -681,6 +682,7 @@ export default class BaseApplication {
SyncTargetRegistry.addClass(SyncTargetWebDAV);
SyncTargetRegistry.addClass(SyncTargetDropbox);
SyncTargetRegistry.addClass(SyncTargetAmazonS3);
SyncTargetRegistry.addClass(SyncTargetJoplinServer);
try {
await shim.fsDriver().remove(tempDir);

174
packages/lib/JoplinServerApi2.ts Normal file
View File

@ -0,0 +1,174 @@
import shim from './shim';
const { rtrimSlashes } = require('./path-utils.js');
const JoplinError = require('./JoplinError');
const { stringify } = require('query-string');
interface Options {
baseUrl(): string;
username(): string;
password(): string;
}
enum ExecOptionsResponseFormat {
Json = 'json',
Text = 'text',
}
enum ExecOptionsTarget {
String = 'string',
File = 'file',
}
interface ExecOptions {
responseFormat?: ExecOptionsResponseFormat;
target?: ExecOptionsTarget;
path?: string;
source?: string;
}
export default class JoplinServerApi {
private options_: Options;
private session_: any;
public constructor(options: Options) {
this.options_ = options;
}
private baseUrl() {
return rtrimSlashes(this.options_.baseUrl());
}
private async session() {
// TODO: handle invalid session
if (this.session_) return this.session_;
this.session_ = await this.exec('POST', 'api/sessions', null, {
email: this.options_.username(),
password: this.options_.password(),
});
return this.session_;
}
private async sessionId() {
const session = await this.session();
return session ? session.id : '';
}
// private requestToCurl_(url: string, options: any) {
// const output = [];
// output.push('curl');
// output.push('-v');
// if (options.method) output.push(`-X ${options.method}`);
// if (options.headers) {
// for (const n in options.headers) {
// if (!options.headers.hasOwnProperty(n)) continue;
// output.push(`${'-H ' + '"'}${n}: ${options.headers[n]}"`);
// }
// }
// if (options.body) output.push(`${'--data ' + '\''}${JSON.stringify(options.body)}'`);
// output.push(url);
// return output.join(' ');
// }
public async exec(method: string, path: string = '', query: Record<string, any> = null, body: any = null, headers: any = null, options: ExecOptions = null) {
if (headers === null) headers = {};
if (options === null) options = {};
if (!options.responseFormat) options.responseFormat = ExecOptionsResponseFormat.Json;
if (!options.target) options.target = ExecOptionsTarget.String;
let sessionId = '';
if (path !== 'api/sessions' && !sessionId) {
sessionId = await this.sessionId();
}
if (sessionId) headers['X-API-AUTH'] = sessionId;
const fetchOptions: any = {};
fetchOptions.headers = headers;
fetchOptions.method = method;
if (options.path) fetchOptions.path = options.path;
if (body) {
if (typeof body === 'object') {
fetchOptions.body = JSON.stringify(body);
fetchOptions.headers['Content-Type'] = 'application/json';
} else {
fetchOptions.body = body;
}
fetchOptions.headers['Content-Length'] = `${shim.stringByteLength(fetchOptions.body)}`;
}
let url = `${this.baseUrl()}/${path}`;
if (query) {
url += url.indexOf('?') < 0 ? '?' : '&';
url += stringify(query);
}
let response: any = null;
// console.info('Joplin API Call', `${method} ${url}`, headers, options);
// console.info(this.requestToCurl_(url, fetchOptions));
if (options.source == 'file' && (method == 'POST' || method == 'PUT')) {
if (fetchOptions.path) {
const fileStat = await shim.fsDriver().stat(fetchOptions.path);
if (fileStat) fetchOptions.headers['Content-Length'] = `${fileStat.size}`;
}
response = await shim.uploadBlob(url, fetchOptions);
} else if (options.target == 'string') {
if (typeof body === 'string') fetchOptions.headers['Content-Length'] = `${shim.stringByteLength(body)}`;
response = await shim.fetch(url, fetchOptions);
} else {
// file
response = await shim.fetchBlob(url, fetchOptions);
}
const responseText = await response.text();
// console.info('Joplin API Response', responseText);
// Creates an error object with as much data as possible as it will appear in the log, which will make debugging easier
const newError = (message: string, code: number = 0) => {
// Gives a shorter response for error messages. Useful for cases where a full HTML page is accidentally loaded instead of
// JSON. That way the error message will still show there's a problem but without filling up the log or screen.
const shortResponseText = (`${responseText}`).substr(0, 1024);
return new JoplinError(`${method} ${path}: ${message} (${code}): ${shortResponseText}`, code);
};
let responseJson_: any = null;
const loadResponseJson = async () => {
if (!responseText) return null;
if (responseJson_) return responseJson_;
responseJson_ = JSON.parse(responseText);
if (!responseJson_) throw newError('Cannot parse JSON response', response.status);
return responseJson_;
};
if (!response.ok) {
if (options.target === 'file') throw newError('fetchBlob error', response.status);
let json = null;
try {
json = await loadResponseJson();
} catch (error) {
// Just send back the plain text in newErro()
}
if (json && json.message) {
throw newError(`${json.message}`, response.status);
}
throw newError('Unknown error', response.status);
}
if (options.responseFormat === 'text') return responseText;
const output = await loadResponseJson();
return output;
}
}

91
packages/lib/SyncTargetJoplinServer.ts Normal file
View File

@ -0,0 +1,91 @@
import FileApiDriverJoplinServer from './file-api-driver-joplinServer';
import Setting from './models/Setting';
import Synchronizer from './Synchronizer';
import { _ } from './locale.js';
import JoplinServerApi from './JoplinServerApi2';
const BaseSyncTarget = require('./BaseSyncTarget.js');
const { FileApi } = require('./file-api.js');
interface FileApiOptions {
path(): string;
username(): string;
password(): string;
directory(): string;
}
export default class SyncTargetJoplinServer extends BaseSyncTarget {
static id() {
return 9;
}
static supportsConfigCheck() {
return true;
}
static targetName() {
return 'joplinServer';
}
static label() {
return _('Joplin Server');
}
async isAuthenticated() {
return true;
}
static async newFileApi_(options: FileApiOptions) {
const apiOptions = {
baseUrl: () => options.path(),
username: () => options.username(),
password: () => options.password(),
};
const api = new JoplinServerApi(apiOptions);
const driver = new FileApiDriverJoplinServer(api);
const fileApi = new FileApi(() => `root:/${options.directory()}`, driver);
fileApi.setSyncTargetId(this.id());
await fileApi.initialize();
return fileApi;
}
static async checkConfig(options: FileApiOptions) {
const fileApi = await SyncTargetJoplinServer.newFileApi_(options);
fileApi.requestRepeatCount_ = 0;
const output = {
ok: false,
errorMessage: '',
};
try {
const result = await fileApi.stat('');
if (!result) throw new Error(`Sync directory not found: "${options.directory()}" on server "${options.path()}"`);
output.ok = true;
} catch (error) {
output.errorMessage = error.message;
if (error.code) output.errorMessage += ` (Code ${error.code})`;
}
return output;
}
async initFileApi() {
const fileApi = await SyncTargetJoplinServer.newFileApi_({
path: () => Setting.value('sync.9.path'),
username: () => Setting.value('sync.9.username'),
password: () => Setting.value('sync.9.password'),
directory: () => Setting.value('sync.9.directory'),
});
fileApi.setLogger(this.logger());
return fileApi;
}
async initSynchronizer() {
return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
}
}

1
packages/lib/Synchronizer.ts
View File

@ -336,6 +336,7 @@ export default class Synchronizer {
let syncLock = null;
try {
await this.api().initialize();
this.api().setTempDirName(Dirnames.Temp);
try {

196
packages/lib/file-api-driver-joplinServer.ts Normal file
View File

@ -0,0 +1,196 @@
import JoplinServerApi from './JoplinServerApi2';
const { dirname, basename } = require('./path-utils');
function removeTrailingColon(path: string) {
if (!path || !path.length) return '';
if (path[path.length - 1] === ':') return path.substr(0, path.length - 1);
return path;
}
// All input paths should be in the format: "SPECIAL_DIR:/path/to/file"
// The trailing colon must not be included as it's automatically added
// when doing the API call.
// Only supported special dir at the moment is "root"
export default class FileApiDriverJoplinServer {
private api_: JoplinServerApi;
public constructor(api: JoplinServerApi) {
this.api_ = api;
}
public async initialize(basePath: string) {
const pieces = removeTrailingColon(basePath).split('/');
if (!pieces.length) return;
let parent = pieces.splice(0, 1)[0];
for (const p of pieces) {
// Syncing with the root, which is ok, and in that
// case there's no sub-dir to create.
if (!p && pieces.length === 1) return;
const subPath = `${parent}/${p}`;
await this.mkdir(subPath);
parent = subPath;
}
}
public api() {
return this.api_;
}
public requestRepeatCount() {
return 3;
}
private metadataToStat_(md: any, path: string, isDeleted: boolean = false) {
const output = {
path: path,
updated_time: md.updated_time,
isDir: !!md.is_directory,
isDeleted: isDeleted,
};
// TODO - HANDLE DELETED
// if (md['.tag'] === 'deleted') output.isDeleted = true;
return output;
}
private metadataToStats_(mds: any[]) {
const output = [];
for (let i = 0; i < mds.length; i++) {
output.push(this.metadataToStat_(mds[i], mds[i].name));
}
return output;
}
private apiFilePath_(p: string) {
if (p !== 'root') p += ':';
return `api/files/${p}`;
}
public async stat(path: string) {
try {
const response = await this.api().exec('GET', this.apiFilePath_(path));
return this.metadataToStat_(response, path);
} catch (error) {
if (error.code === 404) return null;
throw error;
}
}
public async delta(path: string, options: any) {
const context = options ? options.context : null;
let cursor = context ? context.cursor : null;
while (true) {
try {
const query = cursor ? { cursor } : {};
const response = await this.api().exec('GET', `${this.apiFilePath_(path)}/delta`, query);
const stats = response.items.map((item: any) => {
return this.metadataToStat_(item.item, item.item.name, item.type === 3);
});
const output = {
items: stats,
hasMore: response.has_more,
context: { cursor: response.cursor },
};
return output;
} catch (error) {
// If there's an error related to an invalid cursor, clear the cursor and retry.
if (cursor && error.code === 'resyncRequired') {
cursor = null;
continue;
}
throw error;
}
}
}
public async list(path: string, options: any = null) {
options = {
context: null,
...options,
};
const query = options.context?.cursor ? { cursor: options.context.cursor } : null;
const results = await this.api().exec('GET', `${this.apiFilePath_(path)}/children`, query);
const newContext: any = {};
if (results.cursor) newContext.cursor = results.cursor;
return {
items: this.metadataToStats_(results.items),
hasMore: results.has_more,
context: newContext,
} as any;
}
public async get(path: string, options: any) {
if (!options) options = {};
if (!options.responseFormat) options.responseFormat = 'text';
try {
const response = await this.api().exec('GET', `${this.apiFilePath_(path)}/content`, null, null, null, options);
return response;
} catch (error) {
if (error.code !== 404) throw error;
return null;
}
}
private parentPath_(path: string) {
let output = dirname(path);
// This is the root or a special folder
if (output.split('/').length === 1) {
output = output.substr(0, output.length - 1);
}
return output;
}
private basename_(path: string) {
return basename(path);
}
public async mkdir(path: string) {
const parentPath = this.parentPath_(path);
const filename = this.basename_(path);
try {
const response = await this.api().exec('POST', `${this.apiFilePath_(parentPath)}/children`, null, {
name: filename,
is_directory: 1,
});
return response;
} catch (error) {
// 409 is OK - directory already exists
if (error.code !== 409) throw error;
}
}
public async put(path: string, content: any, options: any = null) {
return this.api().exec('PUT', `${this.apiFilePath_(path)}/content`, null, content, {
'Content-Type': 'application/octet-stream',
}, options);
}
public async delete(path: string) {
return this.api().exec('DELETE', this.apiFilePath_(path));
}
public format() {
throw new Error('Not supported');
}
public async clearRoot(path: string) {
await this.delete(path);
await this.mkdir(path);
}
}

30
packages/lib/file-api.js
View File

@ -8,6 +8,8 @@ const time = require('./time').default;
const { sprintf } = require('sprintf-js');
const Mutex = require('async-mutex').Mutex;
const logger = Logger.create('FileApi');
function requestCanBeRepeated(error) {
const errorCode = typeof error === 'object' && error.code ? error.code : null;
@ -61,8 +63,14 @@ class FileApi {
this.remoteDateOffset_ = 0;
this.remoteDateNextCheckTime_ = 0;
this.remoteDateMutex_ = new Mutex();
this.initialized_ = false;
}
async initialize() {
if (this.initialized_) return;
this.initialized_ = true;
if (this.driver_.initialize) return this.driver_.initialize(this.fullPath_(''));
}
async fetchRemoteDateOffset_() {
const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
@ -108,7 +116,7 @@ class FileApi {
this.remoteDateNextCheckTime_ = Date.now() + 10 * 60 * 1000;
}
} catch (error) {
this.logger().warn('Could not retrieve remote date - defaulting to device date:', error);
logger.warn('Could not retrieve remote date - defaulting to device date:', error);
this.remoteDateOffset_ = 0;
this.remoteDateNextCheckTime_ = Date.now() + 60 * 1000;
} finally {
@ -137,7 +145,7 @@ class FileApi {
}
baseDir() {
return this.baseDir_;
return typeof this.baseDir_ === 'function' ? this.baseDir_() : this.baseDir_;
}
tempDirName() {
@ -191,7 +199,7 @@ class FileApi {
if (!('includeDirs' in options)) options.includeDirs = true;
if (!('syncItemsOnly' in options)) options.syncItemsOnly = false;
this.logger().debug(`list ${this.baseDir()}`);
logger.debug(`list ${this.baseDir()}`);
const result = await tryAndRepeat(() => this.driver_.list(this.fullPath_(path), options), this.requestRepeatCount());
@ -216,18 +224,18 @@ class FileApi {
// Deprectated
setTimestamp(path, timestampMs) {
this.logger().debug(`setTimestamp ${this.fullPath_(path)}`);
logger.debug(`setTimestamp ${this.fullPath_(path)}`);
return tryAndRepeat(() => this.driver_.setTimestamp(this.fullPath_(path), timestampMs), this.requestRepeatCount());
// return this.driver_.setTimestamp(this.fullPath_(path), timestampMs);
}
mkdir(path) {
this.logger().debug(`mkdir ${this.fullPath_(path)}`);
logger.debug(`mkdir ${this.fullPath_(path)}`);
return tryAndRepeat(() => this.driver_.mkdir(this.fullPath_(path)), this.requestRepeatCount());
}
async stat(path) {
this.logger().debug(`stat ${this.fullPath_(path)}`);
logger.debug(`stat ${this.fullPath_(path)}`);
const output = await tryAndRepeat(() => this.driver_.stat(this.fullPath_(path)), this.requestRepeatCount());
@ -246,12 +254,12 @@ class FileApi {
get(path, options = null) {
if (!options) options = {};
if (!options.encoding) options.encoding = 'utf8';
this.logger().debug(`get ${this.fullPath_(path)}`);
logger.debug(`get ${this.fullPath_(path)}`);
return tryAndRepeat(() => this.driver_.get(this.fullPath_(path), options), this.requestRepeatCount());
}
async put(path, content, options = null) {
this.logger().debug(`put ${this.fullPath_(path)}`, options);
logger.debug(`put ${this.fullPath_(path)}`, options);
if (options && options.source === 'file') {
if (!(await this.fsDriver().exists(options.path))) throw new JoplinError(`File not found: ${options.path}`, 'fileNotFound');
@ -261,13 +269,13 @@ class FileApi {
}
delete(path) {
this.logger().debug(`delete ${this.fullPath_(path)}`);
logger.debug(`delete ${this.fullPath_(path)}`);
return tryAndRepeat(() => this.driver_.delete(this.fullPath_(path)), this.requestRepeatCount());
}
// Deprectated
move(oldPath, newPath) {
this.logger().debug(`move ${this.fullPath_(oldPath)} => ${this.fullPath_(newPath)}`);
logger.debug(`move ${this.fullPath_(oldPath)} => ${this.fullPath_(newPath)}`);
return tryAndRepeat(() => this.driver_.move(this.fullPath_(oldPath), this.fullPath_(newPath)), this.requestRepeatCount());
}
@ -281,7 +289,7 @@ class FileApi {
}
delta(path, options = null) {
this.logger().debug(`delta ${this.fullPath_(path)}`);
logger.debug(`delta ${this.fullPath_(path)}`);
return tryAndRepeat(() => this.driver_.delta(this.fullPath_(path), options), this.requestRepeatCount());
}
}

49
packages/lib/models/Setting.ts
View File

@ -1,5 +1,6 @@
import shim from '../shim';
import { _, supportedLocalesToLanguages, defaultLocale } from '../locale';
import { ltrimSlashes } from '../path-utils';
const BaseModel = require('../BaseModel').default;
const { Database } = require('../database.js');
const SyncTargetRegistry = require('../SyncTargetRegistry.js');
@ -309,6 +310,52 @@ class Setting extends BaseModel {
secure: true,
},
'sync.9.path': {
value: '',
type: SettingItemType.String,
section: 'sync',
show: (settings: any) => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('joplinServer');
},
public: true,
label: () => _('Joplin Server URL'),
description: () => emptyDirWarning,
},
'sync.9.directory': {
value: 'Apps/Joplin',
type: SettingItemType.String,
section: 'sync',
show: (settings: any) => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('joplinServer');
},
filter: value => {
return value ? ltrimSlashes(rtrimSlashes(value)) : '';
},
public: true,
label: () => _('Joplin Server Directory'),
},
'sync.9.username': {
value: '',
type: SettingItemType.String,
section: 'sync',
show: (settings: any) => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('joplinServer');
},
public: true,
label: () => _('Joplin Server username'),
},
'sync.9.password': {
value: '',
type: SettingItemType.String,
section: 'sync',
show: (settings: any) => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('joplinServer');
},
public: true,
label: () => _('Joplin Server password'),
secure: true,
},
'sync.5.syncTargets': { value: {}, type: SettingItemType.Object, public: false },
'sync.resourceDownloadMode': {
@ -333,6 +380,7 @@ class Setting extends BaseModel {
'sync.3.auth': { value: '', type: SettingItemType.String, public: false },
'sync.4.auth': { value: '', type: SettingItemType.String, public: false },
'sync.7.auth': { value: '', type: SettingItemType.String, public: false },
'sync.9.auth': { value: '', type: SettingItemType.String, public: false },
'sync.1.context': { value: '', type: SettingItemType.String, public: false },
'sync.2.context': { value: '', type: SettingItemType.String, public: false },
'sync.3.context': { value: '', type: SettingItemType.String, public: false },
@ -341,6 +389,7 @@ class Setting extends BaseModel {
'sync.6.context': { value: '', type: SettingItemType.String, public: false },
'sync.7.context': { value: '', type: SettingItemType.String, public: false },
'sync.8.context': { value: '', type: SettingItemType.String, public: false },
'sync.9.context': { value: '', type: SettingItemType.String, public: false },
'sync.maxConcurrentConnections': { value: 5, type: SettingItemType.Int, public: true, advanced: true, section: 'sync', label: () => _('Max concurrent connections'), minimum: 1, maximum: 20, step: 1 },

2
packages/lib/shim.ts
View File

@ -181,7 +181,7 @@ const shim = {
throw new Error('Not implemented');
},
uploadBlob: () => {
uploadBlob: (_url: string, _options: any) => {
throw new Error('Not implemented');
},

2
packages/lib/time.ts
View File

@ -109,7 +109,7 @@ class Time {
}
msleep(ms: number) {
return new Promise((resolve) => {
return new Promise((resolve: Function) => {
shim.setTimeout(() => {
resolve();
}, ms);

5
packages/lib/uuid.ts
View File

@ -1,6 +1,11 @@
const createUuidV4 = require('uuid/v4');
const { customAlphabet } = require('nanoid/non-secure');
// https://zelark.github.io/nano-id-cc/
// https://security.stackexchange.com/a/41749/1873
// > On the other hand, 128 bits (between 21 and 22 characters
// > alphanumeric) is beyond the reach of brute-force attacks pretty much
// > indefinitely
const nanoid = customAlphabet('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', 22);
export default {

9
packages/server/.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
node_modules/
dist
db.sqlite
db-*.sqlite
*.sqlite-journal
*.pid
logs/
tests/temp/
temp/

66
packages/server/README.md Normal file
View File

@ -0,0 +1,66 @@
# Installing
## Configuration
First copy `.env-sample` to `.env` and edit the values in there:
- `JOPLIN_BASE_URL`: This is the base public URL where the service will be running. For example, if you want it to run from `https://example.com/joplin`, this is what you should set the URL to. The base URL can include the port.
- `JOPLIN_PORT`: The local port on which the Docker container will listen. You would typically map this port to 443 (TLS) with a reverse proxy.
## Install application
```shell
git clone https://github.com/laurent22/joplin
cd joplin
npm install
docker-compose --file docker-compose.server.yml up --detach
```
This will start the server, which will listen on port **22300** on **localhost**.
Due to the restart policy defined in the docker-compose file, the server will be restarted automatically whenever the host reboots.
## Setup reverse proxy
You will then need to expose this server to the internet by setting up a reverse proxy, and that will depend on how your server is currently configured, and whether you already have Nginx or Apache running:
- [Apache Reverse Proxy](https://httpd.apache.org/docs/current/mod/mod_proxy.html)
- [Nginx Reverse Proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/)
## Setup admin user
For the following instructions, we'll assume that the Joplin server is running on `https://example.com/joplin`.
By default, the instance will be setup with an admin user with email **admin@localhost** and password **admin** and you should change this by opening the admin UI. To do so, open `https://example.com/joplin/login`. From there, go to Profile and change the admin password.
## Setup a user for sync
While the admin user can be used for synchronisation, it is recommended to create a separate non-admin user for it. To do, open the admin UI and navigate to the Users page - from there you can create a new user.
Once this is done, you can use the email and password you specified to sync this user account with your Joplin clients.
## Checking the logs
Checking the log can be done the standard Docker way:
```shell
docker-compose --file docker-compose.server.yml logs
```
# Set up for development
## Setting up the database
### SQLite
The server supports SQLite for development and test units. To use it, open `src/config-dev.ts` and uncomment the sqlite3 config.
### PostgreSQL
It's best to use PostgreSQL as this is what is used in production, however it requires Docker.
To use it, from the monorepo root, run `docker-compose --file docker-compose.server-dev.yml up`, which will start the PostgreSQL database.
## Starting the server
From `packages/server`, run `npm run start-dev`

BIN
packages/server/assets/tests/photo.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.7 KiB

BIN
packages/server/assets/tests/poster.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

14
packages/server/jest.config.js Normal file
View File

@ -0,0 +1,14 @@
module.exports = {
testMatch: [
'**/*.test.js',
],
testPathIgnorePatterns: [
'<rootDir>/node_modules/',
'<rootDir>/assets/',
],
testEnvironment: 'node',
slowTestThreshold: 20,
};

4
packages/server/nodemon.json Normal file
View File

@ -0,0 +1,4 @@
{
"verbose": true,
"watch": ["dist/"]
}

8070
packages/server/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

44
packages/server/package.json Normal file
View File

@ -0,0 +1,44 @@
{
"name": "@joplin/server",
"version": "0.0.0",
"private": true,
"scripts": {
"start-dev": "nodemon --config nodemon.json dist/app.js --env dev",
"start": "node dist/app.js",
"generate-types": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-db --env buildTypes && node dist/tools/generate-types.js && rm -f db-buildTypes.sqlite",
"tsc": "tsc --project tsconfig.json",
"test": "jest",
"test-ci": "npm run test",
"watch": "tsc --watch --project tsconfig.json"
},
"dependencies": {
"@joplin/lib": "^1.0.9",
"bcryptjs": "^2.4.3",
"bulma": "^0.9.1",
"bulma-prefers-dark": "^0.1.0-beta.0",
"formidable": "^1.2.2",
"fs-extra": "^8.1.0",
"html-entities": "^1.3.1",
"knex": "^0.19.4",
"koa": "^2.8.1",
"mustache": "^3.1.0",
"nanoid": "^2.1.1",
"nodemon": "^2.0.6",
"pg": "^8.5.1",
"query-string": "^6.8.3",
"sqlite3": "^4.1.0",
"yargs": "^14.0.0"
},
"devDependencies": {
"@joplin/tools": "^1.0.9",
"@rmp135/sql-ts": "^1.7.0",
"@types/fs-extra": "^8.0.0",
"@types/jest": "^26.0.15",
"@types/koa": "^2.0.49",
"@types/mustache": "^0.8.32",
"@types/yargs": "^13.0.2",
"jest": "^26.6.3",
"source-map-support": "^0.5.13",
"typescript": "^4.1.2"
}
}

7
packages/server/public/css/bootstrap-grid.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

8
packages/server/public/css/bootstrap-reboot.min.css vendored Normal file
View File

@ -0,0 +1,8 @@
/*!
* Bootstrap Reboot v4.3.1 (https://getbootstrap.com/)
* Copyright 2011-2019 The Bootstrap Authors
* Copyright 2011-2019 Twitter, Inc.
* Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
* Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md)
*/*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}
/*# sourceMappingURL=bootstrap-reboot.min.css.map */

7
packages/server/public/css/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

3
packages/server/public/css/index/login.css Normal file
View File

@ -0,0 +1,3 @@
.page-login .login-box .container {
max-width: 400px;
}

27
packages/server/public/css/main.css Normal file
View File

@ -0,0 +1,27 @@
* {
box-sizing: border-box;
}
html {
font-size: 100%;
}
input.form-control {
margin-bottom: 0.5rem;
}
.navbar {
padding: 1rem 3rem;
}
.navbar .logo-container {
align-items: center;
}
.navbar .logo {
height: 50px;
}
.main {
padding: 0 3rem;
}

10
packages/server/public/css/oauth2/authorize.css Normal file
View File

@ -0,0 +1,10 @@
.form-signin {
width: 100%;
max-width: 330px;
padding: 15px;
margin: auto;
}
.authCode {
text-align: center;
}

BIN
packages/server/public/images/Logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 22 KiB

7
packages/server/public/js/bootstrap.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

8
packages/server/public/js/main.js Normal file
View File

@ -0,0 +1,8 @@
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
function onDocumentReady(fn) {
if (document.readyState != 'loading') {
fn();
} else {
document.addEventListener('DOMContentLoaded', fn);
}
}

159
packages/server/src/app.ts Normal file
View File

@ -0,0 +1,159 @@
// Allows displaying error stack traces with TypeScript file paths
import * as Koa from 'koa';
import routes from './routes/routes';
import { ErrorNotFound } from './utils/errors';
import * as fs from 'fs-extra';
import { argv } from 'yargs';
import { routeResponseFormat, findMatchingRoute, Response, RouteResponseFormat, MatchedRoute } from './utils/routeUtils';
import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger';
import config, { initConfig, baseUrl } from './config';
import configDev from './config-dev';
import configProd from './config-prod';
import configBuildTypes from './config-buildTypes';
import { createDb, dropDb } from './tools/dbTools';
import { dropTables, connectDb, disconnectDb, migrateDb, waitForConnection } from './db';
import modelFactory from './models/factory';
import controllerFactory from './controllers/factory';
import { AppContext, Config } from './utils/types';
import FsDriverNode from '@joplin/lib/fs-driver-node';
import mustacheService, { isView, View } from './services/MustacheService';
interface Configs {
[name: string]: Config;
}
const configs: Configs = {
dev: configDev,
prod: configProd,
buildTypes: configBuildTypes,
};
require('source-map-support').install();
const env: string = argv.env as string || 'prod';
const { shimInit } = require('@joplin/lib/shim-init-node.js');
shimInit();
let appLogger_: LoggerWrapper = null;
function appLogger(): LoggerWrapper {
if (!appLogger_) appLogger_ = Logger.create('App');
return appLogger_;
}
const app = new Koa();
app.use(async (ctx: Koa.Context) => {
appLogger().info(`${ctx.request.method} ${ctx.path}`);
const match: MatchedRoute = null;
try {
const match = findMatchingRoute(ctx.path, routes);
if (match) {
const responseObject = await match.route.exec(match.subPath, ctx);
if (responseObject instanceof Response) {
ctx.response = responseObject.response;
} else if (isView(responseObject)) {
ctx.response.status = 200;
ctx.response.body = await mustacheService.renderView(responseObject);
} else {
ctx.response.status = 200;
ctx.response.body = responseObject;
}
} else {
throw new ErrorNotFound();
}
} catch (error) {
appLogger().error(error);
ctx.response.status = error.httpCode ? error.httpCode : 500;
const responseFormat = routeResponseFormat(match);
if (responseFormat === RouteResponseFormat.Html) {
ctx.response.set('Content-Type', 'text/html');
const view: View = {
name: 'error',
path: 'index/error',
content: {
error,
},
};
ctx.response.body = await mustacheService.renderView(view);
} else { // JSON
ctx.response.set('Content-Type', 'application/json');
const r: any = { error: error.message };
if (env == 'dev' && error.stack) r.stack = error.stack;
if (error.code) r.code = error.code;
ctx.response.body = r;
}
}
});
async function main() {
const configObject: Config = configs[env];
if (!configObject) throw new Error(`Invalid env: ${env}`);
initConfig(configObject);
await fs.mkdirp(config().logDir);
Logger.fsDriver_ = new FsDriverNode();
const globalLogger = new Logger();
globalLogger.addTarget(TargetType.File, { path: `${config().logDir}/app.txt` });
globalLogger.addTarget(TargetType.Console);
Logger.initializeGlobalLogger(globalLogger);
const pidFile = argv.pidfile as string;
if (pidFile) {
appLogger().info(`Writing PID to ${pidFile}...`);
fs.removeSync(pidFile as string);
fs.writeFileSync(pidFile, `${process.pid}`);
}
if (argv.migrateDb) {
const db = await connectDb(config().database);
await migrateDb(db);
await disconnectDb(db);
} else if (argv.dropDb) {
await dropDb(config().database, { ignoreIfNotExists: true });
} else if (argv.dropTables) {
const db = await connectDb(config().database);
await dropTables(db);
await disconnectDb(db);
} else if (argv.createDb) {
await createDb(config().database);
} else {
appLogger().info(`Starting server (${env}) on port ${config().port} and PID ${process.pid}...`);
appLogger().info('Public base URL:', baseUrl());
appLogger().info('DB Config:', config().database);
const appContext = app.context as AppContext;
appLogger().info('Trying to connect to database...');
const connectionCheck = await waitForConnection(config().database);
const connectionCheckLogInfo = { ...connectionCheck };
delete connectionCheckLogInfo.connection;
appLogger().info('Connection check:', connectionCheckLogInfo);
appContext.db = connectionCheck.connection;//
appContext.models = modelFactory(appContext.db);
appContext.controllers = controllerFactory(appContext.models);
appLogger().info('Migrating database...');
await migrateDb(appContext.db);
appLogger().info(`Call this for testing: \`curl ${baseUrl()}/api/ping\``);
app.listen(config().port);
}
}
main().catch((error: any) => {
console.error(error);
process.exit(1);
});

21
packages/server/src/config-base.ts Normal file
View File

@ -0,0 +1,21 @@
import { Config } from './utils/types';
import * as pathUtils from 'path';
const rootDir = pathUtils.dirname(__dirname);
const viewDir = `${pathUtils.dirname(__dirname)}/src/views`;
const envPort = Number(process.env.JOPLIN_PORT);
const config: Config = {
port: (envPort && !isNaN(envPort)) ? envPort : 22300,
viewDir: viewDir,
rootDir: rootDir,
layoutDir: `${viewDir}/layouts`,
logDir: `${rootDir}/logs`,
database: {
client: 'pg',
name: 'joplin',
},
};
export default config;

13
packages/server/src/config-buildTypes.ts Normal file
View File

@ -0,0 +1,13 @@
import { Config } from './utils/types';
import configBase from './config-base';
const config: Config = {
...configBase,
database: {
name: 'buildTypes',
client: 'sqlite3',
asyncStackTraces: true,
},
};
export default config;

22
packages/server/src/config-dev.ts Normal file
View File

@ -0,0 +1,22 @@
import { Config } from './utils/types';
import configBase from './config-base';
const config: Config = {
...configBase,
// database: {
// name: 'dev',
// client: 'sqlite3',
// asyncStackTraces: true,
// },
database: {
client: 'pg',
name: 'joplin',
user: 'joplin',
host: 'localhost',
port: 5432,
password: 'joplin',
asyncStackTraces: true,
},
};
export default config;

20
packages/server/src/config-prod.ts Normal file
View File

@ -0,0 +1,20 @@
import { Config } from './utils/types';
import configBase from './config-base';
const rootDir = '/home/joplin/';
const config: Config = {
...configBase,
rootDir: rootDir,
logDir: `${rootDir}/logs`,
database: {
client: 'pg',
name: 'joplin',
user: 'joplin',
host: 'db',
port: 5432,
password: 'joplin',
},
};
export default config;

13
packages/server/src/config-tests.ts Normal file
View File

@ -0,0 +1,13 @@
import { Config } from './utils/types';
import configBase from './config-base';
const config: Config = {
...configBase,
database: {
name: 'DYNAMIC',
client: 'sqlite3',
asyncStackTraces: true,
},
};
export default config;

28
packages/server/src/config.ts Normal file
View File

@ -0,0 +1,28 @@
import { rtrimSlashes } from '@joplin/lib/path-utils';
import { Config } from './utils/types';
let baseConfig_: Config = null;
let baseUrl_: string = null;
export function initConfig(baseConfig: Config) {
baseConfig_ = baseConfig;
}
function config(): Config {
if (!baseConfig_) throw new Error('Config has not been initialized!');
return baseConfig_;
}
export function baseUrl() {
if (baseUrl_) return baseUrl_;
if (process.env.JOPLIN_BASE_URL) {
baseUrl_ = rtrimSlashes(process.env.JOPLIN_BASE_URL);
} else {
baseUrl_ = `http://localhost:${config().port}`;
}
return baseUrl_;
}
export default config;

25
packages/server/src/controllers/BaseController.ts Normal file
View File

@ -0,0 +1,25 @@
import { User } from '../db';
import { Models } from '../models/factory';
import { ErrorForbidden } from '../utils/errors';
export default abstract class BaseController {
private models_: Models;
public constructor(models: Models) {
this.models_ = models;
}
protected get models(): Models {
return this.models_;
}
protected async initSession(sessionId: string, mustBeAdmin: boolean = false): Promise<User> {
if (!sessionId) throw new ErrorForbidden('Session is required');
const user: User = await this.models.session().sessionUser(sessionId);
if (!user) throw new ErrorForbidden(`Invalid session ID: ${sessionId}`);
if (!user.is_admin && mustBeAdmin) throw new ErrorForbidden('Non-admin user is not allowed');
return user;
}
}

535
packages/server/src/controllers/api/FileController.test.ts Normal file
View File

@ -0,0 +1,535 @@
import { testAssetDir, createUserAndSession, createUser, checkThrowAsync, beforeAllDb, afterAllDb, beforeEachDb, models, controllers } from '../../utils/testUtils';
import * as fs from 'fs-extra';
import { ChangeType, File } from '../../db';
import { ErrorConflict, ErrorForbidden, ErrorNotFound, ErrorUnprocessableEntity } from '../../utils/errors';
import { filePathInfo } from '../../utils/routeUtils';
import { defaultPagination, Pagination, PaginationOrderDir } from '../../models/utils/pagination';
import { msleep } from '../../utils/time';
async function makeTestFile(id: number = 1, ext: string = 'jpg', parentId: string = ''): Promise<File> {
const basename = ext === 'jpg' ? 'photo' : 'poster';
const file: File = {
name: id > 1 ? `${basename}-${id}.${ext}` : `${basename}.${ext}`,
content: await fs.readFile(`${testAssetDir}/${basename}.${ext}`),
// mime_type: `image/${ext}`,
parent_id: parentId,
};
return file;
}
async function makeTestContent(ext: string = 'jpg') {
const basename = ext === 'jpg' ? 'photo' : 'poster';
return await fs.readFile(`${testAssetDir}/${basename}.${ext}`);
}
async function makeTestDirectory(name: string = 'Docs'): Promise<File> {
const file: File = {
name: name,
parent_id: '',
is_directory: 1,
};
return file;
}
async function saveTestFile(sessionId: string, path: string): Promise<File> {
const fileController = controllers().apiFile();
return fileController.putFileContent(
sessionId,
path,
null
);
}
async function saveTestDir(sessionId: string, path: string): Promise<File> {
const fileController = controllers().apiFile();
const parsed = filePathInfo(path);
return fileController.postChild(
sessionId,
parsed.dirname,
{
name: parsed.basename,
is_directory: 1,
}
);
}
describe('FileController', function() {
beforeAll(async () => {
await beforeAllDb('FileController');
});
afterAll(async () => {
await afterAllDb();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should create a file', async function() {
const { user, session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const fileContent = await makeTestContent();
const newFile = await fileController.putFileContent(
session.id,
'root:/photo.jpg:',
fileContent
);
expect(!!newFile.id).toBe(true);
expect(newFile.name).toBe('photo.jpg');
expect(newFile.mime_type).toBe('image/jpeg');
expect(!!newFile.parent_id).toBe(true);
expect(!newFile.content).toBe(true);
expect(newFile.size > 0).toBe(true);
const fileModel = models().file({ userId: user.id });
const newFileReload = await fileModel.loadWithContent(newFile.id);
expect(!!newFileReload).toBe(true);
const newFileHex = fileContent.toString('hex');
const newFileReloadHex = (newFileReload.content as Buffer).toString('hex');
expect(newFileReloadHex.length > 0).toBe(true);
expect(newFileReloadHex).toBe(newFileHex);
});
test('should create sub-directories', async function() {
const { session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const newDir = await fileController.postFile_(session.id, {
is_directory: 1,
name: 'subdir',
});
expect(!!newDir.id).toBe(true);
expect(newDir.is_directory).toBe(1);
const newDir2 = await fileController.postFile_(session.id, {
is_directory: 1,
name: 'subdir2',
parent_id: newDir.id,
});
const newDirReload2 = await fileController.getFile(session.id, 'root:/subdir/subdir2');
expect(newDirReload2.id).toBe(newDir2.id);
expect(newDirReload2.name).toBe(newDir2.name);
});
test('should create files in sub-directory', async function() {
const { session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
await fileController.postFile_(session.id, {
is_directory: 1,
name: 'subdir',
});
const newFile = await fileController.putFileContent(
session.id,
'root:/subdir/photo.jpg:',
await makeTestContent()
);
const newFileReload = await fileController.getFile(session.id, 'root:/subdir/photo.jpg');
expect(newFileReload.id).toBe(newFile.id);
expect(newFileReload.name).toBe('photo.jpg');
});
test('should not create a file with an invalid path', async function() {
const { session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const fileContent = await makeTestContent();
const error = await checkThrowAsync(async () => fileController.putFileContent(
session.id,
'root:/does/not/exist/photo.jpg:',
fileContent
));
expect(error instanceof ErrorNotFound).toBe(true);
});
test('should get files', async function() {
const { session: session1, user: user1 } = await createUserAndSession(1);
const { session: session2 } = await createUserAndSession(2);
let file1: File = await makeTestFile(1);
let file2: File = await makeTestFile(2);
let file3: File = await makeTestFile(3);
const fileController = controllers().apiFile();
file1 = await fileController.postFile_(session1.id, file1);
file2 = await fileController.postFile_(session1.id, file2);
file3 = await fileController.postFile_(session2.id, file3);
const fileId1 = file1.id;
const fileId2 = file2.id;
// Can't get someone else file
const error = await checkThrowAsync(async () => fileController.getFile(session1.id, file3.id));
expect(error instanceof ErrorForbidden).toBe(true);
file1 = await fileController.getFile(session1.id, file1.id);
expect(file1.id).toBe(fileId1);
const fileModel = models().file({ userId: user1.id });
const paginatedResults = await fileController.getChildren(session1.id, await fileModel.userRootFileId(), defaultPagination());
const allFiles = paginatedResults.items;
expect(allFiles.length).toBe(2);
expect(JSON.stringify(allFiles.map(f => f.id).sort())).toBe(JSON.stringify([fileId1, fileId2].sort()));
});
test('should not let create a file in a directory not owned by user', async function() {
const { session } = await createUserAndSession(1);
const user2 = await createUser(2);
const fileModel2 = models().file({ userId: user2.id });
const rootFile2 = await fileModel2.userRootFile();
const file: File = await makeTestFile();
file.parent_id = rootFile2.id;
const fileController = controllers().apiFile();
const hasThrown = await checkThrowAsync(async () => fileController.postFile_(session.id, file));
expect(!!hasThrown).toBe(true);
});
test('should update file properties', async function() {
const { session, user } = await createUserAndSession(1, true);
const fileModel = models().file({ userId: user.id });
let file: File = await makeTestFile();
const fileController = controllers().apiFile();
file = await fileController.postFile_(session.id, file);
// Can't have file with empty name
const error = await checkThrowAsync(async () => fileController.patchFile(session.id, file.id, { name: '' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
await fileController.patchFile(session.id, file.id, { name: 'modified.jpg' });
file = await fileModel.load(file.id);
expect(file.name).toBe('modified.jpg');
await fileController.patchFile(session.id, file.id, { mime_type: 'image/png' });
file = await fileModel.load(file.id);
expect(file.mime_type).toBe('image/png');
});
test('should not allow duplicate filenames', async function() {
const { session } = await createUserAndSession(1, true);
let file1: File = await makeTestFile(1);
const file2: File = await makeTestFile(1);
const fileController = controllers().apiFile();
file1 = await fileController.postFile_(session.id, file1);
expect(!!file1.id).toBe(true);
expect(file1.name).toBe(file2.name);
const hasThrown = await checkThrowAsync(async () => await fileController.postFile_(session.id, file2));
expect(!!hasThrown).toBe(true);
});
test('should change the file parent', async function() {
const { session: session1, user: user1 } = await createUserAndSession(1);
const { user: user2 } = await createUserAndSession(2);
let hasThrown: any = null;
const fileModel = models().file({ userId: user1.id });
let file: File = await makeTestFile();
let file2: File = await makeTestFile(2);
let dir: File = await makeTestDirectory();
const fileController = controllers().apiFile();
file = await fileController.postFile_(session1.id, file);
file2 = await fileController.postFile_(session1.id, file2);
dir = await fileController.postFile_(session1.id, dir);
// Can't set parent to another non-directory file
hasThrown = await checkThrowAsync(async () => await fileController.patchFile(session1.id, file.id, { parent_id: file2.id }));
expect(!!hasThrown).toBe(true);
const fileModel2 = models().file({ userId: user2.id });
const userRoot2 = await fileModel2.userRootFile();
// Can't set parent to someone else directory
hasThrown = await checkThrowAsync(async () => await fileController.patchFile(session1.id, file.id, { parent_id: userRoot2.id }));
expect(!!hasThrown).toBe(true);
await fileController.patchFile(session1.id, file.id, { parent_id: dir.id });
file = await fileModel.load(file.id);
expect(!!file.parent_id).toBe(true);
expect(file.parent_id).toBe(dir.id);
});
test('should delete a file', async function() {
const { user, session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const fileModel = models().file({ userId: user.id });
const file1: File = await makeTestFile(1);
let file2: File = await makeTestFile(2);
await fileController.postFile_(session.id, file1);
file2 = await fileController.postFile_(session.id, file2);
let allFiles: File[] = await fileModel.all();
const beforeCount: number = allFiles.length;
await fileController.deleteFile(session.id, file2.id);
allFiles = await fileModel.all();
expect(allFiles.length).toBe(beforeCount - 1);
});
test('should create and delete directories', async function() {
const { user, session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const dir1: File = await fileController.postChild(session.id, 'root', { name: 'dir1', is_directory: 1 });
const dir2: File = await fileController.postChild(session.id, 'root:/dir1', { name: 'dir2', is_directory: 1 });
const dirReload2: File = await fileController.getFile(session.id, 'root:/dir1/dir2');
expect(dirReload2.id).toBe(dir2.id);
// Delete one directory
await fileController.deleteFile(session.id, 'root:/dir1/dir2');
const error = await checkThrowAsync(async () => fileController.getFile(session.id, 'root:/dir1/dir2'));
expect(error instanceof ErrorNotFound).toBe(true);
// Delete a directory and its sub-directories and files
const dir3: File = await fileController.postChild(session.id, 'root:/dir1', { name: 'dir3', is_directory: 1 });
const file1: File = await fileController.postFile_(session.id, { name: 'file1', parent_id: dir1.id });
const file2: File = await fileController.postFile_(session.id, { name: 'file2', parent_id: dir3.id });
await fileController.deleteFile(session.id, 'root:/dir1');
const fileModel = models().file({ userId: user.id });
expect(!(await fileModel.load(dir1.id))).toBe(true);
expect(!(await fileModel.load(dir3.id))).toBe(true);
expect(!(await fileModel.load(file1.id))).toBe(true);
expect(!(await fileModel.load(file2.id))).toBe(true);
});
test('should not change the parent when updating a file', async function() {
const { user, session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const fileModel = models().file({ userId: user.id });
const dir1: File = await fileController.postChild(session.id, 'root', { name: 'dir1', is_directory: 1 });
const file1: File = await fileController.putFileContent(session.id, 'root:/dir1/myfile.md', Buffer.from('testing'));
await fileController.putFileContent(session.id, 'root:/dir1/myfile.md', Buffer.from('new content'));
const fileReloaded1 = await fileModel.load(file1.id);
expect(fileReloaded1.parent_id).toBe(dir1.id);
});
test('should not delete someone else file', async function() {
const { session: session1 } = await createUserAndSession(1);
const { session: session2 } = await createUserAndSession(2);
const fileController = controllers().apiFile();
const file1: File = await makeTestFile(1);
let file2: File = await makeTestFile(2);
await fileController.postFile_(session1.id, file1);
file2 = await fileController.postFile_(session2.id, file2);
const error = await checkThrowAsync(async () => await fileController.deleteFile(session1.id, file2.id));
expect(error instanceof ErrorForbidden).toBe(true);
});
test('should let admin change or delete files', async function() {
const { session: adminSession } = await createUserAndSession(1, true);
const { session, user } = await createUserAndSession(2);
let file: File = await makeTestFile();
const fileModel = models().file({ userId: user.id });
const fileController = controllers().apiFile();
file = await fileController.postFile_(session.id, file);
await fileController.patchFile(adminSession.id, file.id, { name: 'modified.jpg' });
file = await fileModel.load(file.id);
expect(file.name).toBe('modified.jpg');
await fileController.deleteFile(adminSession.id, file.id);
expect(!(await fileModel.load(file.id))).toBe(true);
});
test('should update a file content', async function() {
const { session } = await createUserAndSession(1, true);
const file: File = await makeTestFile(1);
const file2: File = await makeTestFile(2, 'png');
const fileController = controllers().apiFile();
const newFile = await fileController.postFile_(session.id, file);
await fileController.putFileContent(session.id, newFile.id, file2.content);
const modFile = await fileController.getFileContent(session.id, newFile.id);
const originalFileHex = (file.content as Buffer).toString('hex');
const modFileHex = (modFile.content as Buffer).toString('hex');
expect(modFileHex.length > 0).toBe(true);
expect(modFileHex === originalFileHex).toBe(false);
expect(modFile.size).toBe(modFile.content.byteLength);
expect(newFile.size).toBe(file.content.byteLength);
});
test('should delete a file content', async function() {
const { session } = await createUserAndSession(1, true);
const file: File = await makeTestFile(1);
const fileController = controllers().apiFile();
const newFile = await fileController.postFile_(session.id, file);
await fileController.putFileContent(session.id, newFile.id, file.content);
await fileController.deleteFileContent(session.id, newFile.id);
const modFile = await fileController.getFile(session.id, newFile.id);
expect(modFile.size).toBe(0);
});
test('should not allow reserved characters', async function() {
const { session } = await createUserAndSession(1, true);
const filenames = [
'invalid*invalid',
'invalid#invalid',
'invalid\\invalid',
];
const fileController = controllers().apiFile();
for (const filename of filenames) {
const error = await checkThrowAsync(async () => fileController.putFileContent(session.id, `root:/${filename}`, null));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
}
});
test('should not allow a directory with the same name', async function() {
const { session } = await createUserAndSession(1, true);
await saveTestDir(session.id, 'root:/somedir:');
let error = await checkThrowAsync(async () => saveTestFile(session.id, 'root:/somedir:'));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
await saveTestFile(session.id, 'root:/somefile.md:');
error = await checkThrowAsync(async () => saveTestDir(session.id, 'root:/somefile.md:'));
expect(error instanceof ErrorConflict).toBe(true);
});
test('should not be possible to delete the root directory', async function() {
const { session } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const error = await checkThrowAsync(async () => fileController.deleteFile(session.id, 'root'));
expect(error instanceof ErrorForbidden).toBe(true);
});
test('should support root:/: format, which means root', async function() {
const { session, user } = await createUserAndSession(1, true);
const fileController = controllers().apiFile();
const fileModel = models().file({ userId: user.id });
const root = await fileController.getFile(session.id, 'root:/:');
expect(root.id).toBe(await fileModel.userRootFileId());
});
test('should paginate results', async function() {
const { session: session1, user: user1 } = await createUserAndSession(1);
let file1: File = await makeTestFile(1);
let file2: File = await makeTestFile(2);
let file3: File = await makeTestFile(3);
const fileController = controllers().apiFile();
file1 = await fileController.postFile_(session1.id, file1);
await msleep(1);
file2 = await fileController.postFile_(session1.id, file2);
await msleep(1);
file3 = await fileController.postFile_(session1.id, file3);
const fileModel = models().file({ userId: user1.id });
const rootId = await fileModel.userRootFileId();
const pagination: Pagination = {
limit: 2,
order: [
{
by: 'updated_time',
dir: PaginationOrderDir.ASC,
},
],
page: 1,
};
for (const method of ['page', 'cursor']) {
const page1 = await fileController.getChildren(session1.id, rootId, pagination);
expect(page1.items.length).toBe(2);
expect(page1.has_more).toBe(true);
expect(page1.items[0].id).toBe(file1.id);
expect(page1.items[1].id).toBe(file2.id);
const p = method === 'page' ? { ...pagination, page: 2 } : { cursor: page1.cursor };
const page2 = await fileController.getChildren(session1.id, rootId, p);
expect(page2.items.length).toBe(1);
expect(page2.has_more).toBe(false);
expect(page2.items[0].id).toBe(file3.id);
}
});
test('should track file changes', async function() {
// We only do a basic check because most of the tests for this are in
// ChangeModel.test.ts
const { session: session1 } = await createUserAndSession(1);
let file1: File = await makeTestFile(1);
let file2: File = await makeTestFile(2);
const fileController = controllers().apiFile();
file1 = await fileController.postFile_(session1.id, file1);
await msleep(1); file2 = await fileController.postFile_(session1.id, file2);
const page1 = await fileController.getDelta(session1.id, file1.parent_id, { limit: 1 });
expect(page1.has_more).toBe(true);
expect(page1.items.length).toBe(1);
expect(page1.items[0].type).toBe(ChangeType.Create);
expect(page1.items[0].item.id).toBe(file1.id);
const page2 = await fileController.getDelta(session1.id, file1.parent_id, { cursor: page1.cursor, limit: 1 });
expect(page2.has_more).toBe(true);
expect(page2.items.length).toBe(1);
expect(page2.items[0].type).toBe(ChangeType.Create);
expect(page2.items[0].item.id).toBe(file2.id);
const page3 = await fileController.getDelta(session1.id, file1.parent_id, { cursor: page2.cursor, limit: 1 });
expect(page3.has_more).toBe(false);
expect(page3.items.length).toBe(0);
});
});

101
packages/server/src/controllers/api/FileController.ts Normal file
View File

@ -0,0 +1,101 @@
import { File } from '../../db';
import BaseController from '../BaseController';
import { ErrorNotFound } from '../../utils/errors';
import { Pagination } from '../../models/utils/pagination';
import { PaginatedFiles } from '../../models/FileModel';
import { ChangePagination, PaginatedChanges } from '../../models/ChangeModel';
export default class FileController extends BaseController {
// Note: this is only used in tests. To create files with no content
// or directories, use postChild()
public async postFile_(sessionId: string, file: File): Promise<File> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
let newFile = fileModel.fromApiInput(file);
newFile = await fileModel.save(file);
return fileModel.toApiOutput(newFile);
}
public async getFile(sessionId: string, fileId: string): Promise<File> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const file: File = await fileModel.entityFromItemId(fileId);
const loadedFile = await fileModel.load(file.id);
if (!loadedFile) throw new ErrorNotFound();
return fileModel.toApiOutput(loadedFile);
}
public async getFileContent(sessionId: string, fileId: string): Promise<File> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
let file: File = await fileModel.entityFromItemId(fileId);
file = await fileModel.loadWithContent(file.id);
if (!file) throw new ErrorNotFound();
return file;
}
public async patchFile(sessionId: string, fileId: string, file: File): Promise<File> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const existingFile: File = await fileModel.entityFromItemId(fileId);
const newFile = fileModel.fromApiInput(file);
newFile.id = existingFile.id;
return fileModel.toApiOutput(await fileModel.save(newFile));
}
public async putFileContent(sessionId: string, fileId: string, content: Buffer): Promise<any> {
if (!content) content = Buffer.alloc(0);
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const file: File = await fileModel.entityFromItemId(fileId, { mustExist: false });
file.content = content;
return fileModel.toApiOutput(await fileModel.save(file, { validationRules: { mustBeFile: true } }));
}
public async deleteFileContent(sessionId: string, fileId: string): Promise<any> {
await this.putFileContent(sessionId, fileId, null);
}
public async getChildren(sessionId: string, fileId: string, pagination: Pagination): Promise<PaginatedFiles> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const parent: File = await fileModel.entityFromItemId(fileId);
return fileModel.toApiOutput(await fileModel.childrens(parent.id, pagination));
}
public async postChild(sessionId: string, fileId: string, child: File): Promise<File> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const parent: File = await fileModel.entityFromItemId(fileId);
child = fileModel.fromApiInput(child);
child.parent_id = parent.id;
return fileModel.toApiOutput(await fileModel.save(child));
}
public async deleteFile(sessionId: string, fileId: string): Promise<void> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
try {
const file: File = await fileModel.entityFromItemId(fileId, { mustExist: false });
if (!file.id) return;
await fileModel.delete(file.id);
} catch (error) {
if (error instanceof ErrorNotFound) {
// That's ok - a no-op
} else {
throw error;
}
}
}
public async getDelta(sessionId: string, dirId: string, pagination: ChangePagination): Promise<PaginatedChanges> {
const user = await this.initSession(sessionId);
const fileModel = this.models.file({ userId: user.id });
const dir: File = await fileModel.entityFromItemId(dirId, { mustExist: true });
const changeModel = this.models.change({ userId: user.id });
return changeModel.byDirectoryId(dir.id, pagination);
}
}

54
packages/server/src/controllers/api/OAuthController.ts Normal file
View File

@ -0,0 +1,54 @@
// import BaseController from '../BaseController';
// import mustacheService from '../../services/MustacheService';
// import { ErrorNotFound } from '../../utils/errors';
// import uuidgen from '../../utils/uuidgen';
// import controllers from '../factory';
// export default class OAuthController extends BaseController {
// async getAuthorize(query: any): Promise<string> {
// const clientModel = this.models.apiClient();
// const client = await clientModel.load(query.client_id);
// if (!client) throw new ErrorNotFound(`client_id missing or invalid client ID: ${query.client_id}`);
// return mustacheService.render('oauth2/authorize', {
// response_type: query.response_type,
// client: client,
// }, {
// cssFiles: ['oauth2/authorize'],
// });
// }
// async postAuthorize(query: any): Promise<string> {
// const clientModel = this.models.apiClient();
// const sessionModel = this.models.session();
// const sessionController = controllers(this.models).session();
// let client = null;
// try {
// client = await clientModel.load(query.client_id);
// if (!client) throw new ErrorNotFound(`client_id missing or invalid client ID: ${query.client_id}`);
// const session = await sessionController.authenticate(query.email, query.password);
// const authCode = uuidgen(32);
// await sessionModel.save({ id: session.id, auth_code: authCode });
// return mustacheService.render('oauth2/authcode', {
// client: client,
// authCode: authCode,
// }, {
// cssFiles: ['oauth2/authorize'],
// });
// } catch (error) {
// return mustacheService.render('oauth2/authorize', {
// response_type: query.response_type,
// client: client,
// error: error,
// }, {
// cssFiles: ['oauth2/authorize'],
// });
// }
// }
// }

38
packages/server/src/controllers/api/SessionController.test.ts Normal file
View File

@ -0,0 +1,38 @@
import { createUser, checkThrowAsync, beforeAllDb, afterAllDb, beforeEachDb, controllers } from '../../utils/testUtils';
import { ErrorForbidden } from '../../utils/errors';
describe('SessionController', function() {
beforeAll(async () => {
await beforeAllDb('SessionController');
});
afterAll(async () => {
await afterAllDb();
});
beforeEach(async () => {
await beforeEachDb();
});
it('should authenticate a user and give back a session', async function() {
const user = await createUser(1);
const controller = controllers().apiSession();
const session = await controller.authenticate(user.email, '123456');
expect(!!session).toBe(true);
expect(!!session.id).toBe(true);
expect(!!session.user_id).toBe(true);
});
it('should not give a session for invalid login', async function() {
const user = await createUser(1);
const controller = controllers().apiSession();
let error = await checkThrowAsync(async () => controller.authenticate(user.email, 'wrong'));
expect(error instanceof ErrorForbidden).toBe(true);
error = await checkThrowAsync(async () => controller.authenticate('wrong@wrong.com', '123456'));
expect(error instanceof ErrorForbidden).toBe(true);
});
});

19
packages/server/src/controllers/api/SessionController.ts Normal file
View File

@ -0,0 +1,19 @@
import { Session, User } from '../../db';
import { checkPassword } from '../../utils/auth';
import { ErrorForbidden } from '../../utils/errors';
import uuidgen from '../../utils/uuidgen';
import BaseController from '../BaseController';
export default class SessionController extends BaseController {
public async authenticate(email: string, password: string): Promise<Session> {
const userModel = this.models.user();
const user: User = await userModel.loadByEmail(email);
if (!user) throw new ErrorForbidden('Invalid username or password');
if (!checkPassword(password, user.password)) throw new ErrorForbidden('Invalid username or password');
const session: Session = { id: uuidgen(), user_id: user.id };
const sessionModel = this.models.session();
return sessionModel.save(session, { isNew: true });
}
}

192
packages/server/src/controllers/api/UserController.test.ts Normal file
View File

@ -0,0 +1,192 @@
import { models, controllers, createUserAndSession, checkThrowAsync, beforeAllDb, afterAllDb, beforeEachDb } from '../../utils/testUtils';
import { File, User } from '../../db';
import { ErrorForbidden, ErrorUnprocessableEntity } from '../../utils/errors';
describe('UserController', function() {
beforeAll(async () => {
await beforeAllDb('UserController');
});
afterAll(async () => {
await afterAllDb();
});
beforeEach(async () => {
await beforeEachDb();
});
it('should create a new user along with his root file', async function() {
const { session } = await createUserAndSession(1, true);
const controller = controllers().apiUser();
const newUser = await controller.postUser(session.id, { email: 'test@example.com', password: '123456' });
expect(!!newUser).toBe(true);
expect(!!newUser.id).toBe(true);
expect(!!newUser.is_admin).toBe(false);
expect(!!newUser.email).toBe(true);
expect(!newUser.password).toBe(true);
const userModel = models().user({ userId: newUser.id });
const userFromModel: User = await userModel.load(newUser.id);
expect(!!userFromModel.password).toBe(true);
expect(userFromModel.password === '123456').toBe(false); // Password has been hashed
const fileModel = models().file({ userId: newUser.id });
const rootFile: File = await fileModel.userRootFile();
expect(!!rootFile).toBe(true);
expect(!!rootFile.id).toBe(true);
});
it('should not create anything, neither user, root file nor permissions, if user creation fail', async function() {
const { user, session } = await createUserAndSession(1, true);
const controller = controllers().apiUser();
const fileModel = models().file({ userId: user.id });
const permissionModel = models().permission();
const userModel = models().user({ userId: user.id });
await controller.postUser(session.id, { email: 'test@example.com', password: '123456' });
const beforeFileCount = (await fileModel.all()).length;
const beforeUserCount = (await userModel.all()).length;
const beforePermissionCount = (await permissionModel.all()).length;
expect(beforeFileCount).toBe(2);
expect(beforeUserCount).toBe(2);
let hasThrown = false;
try {
await controller.postUser(session.id, { email: 'test@example.com', password: '123456' });
} catch (error) {
hasThrown = true;
}
expect(hasThrown).toBe(true);
const afterFileCount = (await fileModel.all()).length;
const afterUserCount = (await userModel.all()).length;
const afterPermissionCount = (await permissionModel.all()).length;
expect(beforeFileCount).toBe(afterFileCount);
expect(beforeUserCount).toBe(afterUserCount);
expect(beforePermissionCount).toBe(afterPermissionCount);
});
it('should change user properties', async function() {
const { user, session } = await createUserAndSession(1, true);
const controller = controllers().apiUser();
const userModel = models().user({ userId: user.id });
await controller.patchUser(session.id, { id: user.id, email: 'test2@example.com' });
let modUser: User = await userModel.load(user.id);
expect(modUser.email).toBe('test2@example.com');
const previousPassword = modUser.password;
await controller.patchUser(session.id, { id: user.id, password: 'abcdefgh' });
modUser = await userModel.load(user.id);
expect(!!modUser.password).toBe(true);
expect(modUser.password === previousPassword).toBe(false);
});
it('should get a user', async function() {
const { user, session } = await createUserAndSession();
const controller = controllers().apiUser();
const gotUser = await controller.getUser(session.id, user.id);
expect(gotUser.id).toBe(user.id);
expect(gotUser.email).toBe(user.email);
});
it('should validate user objects', async function() {
const { user: admin, session: adminSession } = await createUserAndSession(1, true);
const { user: user1, session: userSession1 } = await createUserAndSession(2, false);
const { user: user2 } = await createUserAndSession(3, false);
let error = null;
const controller = controllers().apiUser();
// Non-admin user can't create a user
error = await checkThrowAsync(async () => await controller.postUser(userSession1.id, { email: 'newone@example.com', password: '1234546' }));
expect(error instanceof ErrorForbidden).toBe(true);
// Email must be set
error = await checkThrowAsync(async () => await controller.postUser(adminSession.id, { email: '', password: '1234546' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// Password must be set
error = await checkThrowAsync(async () => await controller.postUser(adminSession.id, { email: 'newone@example.com', password: '' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// ID must be set when updating a user
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { email: 'newone@example.com' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// non-admin user cannot modify another user
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user2.id, email: 'newone@example.com' }));
expect(error instanceof ErrorForbidden).toBe(true);
// email must be set
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user1.id, email: '' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// password must be set
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user1.id, password: '' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// non-admin user cannot make a user an admin
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user1.id, is_admin: 1 }));
expect(error instanceof ErrorForbidden).toBe(true);
// non-admin user cannot remove admin bit from themselves
error = await checkThrowAsync(async () => await controller.patchUser(adminSession.id, { id: admin.id, is_admin: 0 }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// there is already a user with this email
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user1.id, email: user2.email }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
// check that the email is valid
error = await checkThrowAsync(async () => await controller.patchUser(userSession1.id, { id: user1.id, email: 'ohno' }));
expect(error instanceof ErrorUnprocessableEntity).toBe(true);
});
it('should delete a user', async function() {
const { user: admin, session: adminSession } = await createUserAndSession(1, true);
const { user: user1, session: session1 } = await createUserAndSession(2, false);
const { user: user2, session: session2 } = await createUserAndSession(3, false);
const controller = controllers().apiUser();
const userModel = models().user({ userId: admin.id });
const allUsers: File[] = await userModel.all();
const beforeCount: number = allUsers.length;
// Can't delete someone else user
const error = await checkThrowAsync(async () => await controller.deleteUser(session1.id, user2.id));
expect(error instanceof ErrorForbidden).toBe(true);
expect((await userModel.all()).length).toBe(beforeCount);
// Admin can delete any user
await controller.deleteUser(adminSession.id, user1.id);
expect((await userModel.all()).length).toBe(beforeCount - 1);
const allFiles = await models().file().all() as File[];
expect(allFiles.length).toBe(2);
expect(!!allFiles.find(f => f.owner_id === admin.id)).toBe(true);
expect(!!allFiles.find(f => f.owner_id === user2.id)).toBe(true);
// Can delete own user
const fileModel = models().file({ userId: user2.id });
expect(!!(await fileModel.userRootFile())).toBe(true);
await controller.deleteUser(session2.id, user2.id);
expect((await userModel.all()).length).toBe(beforeCount - 2);
expect(!!(await fileModel.userRootFile())).toBe(false);
});
});

33
packages/server/src/controllers/api/UserController.ts Normal file
View File

@ -0,0 +1,33 @@
import { User } from '../../db';
import BaseController from '../BaseController';
export default class UserController extends BaseController {
public async postUser(sessionId: string, user: User): Promise<User> {
const owner = await this.initSession(sessionId, true);
const userModel = this.models.user({ userId: owner.id });
let newUser = userModel.fromApiInput(user);
newUser = await userModel.save(newUser);
return userModel.toApiOutput(newUser);
}
public async getUser(sessionId: string, userId: string): Promise<User> {
const owner = await this.initSession(sessionId);
const userModel = this.models.user({ userId: owner.id });
return userModel.toApiOutput(await userModel.load(userId));
}
public async patchUser(sessionId: string, user: User): Promise<void> {
const owner = await this.initSession(sessionId);
const userModel = this.models.user({ userId: owner.id });
const newUser = userModel.fromApiInput(user);
await userModel.save(newUser, { isNew: false });
}
public async deleteUser(sessionId: string, userId: string): Promise<void> {
const user = await this.initSession(sessionId);
const userModel = this.models.user({ userId: user.id });
await userModel.delete(userId);
}
}

55
packages/server/src/controllers/factory.ts Normal file
View File

@ -0,0 +1,55 @@
import { Models } from '../models/factory';
import FileController from './api/FileController';
// import OAuthController from './api/OAuthController';
import SessionController from './api/SessionController';
import UserController from './api/UserController';
import IndexLoginController from './index/LoginController';
import IndexHomeController from './index/HomeController';
import IndexProfileController from './index/ProfileController';
import IndexUserController from './index/UserController';
export class Controllers {
private models_: Models;
public constructor(models: Models) {
this.models_ = models;
}
public apiFile() {
return new FileController(this.models_);
}
// public oauth() {
// return new OAuthController(this.models_);
// }
public apiSession() {
return new SessionController(this.models_);
}
public apiUser() {
return new UserController(this.models_);
}
public indexLogin() {
return new IndexLoginController(this.models_);
}
public indexHome() {
return new IndexHomeController(this.models_);
}
public indexProfile() {
return new IndexProfileController(this.models_, this.apiUser());
}
public indexUser() {
return new IndexUserController(this.models_);
}
}
export default function(models: Models) {
return new Controllers(models);
}

12
packages/server/src/controllers/index/HomeController.ts Normal file
View File

@ -0,0 +1,12 @@
import BaseController from '../BaseController';
import { View } from '../../services/MustacheService';
import defaultView from '../../utils/defaultView';
export default class HomeController extends BaseController {
public async getIndex(sessionId: string): Promise<View> {
const owner = await this.initSession(sessionId);
return defaultView('home', owner);
}
}

14
packages/server/src/controllers/index/LoginController.ts Normal file
View File

@ -0,0 +1,14 @@
import BaseController from '../BaseController';
import { View } from '../../services/MustacheService';
import defaultView from '../../utils/defaultView';
export default class LoginController extends BaseController {
public async getIndex(error: any = null): Promise<View> {
const view = defaultView('login');
view.content.error = error;
view.partials = ['errorBanner'];
return view;
}
}

31
packages/server/src/controllers/index/ProfileController.ts Normal file
View File

@ -0,0 +1,31 @@
import BaseController from '../BaseController';
import { View } from '../../services/MustacheService';
import defaultView from '../../utils/defaultView';
import { User } from '../../db';
import { Models } from '../../models/factory';
import UserController from '../api/UserController';
export default class ProfileController extends BaseController {
private userController_: UserController;
public constructor(models: Models, userController: UserController) {
super(models);
this.userController_ = userController;
}
public async getIndex(sessionId: string, user: User = null, error: any = null): Promise<View> {
const owner = await this.initSession(sessionId);
const view: View = defaultView('profile', owner);
view.content.user = user ? user : owner;
view.content.error = error;
view.partials.push('errorBanner');
return view;
}
public async patchIndex(sessionId: string, user: User): Promise<void> {
await this.userController_.patchUser(sessionId, user);
}
}

42
packages/server/src/controllers/index/UserController.ts Normal file
View File

@ -0,0 +1,42 @@
import BaseController from '../BaseController';
import { View } from '../../services/MustacheService';
import defaultView from '../../utils/defaultView';
import { User } from '../../db';
import { baseUrl } from '../../config';
export default class UserController extends BaseController {
public async getIndex(sessionId: string): Promise<View> {
const owner = await this.initSession(sessionId);
const userModel = this.models.user({ userId: owner.id });
const users = await userModel.all();
const view: View = defaultView('users', owner);
view.content.users = users;
return view;
}
public async getOne(sessionId: string, isNew: boolean, userIdOrString: string | User = null, error: any = null): Promise<View> {
const owner = await this.initSession(sessionId);
const userModel = this.models.user({ userId: owner.id });
let user: User = {};
if (typeof userIdOrString === 'string') {
user = await userModel.load(userIdOrString as string);
} else {
user = userIdOrString as User;
}
const view: View = defaultView('user', owner);
view.content.user = user;
view.content.isNew = isNew;
view.content.buttonTitle = isNew ? 'Create user' : 'Update profile';
view.content.error = error;
view.content.postUrl = `${baseUrl()}/users${isNew ? '/new' : `/${user.id}`}`;
view.partials.push('errorBanner');
return view;
}
}

324
packages/server/src/db.ts Normal file
View File

@ -0,0 +1,324 @@
import * as Knex from 'knex';
import { DatabaseConfig } from './utils/types';
import * as pathUtils from 'path';
import time from '@joplin/lib/time';
import Logger from '@joplin/lib/Logger';
// Make sure bigInteger values are numbers and not strings
//
// https://github.com/brianc/node-pg-types
//
// In our case, all bigInteger are timestamps, which JavaScript can handle
// fine as numbers.
require('pg').types.setTypeParser(20, function(val: any) {
return parseInt(val, 10);
});
const logger = Logger.create('db');
const migrationDir = `${__dirname}/migrations`;
const sqliteDbDir = pathUtils.dirname(__dirname);
export type DbConnection = Knex;
export interface DbConfigConnection {
host?: string;
port?: number;
user?: string;
database?: string;
filename?: string;
password?: string;
}
export interface KnexDatabaseConfig {
client: string;
connection: DbConfigConnection;
useNullAsDefault?: boolean;
asyncStackTraces?: boolean;
}
export interface ConnectionCheckResult {
isCreated: boolean;
error: any;
latestMigration: any;
connection: DbConnection;
}
export function sqliteFilePath(dbConfig: DatabaseConfig): string {
return `${sqliteDbDir}/db-${dbConfig.name}.sqlite`;
}
export function makeKnexConfig(dbConfig: DatabaseConfig): KnexDatabaseConfig {
const connection: DbConfigConnection = {};
if (dbConfig.client === 'sqlite3') {
connection.filename = sqliteFilePath(dbConfig);
} else {
connection.database = dbConfig.name;
connection.host = dbConfig.host;
connection.port = dbConfig.port;
connection.user = dbConfig.user;
connection.password = dbConfig.password;
}
return {
client: dbConfig.client,
useNullAsDefault: dbConfig.client === 'sqlite3',
asyncStackTraces: dbConfig.asyncStackTraces,
connection,
};
}
export async function waitForConnection(dbConfig: DatabaseConfig): Promise<ConnectionCheckResult> {
const timeout = 30000;
const startTime = Date.now();
let lastError = { message: '' };
while (true) {
try {
const connection = await connectDb(dbConfig);
const check = await connectionCheck(connection);
if (check.error) throw check.error;
return check;
} catch (error) {
logger.info('Could not connect. Will try again.', error.message);
lastError = error;
}
if (Date.now() - startTime > timeout) {
logger.error('Timeout trying to connect to database:', lastError);
throw new Error(`Timeout trying to connect to database. Last error was: ${lastError.message}`);
}
await time.msleep(1000);
}
}
export async function connectDb(dbConfig: DatabaseConfig): Promise<DbConnection> {
return require('knex')(makeKnexConfig(dbConfig));
}
export async function disconnectDb(db: DbConnection) {
await db.destroy();
}
export async function migrateDb(db: DbConnection) {
await db.migrate.latest({
directory: migrationDir,
// Disable transactions because the models might open one too
disableTransactions: true,
});
}
function allTableNames(): string[] {
const tableNames = Object.keys(databaseSchema);
tableNames.push('knex_migrations');
tableNames.push('knex_migrations_lock');
return tableNames;
}
export async function dropTables(db: DbConnection): Promise<void> {
for (const tableName of allTableNames()) {
try {
await db.schema.dropTable(tableName);
} catch (error) {
if (isNoSuchTableError(error)) continue;
throw error;
}
}
}
function isNoSuchTableError(error: any): boolean {
if (error) {
// Postgres error: 42P01: undefined_table
if (error.code === '42P01') return true;
// Sqlite3 error
if (error.message && error.message.includes('no such table: knex_migrations')) return true;
}
return false;
}
export async function latestMigration(db: DbConnection): Promise<any> {
try {
const result = await db('knex_migrations').select('name').orderBy('id', 'asc').first();
return result;
} catch (error) {
// If the database has never been initialized, we return null, so
// for this we need to check the error code, which will be
// different depending on the DBMS.
if (isNoSuchTableError(error)) return null;
throw error;
}
}
export async function connectionCheck(db: DbConnection): Promise<ConnectionCheckResult> {
try {
const result = await latestMigration(db);
return {
latestMigration: result,
isCreated: !!result,
error: null,
connection: db,
};
} catch (error) {
return {
latestMigration: null,
isCreated: false,
error: error,
connection: null,
};
}
}
export type Uuid = string;
export enum ItemAddressingType {
Id = 1,
Path,
}
export enum ItemType {
File = 1,
User,
}
export enum ChangeType {
Create = 1,
Update = 2,
Delete = 3,
}
export interface WithDates {
updated_time?: number;
created_time?: number;
}
export interface WithUuid {
id?: string;
}
interface DatabaseTableColumn {
type: string;
}
interface DatabaseTable {
[key: string]: DatabaseTableColumn;
}
interface DatabaseTables {
[key: string]: DatabaseTable;
}
// AUTO-GENERATED-TYPES
// Auto-generated using `npm run generate-types`
export interface User extends WithDates, WithUuid {
email?: string;
password?: string;
full_name?: string;
is_admin?: number;
}
export interface Session extends WithDates, WithUuid {
user_id?: Uuid;
auth_code?: string;
}
export interface Permission extends WithDates, WithUuid {
user_id?: Uuid;
item_type?: ItemType;
item_id?: Uuid;
can_read?: number;
can_write?: number;
}
export interface File extends WithDates, WithUuid {
owner_id?: Uuid;
name?: string;
content?: Buffer;
mime_type?: string;
size?: number;
is_directory?: number;
is_root?: number;
parent_id?: Uuid;
}
export interface Change extends WithDates, WithUuid {
counter?: number;
owner_id?: Uuid;
item_type?: ItemType;
parent_id?: Uuid;
item_id?: Uuid;
item_name?: string;
type?: ChangeType;
}
export interface ApiClient extends WithDates, WithUuid {
name?: string;
secret?: string;
}
export const databaseSchema: DatabaseTables = {
users: {
id: { type: 'string' },
email: { type: 'string' },
password: { type: 'string' },
full_name: { type: 'string' },
is_admin: { type: 'number' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
sessions: {
id: { type: 'string' },
user_id: { type: 'string' },
auth_code: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
permissions: {
id: { type: 'string' },
user_id: { type: 'string' },
item_type: { type: 'number' },
item_id: { type: 'string' },
can_read: { type: 'number' },
can_write: { type: 'number' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
files: {
id: { type: 'string' },
owner_id: { type: 'string' },
name: { type: 'string' },
content: { type: 'any' },
mime_type: { type: 'string' },
size: { type: 'number' },
is_directory: { type: 'number' },
is_root: { type: 'number' },
parent_id: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
changes: {
counter: { type: 'number' },
id: { type: 'string' },
owner_id: { type: 'string' },
item_type: { type: 'number' },
parent_id: { type: 'string' },
item_id: { type: 'string' },
item_name: { type: 'string' },
type: { type: 'number' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
api_clients: {
id: { type: 'string' },
name: { type: 'string' },
secret: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
};
// AUTO-GENERATED-TYPES

135
packages/server/src/migrations/20190913171451_create.ts Normal file
View File

@ -0,0 +1,135 @@
import * as Knex from 'knex';
import { DbConnection } from '../db';
import { hashPassword } from '../utils/auth';
import uuidgen from '../utils/uuidgen';
export async function up(db: DbConnection): Promise<any> {
await db.schema.createTable('users', function(table: Knex.CreateTableBuilder) {
table.string('id', 32).unique().primary().notNullable();
table.text('email', 'mediumtext').unique().notNullable();
table.text('password', 'mediumtext').notNullable();
table.text('full_name', 'mediumtext').defaultTo('').notNullable();
table.integer('is_admin').defaultTo(0).notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('users', function(table: Knex.CreateTableBuilder) {
table.index(['email']);
});
await db.schema.createTable('sessions', function(table: Knex.CreateTableBuilder) {
table.string('id', 32).unique().primary().notNullable();
table.string('user_id', 32).notNullable();
table.string('auth_code', 32).defaultTo('').notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.createTable('permissions', function(table: Knex.CreateTableBuilder) {
table.string('id', 32).unique().primary().notNullable();
table.string('user_id', 32).notNullable();
table.integer('item_type').notNullable();
table.string('item_id', 32).notNullable();
table.integer('can_read').defaultTo(0).notNullable();
table.integer('can_write').defaultTo(0).notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('permissions', function(table: Knex.CreateTableBuilder) {
table.unique(['user_id', 'item_type', 'item_id']);
table.index(['item_id']);
table.index(['item_type', 'item_id']);
});
await db.schema.createTable('files', function(table: Knex.CreateTableBuilder) {
table.string('id', 32).unique().primary().notNullable();
table.string('owner_id', 32).notNullable();
table.text('name').notNullable();
table.binary('content').defaultTo('').notNullable();
table.string('mime_type', 128).defaultTo('application/octet-stream').notNullable();
table.integer('size').defaultTo(0).notNullable();
table.integer('is_directory').defaultTo(0).notNullable();
table.integer('is_root').defaultTo(0).notNullable();
table.string('parent_id', 32).defaultTo('').notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('files', function(table: Knex.CreateTableBuilder) {
table.unique(['parent_id', 'name']);
table.index(['parent_id']);
});
await db.schema.createTable('changes', function(table: Knex.CreateTableBuilder) {
// Note that in this table, the counter is the primary key, since
// we want it to be automatically incremented. There's also a
// column ID to publicly identify a change.
table.increments('counter').unique().primary().notNullable();
table.string('id', 32).unique().notNullable();
table.string('owner_id', 32).notNullable();
table.integer('item_type').notNullable();
table.string('parent_id', 32).defaultTo('').notNullable();
table.string('item_id', 32).notNullable();
table.text('item_name').defaultTo('').notNullable();
table.integer('type').notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('changes', function(table: Knex.CreateTableBuilder) {
table.index(['id']);
table.index(['parent_id']);
});
await db.schema.createTable('api_clients', function(table: Knex.CreateTableBuilder) {
table.string('id', 32).unique().primary().notNullable();
table.string('name', 32).notNullable();
table.string('secret', 32).notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
const adminId = uuidgen();
const adminRootFileId = uuidgen();
const now = Date.now();
await db('users').insert({
id: adminId,
email: 'admin@localhost',
password: hashPassword('admin'),
full_name: 'Admin',
is_admin: 1,
updated_time: now,
created_time: now,
});
await db('files').insert({
id: adminRootFileId,
owner_id: adminId,
name: adminRootFileId,
size: 0,
is_directory: 1,
is_root: 1,
updated_time: now,
created_time: now,
});
await db('api_clients').insert({
id: uuidgen(),
name: 'Joplin',
secret: 'sdrNUPtKNdY5Z5tF4bthqu',
updated_time: now,
created_time: now,
});
}
export async function down(db: DbConnection): Promise<any> {
await db.schema.dropTable('users');
await db.schema.dropTable('sessions');
await db.schema.dropTable('permissions');
await db.schema.dropTable('files');
await db.schema.dropTable('api_clients');
await db.schema.dropTable('changes');
}

9
packages/server/src/models/ApiClientModel.ts Normal file
View File

@ -0,0 +1,9 @@
import BaseModel from './BaseModel';
export default class ApiClientModel extends BaseModel {
protected get tableName(): string {
return 'api_clients';
}
}

239
packages/server/src/models/BaseModel.ts Normal file
View File

@ -0,0 +1,239 @@
import { WithDates, WithUuid, File, User, Session, Permission, databaseSchema, ApiClient, DbConnection, Change, ItemType, ChangeType } from '../db';
import TransactionHandler from '../utils/TransactionHandler';
import uuidgen from '../utils/uuidgen';
import { ErrorUnprocessableEntity, ErrorBadRequest } from '../utils/errors';
import { Models } from './factory';
export type AnyItemType = File | User | Session | Permission | ApiClient | Change;
export type AnyItemTypes = File[] | User[] | Session[] | Permission[] | ApiClient[] | Change[];
export interface ModelOptions {
userId?: string;
}
export interface SaveOptions {
isNew?: boolean;
skipValidation?: boolean;
validationRules?: any;
trackChanges?: boolean;
}
export interface DeleteOptions {
validationRules?: any;
}
export interface ValidateOptions {
isNew?: boolean;
rules?: any;
}
export default abstract class BaseModel {
private options_: ModelOptions = null;
private defaultFields_: string[] = [];
private db_: DbConnection;
private transactionHandler_: TransactionHandler;
private modelFactory_: Function;
public constructor(db: DbConnection, modelFactory: Function, options: ModelOptions = null) {
this.db_ = db;
this.modelFactory_ = modelFactory;
this.options_ = Object.assign({}, options);
this.transactionHandler_ = new TransactionHandler(db);
if ('userId' in this.options && !this.options.userId) throw new Error('If userId is set, it cannot be null');
}
// When a model create an instance of another model, the active
// connection is passed to it. That connection can be the regular db
// connection, or the active transaction.
protected models(db: DbConnection = null): Models {
return this.modelFactory_(db || this.db);
}
protected get options(): ModelOptions {
return this.options_;
}
protected get userId(): string {
return this.options.userId;
}
protected get db(): DbConnection {
if (this.transactionHandler_.activeTransaction) return this.transactionHandler_.activeTransaction;
return this.db_;
}
protected get defaultFields(): string[] {
if (!this.defaultFields_.length) {
this.defaultFields_ = Object.keys(databaseSchema[this.tableName]);
}
return this.defaultFields_.slice();
}
protected get tableName(): string {
throw new Error('Not implemented');
}
protected get itemType(): ItemType {
throw new Error('Not implemented');
}
protected get trackChanges(): boolean {
return false;
}
protected hasUuid(): boolean {
return true;
}
protected hasDateProperties(): boolean {
return true;
}
protected get hasParentId(): boolean {
return false;
}
protected async withTransaction(fn: Function): Promise<void> {
const txIndex = await this.transactionHandler_.start();
try {
await fn();
} catch (error) {
await this.transactionHandler_.rollback(txIndex);
throw error;
}
await this.transactionHandler_.commit(txIndex);
}
public async all(): Promise<AnyItemTypes> {
return this.db(this.tableName).select(...this.defaultFields);
}
public fromApiInput(object: AnyItemType): AnyItemType {
return object;
}
public toApiOutput(object: any): any {
return { ...object };
}
protected async validate(object: AnyItemType, options: ValidateOptions = {}): Promise<AnyItemType> {
if (!options.isNew && !(object as WithUuid).id) throw new ErrorUnprocessableEntity('id is missing');
return object;
}
protected async isNew(object: AnyItemType, options: SaveOptions): Promise<boolean> {
if (options.isNew === false) return false;
if (options.isNew === true) return true;
return !(object as WithUuid).id;
}
private async handleChangeTracking(options: SaveOptions, item: AnyItemType, changeType: ChangeType): Promise<void> {
const trackChanges = this.trackChanges && options.trackChanges !== false;
if (!trackChanges) return;
let parentId = null;
if (this.hasParentId) {
if (!('parent_id' in item)) {
const temp: any = await this.db(this.tableName).select(['parent_id']).where('id', '=', item.id).first();
parentId = temp.parent_id;
} else {
parentId = item.parent_id;
}
}
// Sanity check - shouldn't happen
// Parent ID can be an empty string for root folders, but it shouldn't be null or undefined
if (this.hasParentId && !parentId && parentId !== '') throw new Error(`Could not find parent ID for item: ${item.id}`);
const changeModel = this.models().change({ userId: this.userId });
await changeModel.add(this.itemType, parentId, (item as WithUuid).id, (item as any).name || '', changeType);
}
public async save(object: AnyItemType, options: SaveOptions = {}): Promise<AnyItemType> {
if (!object) throw new Error('Object cannot be empty');
const toSave = Object.assign({}, object);
const isNew = await this.isNew(object, options);
if (isNew && !(toSave as WithUuid).id) {
(toSave as WithUuid).id = uuidgen();
}
if (this.hasDateProperties()) {
const timestamp = Date.now();
if (isNew) {
(toSave as WithDates).created_time = timestamp;
}
(toSave as WithDates).updated_time = timestamp;
}
if (options.skipValidation !== true) object = await this.validate(object, { isNew: isNew, rules: options.validationRules ? options.validationRules : {} });
await this.withTransaction(async () => {
if (isNew) {
await this.db(this.tableName).insert(toSave);
await this.handleChangeTracking(options, toSave, ChangeType.Create);
} else {
const objectId: string = (toSave as WithUuid).id;
if (!objectId) throw new Error('Missing "id" property');
delete (toSave as WithUuid).id;
const updatedCount: number = await this.db(this.tableName).update(toSave).where({ id: objectId });
toSave.id = objectId;
await this.handleChangeTracking(options, toSave, ChangeType.Update);
// Sanity check:
if (updatedCount !== 1) throw new ErrorBadRequest(`one row should have been updated, but ${updatedCount} row(s) were updated`);
}
});
return toSave;
}
public async loadByIds(ids: string[]): Promise<AnyItemType[]> {
if (!ids.length) return [];
return this.db(this.tableName).select(this.defaultFields).whereIn('id', ids);
}
public async load(id: string): Promise<AnyItemType> {
if (!id) throw new Error('id cannot be empty');
return this.db(this.tableName).select(this.defaultFields).where({ id: id }).first();
}
public async delete(id: string | string[]): Promise<void> {
if (!id) throw new Error('id cannot be empty');
const ids = typeof id === 'string' ? [id] : id;
if (!ids.length) throw new Error('no id provided');
const query = this.db(this.tableName).where({ id: ids[0] });
for (let i = 1; i < ids.length; i++) {
await query.orWhere({ id: ids[i] });
}
const trackChanges = this.trackChanges;
let itemsWithParentIds: AnyItemType[] = null;
if (trackChanges) {
itemsWithParentIds = await this.db(this.tableName).select(['id', 'parent_id', 'name']).whereIn('id', ids);
}
await this.withTransaction(async () => {
const deletedCount = await query.del();
if (deletedCount !== ids.length) throw new Error(`${ids.length} row(s) should have been deleted by ${deletedCount} row(s) were deleted`);
if (trackChanges) {
for (const item of itemsWithParentIds) await this.handleChangeTracking({}, item, ChangeType.Delete);
}
});
}
}

119
packages/server/src/models/ChangeModel.test.ts Normal file
View File

@ -0,0 +1,119 @@
import { createUserAndSession, beforeAllDb, afterAllDb, beforeEachDb, models, expectThrow } from '../utils/testUtils';
import { ChangeType, File } from '../db';
import FileModel from './FileModel';
import { msleep } from '../utils/time';
import { ChangePagination } from './ChangeModel';
async function makeTestFile(fileModel: FileModel): Promise<File> {
return fileModel.save({
name: 'test',
parent_id: await fileModel.userRootFileId(),
});
}
describe('ChangeModel', function() {
beforeAll(async () => {
await beforeAllDb('ChangeModel');
});
afterAll(async () => {
await afterAllDb();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should track changes - create', async function() {
const { user } = await createUserAndSession(1, true);
const fileModel = models().file({ userId: user.id });
const changeModel = models().change({ userId: user.id });
const file1 = await makeTestFile(fileModel);
const dirId = await fileModel.userRootFileId();
{
const changes = (await changeModel.byDirectoryId(dirId, { limit: 20 })).items;
expect(changes.length).toBe(1);
expect(changes[0].item.id).toBe(file1.id);
expect(changes[0].type).toBe(ChangeType.Create);
}
});
test('should track changes - create, then update', async function() {
const { user } = await createUserAndSession(1, true);
const fileModel = models().file({ userId: user.id });
const changeModel = models().change({ userId: user.id });
let i = 1;
await msleep(1); const file1 = await makeTestFile(fileModel); // CREATE 1
await msleep(1); await fileModel.save({ id: file1.id, name: `test_mod${i++}` }); // UPDATE 1
await msleep(1); await fileModel.save({ id: file1.id, name: `test_mod${i++}` }); // UPDATE 1
await msleep(1); const file2 = await makeTestFile(fileModel); // CREATE 2
await msleep(1); await fileModel.save({ id: file2.id, name: `test_mod${i++}` }); // UPDATE 2
await msleep(1); await fileModel.delete(file1.id); // DELETE 1
await msleep(1); await fileModel.save({ id: file2.id, name: `test_mod${i++}` }); // UPDATE 2
await msleep(1); const file3 = await makeTestFile(fileModel); // CREATE 3
const dirId = await fileModel.userRootFileId();
{
const changes = (await changeModel.byDirectoryId(dirId, { limit: 20 })).items;
expect(changes.length).toBe(2);
expect(changes[0].item.id).toBe(file2.id);
expect(changes[0].type).toBe(ChangeType.Create);
expect(changes[1].item.id).toBe(file3.id);
expect(changes[1].type).toBe(ChangeType.Create);
}
{
const pagination: ChangePagination = { limit: 5 };
// In this page, the "create" change for file1 will not appear
// because this file has been deleted. The "delete" change will
// however appear in the second page.
const page1 = (await changeModel.byDirectoryId(dirId, pagination));
let changes = page1.items;
expect(changes.length).toBe(1);
expect(page1.has_more).toBe(true);
expect(changes[0].item.id).toBe(file2.id);
expect(changes[0].type).toBe(ChangeType.Create);
const page2 = (await changeModel.byDirectoryId(dirId, { ...pagination, cursor: page1.cursor }));
changes = page2.items;
expect(changes.length).toBe(3);
expect(page2.has_more).toBe(false);
expect(changes[0].item.id).toBe(file1.id);
expect(changes[0].type).toBe(ChangeType.Delete);
expect(changes[1].item.id).toBe(file2.id);
expect(changes[1].type).toBe(ChangeType.Update);
expect(changes[2].item.id).toBe(file3.id);
expect(changes[2].type).toBe(ChangeType.Create);
}
});
test('should throw an error if cursor is invalid', async function() {
const { user } = await createUserAndSession(1, true);
const fileModel = models().file({ userId: user.id });
const changeModel = models().change({ userId: user.id });
const dirId = await fileModel.userRootFileId();
let i = 1;
await msleep(1); const file1 = await makeTestFile(fileModel); // CREATE 1
await msleep(1); await fileModel.save({ id: file1.id, name: `test_mod${i++}` }); // UPDATE 1
await expectThrow(async () => changeModel.byDirectoryId(dirId, { limit: 1, cursor: 'invalid' }), 'resyncRequired');
});
test('should throw an error if trying to do get changes for a file', async function() {
const { user } = await createUserAndSession(1, true);
const fileModel = models().file({ userId: user.id });
const changeModel = models().change({ userId: user.id });
const file1 = await makeTestFile(fileModel);
await expectThrow(async () => changeModel.byDirectoryId(file1.id));
});
});

182
packages/server/src/models/ChangeModel.ts Normal file
View File

@ -0,0 +1,182 @@
import { Change, ChangeType, File, ItemType, Uuid } from '../db';
import { ErrorResyncRequired, ErrorUnprocessableEntity } from '../utils/errors';
import BaseModel from './BaseModel';
import { PaginatedResults } from './utils/pagination';
export interface ChangeWithItem {
item: File;
type: ChangeType;
}
export interface PaginatedChanges extends PaginatedResults {
items: ChangeWithItem[];
}
export interface ChangePagination {
limit?: number;
cursor?: string;
}
export function defaultChangePagination(): ChangePagination {
return {
limit: 100,
cursor: '',
};
}
export default class ChangeModel extends BaseModel {
public get tableName(): string {
return 'changes';
}
protected hasUuid(): boolean {
return false;
}
public async add(itemType: ItemType, parentId: Uuid, itemId: Uuid, itemName: string, changeType: ChangeType): Promise<Change> {
const change: Change = {
item_type: itemType,
parent_id: parentId || '',
item_id: itemId,
item_name: itemName,
type: changeType,
owner_id: this.userId,
};
return this.save(change);
}
// Note: doesn't currently support checking for changes recursively but this
// is not needed for Joplin synchronisation.
public async byDirectoryId(dirId: string, pagination: ChangePagination = null): Promise<PaginatedChanges> {
pagination = {
...defaultChangePagination(),
...pagination,
};
let changeAtCursor: Change = null;
if (pagination.cursor) {
changeAtCursor = await this.load(pagination.cursor);
if (!changeAtCursor) throw new ErrorResyncRequired();
}
// Load the directory object to check that it exists and that we have
// the right permissions (loading will check permissions)
const fileModel = this.models().file({ userId: this.userId });
const directory = await fileModel.load(dirId);
if (!directory.is_directory) throw new ErrorUnprocessableEntity(`Item with id "${dirId}" is not a directory.`);
// Rather than query the changes, then use JS to compress them, it might
// be possible to do both in one query.
// https://stackoverflow.com/questions/65348794
const query = this.db(this.tableName)
.select([
'counter',
'id',
'item_id',
'item_name',
'type',
])
.where('parent_id', dirId)
.orderBy('counter', 'asc')
.limit(pagination.limit);
if (changeAtCursor) {
void query.where('counter', '>', changeAtCursor.counter);
}
const changes: Change[] = await query;
const compressedChanges = this.compressChanges(changes);
const changeWithItems = await this.loadChangeItems(compressedChanges);
return {
items: changeWithItems,
// If we have changes, we return the ID of the latest changes from which delta sync can resume.
// If there's no change, we return the previous cursor.
cursor: changes.length ? changes[changes.length - 1].id : pagination.cursor,
has_more: changes.length >= pagination.limit,
};
}
private async loadChangeItems(changes: Change[]): Promise<ChangeWithItem[]> {
const itemIds = changes.map(c => c.item_id);
const fileModel = this.models().file({ userId: this.userId });
const items: File[] = await fileModel.loadByIds(itemIds);
const output: ChangeWithItem[] = [];
for (const change of changes) {
let item = items.find(f => f.id === change.item_id);
// If the item associated with this change has been deleted, we have
// two cases:
// - If it's a "delete" change, add it to the list.
// - If it's anything else, skip it. The "delete" change will be
// sent on one of the next pages.
if (!item) {
if (change.type === ChangeType.Delete) {
item = {
id: change.item_id,
name: change.item_name,
};
} else {
continue;
}
}
output.push({
type: change.type,
item: item,
});
}
return output;
}
private compressChanges(changes: Change[]): Change[] {
const itemChanges: Record<Uuid, Change> = {};
for (const change of changes) {
const previous = itemChanges[change.item_id];
if (previous) {
// create - update => create
// create - delete => NOOP
// update - update => update
// update - delete => delete
if (previous.type === ChangeType.Create && change.type === ChangeType.Update) {
continue;
}
if (previous.type === ChangeType.Create && change.type === ChangeType.Delete) {
delete itemChanges[change.item_id];
}
if (previous.type === ChangeType.Update && change.type === ChangeType.Update) {
itemChanges[change.item_id] = change;
}
if (previous.type === ChangeType.Update && change.type === ChangeType.Delete) {
itemChanges[change.item_id] = change;
}
} else {
itemChanges[change.item_id] = change;
}
}
const output = [];
for (const itemId in itemChanges) {
output.push(itemChanges[itemId]);
}
output.sort((a: Change, b: Change) => a.counter < b.counter ? -1 : +1);
return output;
}
}

345
packages/server/src/models/FileModel.ts Normal file
View File

@ -0,0 +1,345 @@
import BaseModel, { ValidateOptions, SaveOptions, DeleteOptions } from './BaseModel';
import { File, ItemType, databaseSchema } from '../db';
import { ErrorForbidden, ErrorUnprocessableEntity, ErrorNotFound, ErrorBadRequest, ErrorConflict } from '../utils/errors';
import uuidgen from '../utils/uuidgen';
import { splitItemPath, filePathInfo } from '../utils/routeUtils';
import { paginateDbQuery, PaginatedResults, Pagination } from './utils/pagination';
const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
const nodeEnv = process.env.NODE_ENV || 'development';
export interface PaginatedFiles extends PaginatedResults {
items: File[];
}
export interface EntityFromItemIdOptions {
mustExist?: boolean;
}
export default class FileModel extends BaseModel {
private readonly reservedCharacters = ['/', '\\', '*', '<', '>', '?', ':', '|', '#', '%'];
protected get tableName(): string {
return 'files';
}
protected get trackChanges(): boolean {
return true;
}
protected get itemType(): ItemType {
return ItemType.File;
}
protected get hasParentId(): boolean {
return true;
}
public async userRootFile(): Promise<File> {
const file: File = await this.db<File>(this.tableName).select(...this.defaultFields).from(this.tableName).where({
'owner_id': this.userId,
'is_root': 1,
}).first();
if (file) await this.checkCanReadPermissions(file);
return file;
}
public async userRootFileId(): Promise<string> {
const r = await this.userRootFile();
return r ? r.id : '';
}
private async specialDirId(dirname: string): Promise<string> {
if (dirname === 'root') return this.userRootFileId();
return null; // Not a special dir
}
public async entityFromItemId(idOrPath: string, options: EntityFromItemIdOptions = {}): Promise<File> {
options = { mustExist: true, ...options };
const specialDirId = await this.specialDirId(idOrPath);
if (specialDirId) {
return { id: specialDirId };
} else if (idOrPath.indexOf(':') < 0) {
return { id: idOrPath };
} else {
// When this input is a path, there can be two cases:
// - A path to an existing file - in which case we return the file
// - A path to a file that needs to be created - in which case we
// return a file with all the relevant properties populated. This
// file might then be created by the caller.
// The caller can check file.id to see if it's a new or existing file.
// In both cases the directories before the filename must exist.
const fileInfo = filePathInfo(idOrPath);
const parentFiles = await this.pathToFiles(fileInfo.dirname);
const parentId = parentFiles[parentFiles.length - 1].id;
if (!fileInfo.basename) {
const specialDirId = await this.specialDirId(fileInfo.dirname);
// The path simply refers to a special directory. Can happen
// for example with a path like `root:/:` (which is the same
// as just `root`).
if (specialDirId) return { id: specialDirId };
}
// This is an existing file
const existingFile = await this.fileByName(parentId, fileInfo.basename);
if (existingFile) return { id: existingFile.id };
if (options.mustExist) throw new ErrorNotFound(`file not found: ${idOrPath}`);
// This is a potentially new file
return {
name: fileInfo.basename,
parent_id: parentId,
};
}
}
protected get defaultFields(): string[] {
return Object.keys(databaseSchema[this.tableName]).filter(f => f !== 'content');
}
private async fileByName(parentId: string, name: string): Promise<File> {
return this.db<File>(this.tableName).select(...this.defaultFields).where({
parent_id: parentId,
name: name,
}).first();
}
protected async validate(object: File, options: ValidateOptions = {}): Promise<File> {
const file: File = object;
const mustBeFile = options.rules.mustBeFile === true;
if (options.isNew) {
if (!file.is_root && !file.name) throw new ErrorUnprocessableEntity('name cannot be empty');
if (file.is_directory && mustBeFile) throw new ErrorUnprocessableEntity('item must not be a directory');
} else {
if ('name' in file && !file.name) throw new ErrorUnprocessableEntity('name cannot be empty');
if ('is_directory' in file) throw new ErrorUnprocessableEntity('cannot turn a file into a directory or vice-versa');
if (mustBeFile && !('is_directory' in file)) {
const existingFile = await this.load(file.id);
if (existingFile.is_directory) throw new ErrorUnprocessableEntity('item must not be a directory');
} else {
if (file.is_directory) throw new ErrorUnprocessableEntity('item must not be a directory');
}
}
let parentId = file.parent_id;
if (!parentId) parentId = await this.userRootFileId();
if ('parent_id' in file && !file.is_root) {
const invalidParentError = function(extraInfo: string) {
let msg = `Invalid parent ID or no permission to write to it: ${parentId}`;
if (nodeEnv !== 'production') msg += ` (${extraInfo})`;
return new ErrorForbidden(msg);
};
if (!parentId) throw invalidParentError('No parent ID');
try {
const parentFile: File = await this.load(parentId);
if (!parentFile) throw invalidParentError('Cannot load parent file');
if (!parentFile.is_directory) throw invalidParentError('Specified parent is not a directory');
await this.checkCanWritePermission(parentFile);
} catch (error) {
if (error.message.indexOf('Invalid parent') === 0) throw error;
throw invalidParentError(`Unknown: ${error.message}`);
}
}
if ('name' in file && !file.is_root) {
const existingFile = await this.fileByName(parentId, file.name);
if (existingFile && options.isNew) throw new ErrorConflict(`Already a file with name "${file.name}"`);
if (existingFile && file.id === existingFile.id) throw new ErrorConflict(`Already a file with name "${file.name}"`);
}
if ('name' in file) {
if (this.includesReservedCharacter(file.name)) throw new ErrorUnprocessableEntity(`File name may not contain any of these characters: ${this.reservedCharacters.join('')}`);
}
return file;
}
public fromApiInput(object: File): File {
const file: File = {};
if ('id' in object) file.id = object.id;
if ('name' in object) file.name = object.name;
if ('parent_id' in object) file.parent_id = object.parent_id;
if ('mime_type' in object) file.mime_type = object.mime_type;
if ('is_directory' in object) file.is_directory = object.is_directory;
return file;
}
public toApiOutput(object: any): any {
if (Array.isArray(object)) {
return object.map(f => this.toApiOutput(f));
} else {
const output: File = { ...object };
delete output.content;
return output;
}
}
public async createRootFile(): Promise<File> {
const existingRootFile = await this.userRootFile();
if (existingRootFile) throw new Error(`User ${this.userId} has already a root file`);
const id = uuidgen();
return this.save({
id: id,
is_directory: 1,
is_root: 1,
name: id, // Name must be unique so we set it to the ID
}, {
isNew: true,
trackChanges: false, // Root file always exist and never changes so we don't want any change event being generated
});
}
private async checkCanReadOrWritePermissions(methodName: 'canRead' | 'canWrite', file: File | File[]): Promise<void> {
const files = Array.isArray(file) ? file : [file];
if (!files.length || !files[0]) throw new ErrorNotFound();
const fileIds = files.map(f => f.id);
const permissionModel = this.models().permission();
const permissionGrantedMap = await permissionModel[methodName](fileIds, this.userId);
for (const file of files) {
if (file.owner_id === this.userId) permissionGrantedMap[file.id] = true;
}
for (const fileId in permissionGrantedMap) {
if (!permissionGrantedMap[fileId]) throw new ErrorForbidden(`No read access to: ${fileId}`);
}
}
private async checkCanReadPermissions(file: File | File[]): Promise<void> {
await this.checkCanReadOrWritePermissions('canRead', file);
}
private async checkCanWritePermission(file: File): Promise<void> {
await this.checkCanReadOrWritePermissions('canWrite', file);
}
private includesReservedCharacter(path: string): boolean {
return this.reservedCharacters.some(c => path.indexOf(c) >= 0);
}
private async pathToFiles(path: string, mustExist: boolean = true): Promise<File[]> {
const filenames = splitItemPath(path);
const output: File[] = [];
let parent: File = null;
for (let i = 0; i < filenames.length; i++) {
const filename = filenames[i];
let file: File = null;
if (i === 0) {
// For now we only support "root" as a root component, but potentially it could
// be any special directory like "documents", "pictures", etc.
if (filename !== 'root') throw new ErrorBadRequest(`unknown path root component: ${filename}`);
file = await this.userRootFile();
} else {
file = await this.fileByName(parent.id, filename);
}
if (!file && mustExist) throw new ErrorNotFound(`file not found: "${filename}" on parent "${parent ? parent.name : ''}"`);
output.push(file);
parent = { ...file };
}
if (!output.length && mustExist) throw new ErrorBadRequest(`path without a base directory: ${path}`);
return output;
}
public async loadWithContent(id: string): Promise<any> {
const file: File = await this.db<File>(this.tableName).select('*').where({ id: id }).first();
if (!file) return null;
await this.checkCanReadPermissions(file);
return file;
}
public async loadByIds(ids: string[]): Promise<File[]> {
const files: File[] = await super.loadByIds(ids);
if (!files.length) return [];
await this.checkCanReadPermissions(files);
return files;
}
public async load(id: string): Promise<File> {
const file: File = await super.load(id);
if (!file) return null;
await this.checkCanReadPermissions(file);
return file;
}
public async save(object: File, options: SaveOptions = {}): Promise<File> {
const isNew = await this.isNew(object, options);
const file: File = { ... object };
if ('content' in file) file.size = file.content ? file.content.byteLength : 0;
if (isNew) {
if (!file.parent_id && !file.is_root) file.parent_id = await this.userRootFileId();
// Even if there's no content, set the mime type based on the extension
if (!file.is_directory) file.mime_type = mimeUtils.fromFilename(file.name);
// Make sure it's not NULL, which is not allowed
if (!file.mime_type) file.mime_type = '';
file.owner_id = this.userId;
}
return super.save(file, options);
}
public async childrens(id: string, pagination: Pagination): Promise<PaginatedFiles> {
const parent = await this.load(id);
await this.checkCanReadPermissions(parent);
return paginateDbQuery(this.db(this.tableName).select(...this.defaultFields).where('parent_id', id), pagination);
}
private async childrenIds(id: string): Promise<string[]> {
const output = await this.db(this.tableName).select('id').where('parent_id', id);
return output.map(r => r.id);
}
public async delete(id: string, options: DeleteOptions = {}): Promise<void> {
const file: File = await this.load(id);
if (!file) return;
await this.checkCanWritePermission(file);
const canDeleteRoot = !!options.validationRules && !!options.validationRules.canDeleteRoot;
if (id === await this.userRootFileId() && !canDeleteRoot) throw new ErrorForbidden('the root directory may not be deleted');
await this.withTransaction(async () => {
await this.models().permission().deleteByFileId(id);
if (file.is_directory) {
const childrenIds = await this.childrenIds(file.id);
for (const childId of childrenIds) {
await this.delete(childId);
}
}
await super.delete(id);
});
}
}

82
packages/server/src/models/PermissionModel.ts Normal file
View File

@ -0,0 +1,82 @@
import BaseModel from './BaseModel';
import { Permission, ItemType, User, Uuid } from '../db';
enum ReadOrWriteKeys {
CanRead = 'can_read',
CanWrite = 'can_write',
}
// Tells whether the given item has the permission to do the required operation
// (can be read or write).
export type PermissionGrantedMap = Record<Uuid, boolean>;
export type PermissionMap = Record<Uuid, Permission[]>;
export default class PermissionModel extends BaseModel {
protected get tableName(): string {
return 'permissions';
}
private async filePermissions(fileId: string, userId: string = null): Promise<Permission[]> {
const p = await this.filesPermissions([fileId], userId);
return p[fileId];
}
private async filesPermissions(fileIds: string[], userId: string = null): Promise<PermissionMap> {
const p: Permission = {
item_type: ItemType.File,
};
if (userId) p.user_id = userId;
const permissions: Permission[] = await this.db<Permission>(this.tableName).where(p).whereIn('item_id', fileIds).select();
const output: PermissionMap = {};
for (const fileId of fileIds) {
output[fileId] = [];
}
for (const permission of permissions) {
output[permission.item_id].push(permission);
}
return output;
}
private async canReadOrWrite(fileIds: string[], userId: string, method: ReadOrWriteKeys): Promise<PermissionGrantedMap> {
const output: PermissionGrantedMap = {};
if (!fileIds.length) throw new Error('No files specified');
if (!userId) throw new Error('No user specified');
const permissionMap = await this.filesPermissions(fileIds, userId);
const userModel = this.models().user({ userId: userId });
const user: User = await userModel.load(userId);
for (const fileId in permissionMap) {
const permissions = permissionMap[fileId];
output[fileId] = !!user.is_admin || !!permissions.find(p => !!p[method]);
}
return output;
}
public async canRead(fileId: string | string[], userId: string): Promise<PermissionGrantedMap> {
fileId = Array.isArray(fileId) ? fileId : [fileId];
return this.canReadOrWrite(fileId, userId, ReadOrWriteKeys.CanRead);
}
public async canWrite(fileId: string | string[], userId: string): Promise<PermissionGrantedMap> {
fileId = Array.isArray(fileId) ? fileId : [fileId];
return this.canReadOrWrite(fileId, userId, ReadOrWriteKeys.CanWrite);
}
public async deleteByFileId(fileId: string): Promise<void> {
const permissions = await this.filePermissions(fileId);
if (!permissions.length) return;
const ids = permissions.map(m => m.id);
await super.delete(ids);
}
}

17
packages/server/src/models/SessionModel.ts Normal file
View File

@ -0,0 +1,17 @@
import BaseModel from './BaseModel';
import { User, Session } from '../db';
export default class SessionModel extends BaseModel {
protected get tableName(): string {
return 'sessions';
}
public async sessionUser(sessionId: string): Promise<User> {
const session: Session = await this.load(sessionId);
if (!session) return null;
const userModel = this.models().user({ userId: session.user_id });
return userModel.load(session.user_id);
}
}

111
packages/server/src/models/UserModel.ts Normal file
View File

@ -0,0 +1,111 @@
import BaseModel, { SaveOptions, ValidateOptions } from './BaseModel';
import { User } from '../db';
import * as auth from '../utils/auth';
import { ErrorUnprocessableEntity, ErrorForbidden } from '../utils/errors';
export default class UserModel extends BaseModel {
public get tableName(): string {
return 'users';
}
public async loadByEmail(email: string): Promise<User> {
const user: User = { email: email };
return this.db<User>(this.tableName).where(user).first();
}
public fromApiInput(object: User): User {
const user: User = {};
if ('id' in object) user.id = object.id;
if ('email' in object) user.email = object.email;
if ('password' in object) user.password = object.password;
if ('is_admin' in object) user.is_admin = object.is_admin;
if ('full_name' in object) user.full_name = object.full_name;
return user;
}
public toApiOutput(object: User): User {
const output: User = { ...object };
delete output.password;
return output;
}
protected async validate(object: User, options: ValidateOptions = {}): Promise<User> {
const user: User = await super.validate(object, options);
const owner: User = await this.load(this.userId);
if (options.isNew) {
if (!owner.is_admin) throw new ErrorForbidden('non-admin user cannot create a new user');
if (!user.email) throw new ErrorUnprocessableEntity('email must be set');
if (!user.password) throw new ErrorUnprocessableEntity('password must be set');
} else {
if (!owner.is_admin && user.id !== owner.id) throw new ErrorForbidden('non-admin user cannot modify another user');
if ('email' in user && !user.email) throw new ErrorUnprocessableEntity('email must be set');
if ('password' in user && !user.password) throw new ErrorUnprocessableEntity('password must be set');
if (!owner.is_admin && 'is_admin' in user) throw new ErrorForbidden('non-admin user cannot make a user an admin');
if (owner.is_admin && owner.id === user.id && 'is_admin' in user && !user.is_admin) throw new ErrorUnprocessableEntity('non-admin user cannot remove admin bit from themselves');
}
if ('email' in user) {
const existingUser = await this.loadByEmail(user.email);
if (existingUser && existingUser.id !== user.id) throw new ErrorUnprocessableEntity(`there is already a user with this email: ${user.email}`);
if (!this.validateEmail(user.email)) throw new ErrorUnprocessableEntity(`Invalid email: ${user.email}`);
}
return user;
}
private validateEmail(email: string): boolean {
const s = email.split('@');
if (s.length !== 2) return false;
return !!s[0].length && !!s[1].length;
}
private async checkIsOwnerOrAdmin(userId: string): Promise<void> {
if (!this.userId) throw new ErrorForbidden('no user is active');
if (userId === this.userId) return;
const owner = await this.load(this.userId);
if (!owner.is_admin) throw new ErrorForbidden();
}
public async load(id: string): Promise<User> {
await this.checkIsOwnerOrAdmin(id);
return super.load(id);
}
public async delete(id: string): Promise<void> {
await this.checkIsOwnerOrAdmin(id);
await this.withTransaction(async () => {
const fileModel = this.models().file({ userId: id });
const rootFile = await fileModel.userRootFile();
await fileModel.delete(rootFile.id, { validationRules: { canDeleteRoot: true } });
await super.delete(id);
});
}
public async save(object: User, options: SaveOptions = {}): Promise<User> {
const isNew = await this.isNew(object, options);
let newUser = { ...object };
if (isNew && newUser.password) newUser.password = auth.hashPassword(newUser.password);
await this.withTransaction(async () => {
newUser = await super.save(newUser, options);
if (isNew) {
const fileModel = this.models().file({ userId: newUser.id });
await fileModel.createRootFile();
}
});
return newUser;
}
}

101
packages/server/src/models/factory.ts Normal file
View File

@ -0,0 +1,101 @@
// Each method of this class returns a new model instance, which can be
// used to manipulate the database.
//
// These instances should be used within the current function, then
// **discarded**. The caller in particular should not keep a copy of the
// model and re-use it across multiple calls as doing so might cause issues
// with the way transactions are managed, especially when concurrency is
// involved.
//
// If a copy of the model is kept, the following could happen:
//
// - Async function1 calls some model function that initiates a transaction
// - Async function2, in parallel, calls a function that also initiates a
// transaction.
//
// Because of this, the transaction stack in BaseModel will be out of
// order, and function2 might pop the transaction of function1 or
// vice-versa. Possibly also commit or rollback the transaction of the
// other function.
//
// For that reason, models should be used in a linear way, with each
// function call being awaited before starting the next one.
//
// If multiple parallel calls are needed, multiple models should be
// created, one for each "thread".
//
// Creating a model is cheap, or should be, so it is not an issue to create
// and destroy them frequently.
//
// Perhaps all this could be enforced in code, but not clear how.
// So this is GOOD:
// class FileController {
// public async deleteFile(id:string) {
// const fileModel = this.models.file();
// await fileModel.delete(id);
// }
// }
// This is BAD:
// class FileController {
//
// private fileModel;
//
// public constructor() {
// // BAD - Don't keep and re-use a copy of it!
// this.fileModel = this.models.file();
// }
//
// public async deleteFile(id:string) {
// await this.fileModel.delete(id);
// }
// }
import { DbConnection } from '../db';
import ApiClientModel from './ApiClientModel';
import { ModelOptions } from './BaseModel';
import FileModel from './FileModel';
import UserModel from './UserModel';
import PermissionModel from './PermissionModel';
import SessionModel from './SessionModel';
import ChangeModel from './ChangeModel';
export class Models {
private db_: DbConnection;
public constructor(db: DbConnection) {
this.db_ = db;
}
public file(options: ModelOptions = null) {
return new FileModel(this.db_, newModelFactory, options);
}
public user(options: ModelOptions = null) {
return new UserModel(this.db_, newModelFactory, options);
}
public apiClient(options: ModelOptions = null) {
return new ApiClientModel(this.db_, newModelFactory, options);
}
public permission(options: ModelOptions = null) {
return new PermissionModel(this.db_, newModelFactory, options);
}
public session(options: ModelOptions = null) {
return new SessionModel(this.db_, newModelFactory, options);
}
public change(options: ModelOptions = null) {
return new ChangeModel(this.db_, newModelFactory, options);
}
}
export default function newModelFactory(db: DbConnection): Models {
return new Models(db);
}

72
packages/server/src/models/utils/pagination.test.ts Normal file
View File

@ -0,0 +1,72 @@
import { expectThrow } from '../../utils/testUtils';
import { defaultPagination, Pagination, requestPagination } from './pagination';
describe('pagination', function() {
test('should create options from request query parameters', async function() {
const d = defaultPagination();
const testCases: any = [
[
null,
d,
],
[
{
order_by: 'title',
},
{
...d,
order: [{
by: 'title',
dir: d.order[0].dir,
}],
},
],
[
{
order_by: 'title',
order_dir: 'asc',
},
{
...d,
order: [{
by: 'title',
dir: 'asc',
}],
},
],
[
{
limit: 55,
},
{
...d,
limit: 55,
},
],
[
{
page: 3,
},
{
...d,
page: 3,
},
],
];
for (const t of testCases) {
const input: any = t[0];
const expected: Pagination = t[1];
const actual: Pagination = requestPagination(input);
expect(actual).toEqual(expected);
}
await expectThrow(async () => requestPagination({ order_dir: 'ASC' }));
await expectThrow(async () => requestPagination({ order_dir: 'DESC' }));
await expectThrow(async () => requestPagination({ page: 0 }));
});
});

122
packages/server/src/models/utils/pagination.ts Normal file
View File

@ -0,0 +1,122 @@
import { ErrorBadRequest } from '../../utils/errors';
import { decodeBase64, encodeBase64 } from '../../utils/base64';
import { ChangePagination, defaultChangePagination } from '../ChangeModel';
import Knex = require('knex');
export enum PaginationOrderDir {
ASC = 'asc',
DESC = 'desc',
}
export interface PaginationOrder {
by: string;
dir: PaginationOrderDir;
}
export interface Pagination {
limit?: number;
order?: PaginationOrder[];
page?: number;
cursor?: string;
}
export interface PaginatedResults {
items: any[];
has_more: boolean;
cursor?: string;
}
const pageMaxSize = 1000;
const defaultOrderField = 'updated_time';
const defaultOrderDir = PaginationOrderDir.DESC;
export function defaultPagination(): Pagination {
return {
limit: pageMaxSize,
order: [
{
by: defaultOrderField,
dir: defaultOrderDir,
},
],
page: 1,
};
}
function dbOffset(pagination: Pagination): number {
return pagination.limit * (pagination.page - 1);
}
function requestPaginationOrder(query: any): PaginationOrder[] {
const orderBy: string = 'order_by' in query ? query.order_by : defaultOrderField;
const orderDir: PaginationOrderDir = 'order_dir' in query ? query.order_dir : defaultOrderDir;
if (![PaginationOrderDir.ASC, PaginationOrderDir.DESC].includes(orderDir)) throw new ErrorBadRequest(`Invalid order_dir parameter: ${orderDir}`);
return [{
by: orderBy,
dir: orderDir,
}];
}
function validatePagination(p: Pagination): Pagination {
if (p.limit < 0 || p.limit > pageMaxSize) throw new ErrorBadRequest(`Limit out of bond: ${p.limit}`);
if (p.page <= 0) throw new ErrorBadRequest(`Invalid page number: ${p.page}`);
for (const o of p.order) {
if (![PaginationOrderDir.ASC, PaginationOrderDir.DESC].includes(o.dir)) throw new ErrorBadRequest(`Invalid order_dir parameter: ${o.dir}`);
}
return p;
}
function processCursor(pagination: Pagination): Pagination {
// If a cursor is present, we parse it and move to the next page.
if (pagination.cursor) {
const p = validatePagination(JSON.parse(decodeBase64(pagination.cursor)));
p.page++;
return p;
}
return pagination as Pagination;
}
export function requestPagination(query: any): Pagination {
if (!query) return defaultPagination();
if ('cursor' in query) {
return processCursor(query);
}
const limit = 'limit' in query ? query.limit : pageMaxSize;
const order: PaginationOrder[] = requestPaginationOrder(query);
const page: number = 'page' in query ? query.page : 1;
return validatePagination({ limit, order, page });
}
export function requestChangePagination(query: any): ChangePagination {
if (!query) return defaultChangePagination();
const output: ChangePagination = {};
if ('limit' in query) output.limit = query.limit;
if ('cursor' in query) output.cursor = query.cursor;
return output;
}
export async function paginateDbQuery(query: Knex.QueryBuilder, pagination: Pagination): Promise<PaginatedResults> {
pagination = processCursor(pagination);
const items = await query
.orderBy(pagination.order[0].by, pagination.order[0].dir)
.offset(dbOffset(pagination))
.limit(pagination.limit);
const hasMore = items.length >= pagination.limit;
return {
items,
has_more: hasMore,
cursor: hasMore ? encodeBase64(JSON.stringify(pagination)) : null,
};
}

87
packages/server/src/routes/api/files.ts Normal file
View File

@ -0,0 +1,87 @@
import { ErrorNotFound, ErrorMethodNotAllowed, ErrorBadRequest } from '../../utils/errors';
import { File } from '../../db';
import { bodyFields, formParse, headerSessionId } from '../../utils/requestUtils';
import { SubPath, Route, ResponseType, Response } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import * as fs from 'fs-extra';
import { requestChangePagination, requestPagination } from '../../models/utils/pagination';
const route: Route = {
exec: async function(path: SubPath, ctx: AppContext) {
const fileController = ctx.controllers.apiFile();
// console.info(`${ctx.method} ${path.id}${path.link ? `/${path.link}` : ''}`);
if (!path.link) {
if (ctx.method === 'GET') {
return fileController.getFile(headerSessionId(ctx.headers), path.id);
}
if (ctx.method === 'PATCH') {
return fileController.patchFile(headerSessionId(ctx.headers), path.id, await bodyFields(ctx.req));
}
if (ctx.method === 'DELETE') {
return fileController.deleteFile(headerSessionId(ctx.headers), path.id);
}
throw new ErrorMethodNotAllowed();
}
if (path.link === 'content') {
if (ctx.method === 'GET') {
const koaResponse = ctx.response;
const file: File = await fileController.getFileContent(headerSessionId(ctx.headers), path.id);
koaResponse.body = file.content;
koaResponse.set('Content-Type', file.mime_type);
koaResponse.set('Content-Length', file.size.toString());
return new Response(ResponseType.KoaResponse, koaResponse);
}
if (ctx.method === 'PUT') {
const result = await formParse(ctx.req);
if (!result?.files?.file) throw new ErrorBadRequest('File data is missing');
const buffer = await fs.readFile(result.files.file.path);
return fileController.putFileContent(headerSessionId(ctx.headers), path.id, buffer);
}
if (ctx.method === 'DELETE') {
return fileController.deleteFileContent(headerSessionId(ctx.headers), path.id);
}
throw new ErrorMethodNotAllowed();
}
if (path.link === 'delta') {
if (ctx.method === 'GET') {
return fileController.getDelta(
headerSessionId(ctx.headers),
path.id,
requestChangePagination(ctx.query)
);
}
throw new ErrorMethodNotAllowed();
}
if (path.link === 'children') {
if (ctx.method === 'GET') {
return fileController.getChildren(headerSessionId(ctx.headers), path.id, requestPagination(ctx.query));
}
if (ctx.method === 'POST') {
return fileController.postChild(headerSessionId(ctx.headers), path.id, await bodyFields(ctx.req));
}
throw new ErrorMethodNotAllowed();
}
throw new ErrorNotFound(`Invalid link: ${path.link}`);
},
needsBodyMiddleware: true,
};
export default route;

11
packages/server/src/routes/api/index.ts Normal file
View File

@ -0,0 +1,11 @@
import { Route } from '../../utils/routeUtils';
const route: Route = {
exec: async function() {
return { status: 'ok', message: 'Joplin Server is running' };
},
};
export default route;

11
packages/server/src/routes/api/ping.ts Normal file
View File

@ -0,0 +1,11 @@
import { Route } from '../../utils/routeUtils';
const route: Route = {
exec: async function() {
return { status: 'ok', message: 'Joplin Server is running' };
},
};
export default route;

23
packages/server/src/routes/api/sessions.ts Normal file
View File

@ -0,0 +1,23 @@
import { SubPath, Route } from '../../utils/routeUtils';
import { ErrorNotFound } from '../../utils/errors';
import { AppContext } from '../../utils/types';
import { bodyFields } from '../../utils/requestUtils';
const route: Route = {
exec: async function(path: SubPath, ctx: AppContext) {
if (!path.link) {
if (ctx.method === 'POST') {
const user = await bodyFields(ctx.req);
const sessionController = ctx.controllers.apiSession();
const session = await sessionController.authenticate(user.email, user.password);
return { id: session.id };
}
}
throw new ErrorNotFound(`Invalid link: ${path.link}`);
},
};
export default route;

60
packages/server/src/routes/default.ts Normal file
View File

@ -0,0 +1,60 @@
import * as Koa from 'koa';
import { SubPath, Route, Response, ResponseType } from '../utils/routeUtils';
import { ErrorNotFound, ErrorForbidden } from '../utils/errors';
import { dirname, normalize } from 'path';
import { pathExists } from 'fs-extra';
import * as fs from 'fs-extra';
const { mime } = require('@joplin/lib/mime-utils.js');
const publicDir = `${dirname(dirname(__dirname))}/public`;
interface PathToFileMap {
[path: string]: string;
}
// Most static assets should be in /public, but for those that are not, for
// example if they are in node_modules, use the map below
const pathToFileMap: PathToFileMap = {
'css/bulma.min.css': 'node_modules/bulma/css/bulma.min.css',
'css/bulma-prefers-dark.min.css': 'node_modules/bulma-prefers-dark/css/bulma-prefers-dark.min.css',
};
async function findLocalFile(path: string): Promise<string> {
if (path in pathToFileMap) return pathToFileMap[path];
let localPath = normalize(path);
if (localPath.indexOf('..') >= 0) throw new ErrorNotFound(`Cannot resolve path: ${path}`);
localPath = `${publicDir}/${localPath}`;
if (!(await pathExists(localPath))) throw new ErrorNotFound(`Path not found: ${path}`);
const stat = await fs.stat(localPath);
if (stat.isDirectory()) throw new ErrorForbidden(`Directory listing not allowed: ${path}`);
return localPath;
}
const route: Route = {
exec: async function(path: SubPath, ctx: Koa.Context) {
if (ctx.method === 'GET') {
const localPath = await findLocalFile(path.raw);
let mimeType: string = mime.fromFilename(localPath);
if (!mimeType) mimeType = 'application/octet-stream';
const fileContent: Buffer = await fs.readFile(localPath);
const koaResponse = ctx.response;
koaResponse.body = fileContent;
koaResponse.set('Content-Type', mimeType);
koaResponse.set('Content-Length', fileContent.length.toString());
return new Response(ResponseType.KoaResponse, koaResponse);
}
throw new ErrorNotFound();
},
};
export default route;

21
packages/server/src/routes/index/home.ts Normal file
View File

@ -0,0 +1,21 @@
import { SubPath, Route } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { contextSessionId } from '../../utils/requestUtils';
import { ErrorMethodNotAllowed } from '../../utils/errors';
const route: Route = {
exec: async function(_path: SubPath, ctx: AppContext) {
const sessionId = contextSessionId(ctx);
const homeController = ctx.controllers.indexHome();
if (ctx.method === 'GET') {
return homeController.getIndex(sessionId);
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

33
packages/server/src/routes/index/login.ts Normal file
View File

@ -0,0 +1,33 @@
import { SubPath, Route, redirect } from '../../utils/routeUtils';
import { ErrorMethodNotAllowed } from '../../utils/errors';
import { AppContext } from '../../utils/types';
import { formParse } from '../../utils/requestUtils';
import { baseUrl } from '../../config';
const route: Route = {
exec: async function(_path: SubPath, ctx: AppContext) {
const loginController = ctx.controllers.indexLogin();
if (ctx.method === 'GET') {
return loginController.getIndex();
}
if (ctx.method === 'POST') {
try {
const body = await formParse(ctx.req);
const session = await ctx.controllers.apiSession().authenticate(body.fields.email, body.fields.password);
ctx.cookies.set('sessionId', session.id);
return redirect(ctx, `${baseUrl()}/home`);
} catch (error) {
return loginController.getIndex(error);
}
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

20
packages/server/src/routes/index/logout.ts Normal file
View File

@ -0,0 +1,20 @@
import { SubPath, Route, redirect } from '../../utils/routeUtils';
import { ErrorMethodNotAllowed } from '../../utils/errors';
import { AppContext } from '../../utils/types';
import { baseUrl } from '../../config';
const route: Route = {
exec: async function(_path: SubPath, ctx: AppContext) {
if (ctx.method === 'POST') {
// TODO: also delete the session from the database
ctx.cookies.set('sessionId', '');
return redirect(ctx, `${baseUrl()}/login`);
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

47
packages/server/src/routes/index/profile.ts Normal file
View File

@ -0,0 +1,47 @@
import { SubPath, Route, redirect } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { contextSessionId, formParse } from '../../utils/requestUtils';
import { ErrorMethodNotAllowed, ErrorUnprocessableEntity } from '../../utils/errors';
import { User } from '../../db';
import { baseUrl } from '../../config';
import { hashPassword } from '../../utils/auth';
const route: Route = {
exec: async function(_path: SubPath, ctx: AppContext) {
const sessionId = contextSessionId(ctx);
if (ctx.method === 'GET') {
return ctx.controllers.indexProfile().getIndex(sessionId);
}
if (ctx.method === 'POST') {
let user: User = {};
try {
const body = await formParse(ctx.req);
user = {
id: body.fields.id,
email: body.fields.email,
full_name: body.fields.full_name,
};
if (body.fields.password) {
if (body.fields.password !== body.fields.password2) throw new ErrorUnprocessableEntity('Passwords do not match');
user.password = hashPassword(body.fields.password);
}
await ctx.controllers.indexProfile().patchIndex(sessionId, user);
return redirect(ctx, `${baseUrl()}/profile`);
} catch (error) {
return ctx.controllers.indexProfile().getIndex(sessionId, user, error);
}
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

59
packages/server/src/routes/index/user.ts Normal file
View File

@ -0,0 +1,59 @@
import { SubPath, Route, redirect } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { contextSessionId, formParse } from '../../utils/requestUtils';
import { ErrorMethodNotAllowed, ErrorUnprocessableEntity } from '../../utils/errors';
import { User } from '../../db';
import { baseUrl } from '../../config';
function makeUser(isNew: boolean, fields: any): User {
const user: User = {
email: fields.email,
full_name: fields.full_name,
};
if (fields.password) {
if (fields.password !== fields.password2) throw new ErrorUnprocessableEntity('Passwords do not match');
user.password = fields.password;
}
if (!isNew) user.id = fields.id;
return user;
}
const route: Route = {
exec: async function(_path: SubPath, ctx: AppContext) {
const sessionId = contextSessionId(ctx);
// if (ctx.method === 'GET') {
// return ctx.controllers.indexUser().getOne(sessionId);
// }
if (ctx.method === 'POST') {
const user: User = {};
try {
const body = await formParse(ctx.req);
const fields = body.fields;
const isNew = !!Number(fields.is_new);
const user = makeUser(isNew, fields);
if (isNew) {
await ctx.controllers.apiUser().postUser(sessionId, user);
} else {
await ctx.controllers.apiUser().patchUser(sessionId, user);
}
return redirect(ctx, `${baseUrl()}/users`);
} catch (error) {
return ctx.controllers.indexProfile().getIndex(sessionId, user, error);
}
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

69
packages/server/src/routes/index/users.ts Normal file
View File

@ -0,0 +1,69 @@
import { SubPath, Route, redirect } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { contextSessionId, formParse } from '../../utils/requestUtils';
import { ErrorMethodNotAllowed, ErrorUnprocessableEntity } from '../../utils/errors';
import { User } from '../../db';
import { baseUrl } from '../../config';
function makeUser(isNew: boolean, fields: any): User {
const user: User = {
email: fields.email,
full_name: fields.full_name,
};
if (fields.password) {
if (fields.password !== fields.password2) throw new ErrorUnprocessableEntity('Passwords do not match');
user.password = fields.password;
}
if (!isNew) user.id = fields.id;
return user;
}
const route: Route = {
exec: async function(path: SubPath, ctx: AppContext) {
const sessionId = contextSessionId(ctx);
const isNew = path.id === 'new';
if (ctx.method === 'GET') {
if (path.id) {
return ctx.controllers.indexUser().getOne(sessionId, isNew, !isNew ? path.id : null);
} else {
return ctx.controllers.indexUser().getIndex(sessionId);
}
}
if (ctx.method === 'POST') {
let user: User = {};
try {
const body = await formParse(ctx.req);
const fields = body.fields;
user = makeUser(isNew, fields);
if (fields.post_button) {
if (isNew) {
await ctx.controllers.apiUser().postUser(sessionId, user);
} else {
await ctx.controllers.apiUser().patchUser(sessionId, user);
}
} else if (fields.delete_button) {
await ctx.controllers.apiUser().deleteUser(sessionId, path.id);
} else {
throw new Error('Invalid form button');
}
return redirect(ctx, `${baseUrl()}/users`);
} catch (error) {
return ctx.controllers.indexUser().getOne(sessionId, isNew, user, error);
}
}
throw new ErrorMethodNotAllowed();
},
};
export default route;

27
packages/server/src/routes/oauth2/authorize.ts Normal file
View File

@ -0,0 +1,27 @@
// import { ErrorMethodNotAllowed } from '../../utils/errors';
// import { SubPath, Route } from '../../utils/routeUtils';
// import { AppContext } from '../../utils/types';
// const route: Route = {
// exec: async function(_: SubPath, ctx: AppContext) {
// const controller = ctx.controllers.oauth();
// if (ctx.method === 'GET') {
// return controller.getAuthorize(ctx.request.query);
// }
// if (ctx.method === 'POST') {
// return controller.postAuthorize(ctx.request.body);
// }
// throw new ErrorMethodNotAllowed();
// },
// needsBodyMiddleware: true,
// // responseFormat: 'html',
// };
// export default route;

31
packages/server/src/routes/routes.ts Normal file
View File

@ -0,0 +1,31 @@
import { Routes } from '../utils/routeUtils';
import apiSessions from './api/sessions';
import apiPing from './api/ping';
import apiFiles from './api/files';
// import oauth2Authorize from './oauth2/authorize';
import indexLoginRoute from './index/login';
import indexLogoutRoute from './index/logout';
import indexHomeRoute from './index/home';
import indexProfileRoute from './index/profile';
import indexUsersRoute from './index/users';
import indexUserRoute from './index/user';
import defaultRoute from './default';
const routes: Routes = {
'api/ping': apiPing,
'api/sessions': apiSessions,
'api/files': apiFiles,
// 'oauth2/authorize': oauth2Authorize,
'login': indexLoginRoute,
'logout': indexLogoutRoute,
'home': indexHomeRoute,
'profile': indexProfileRoute,
'users': indexUsersRoute,
'user': indexUserRoute,
'': defaultRoute,
};
export default routes;

103
packages/server/src/services/MustacheService.ts Normal file
View File

@ -0,0 +1,103 @@
import * as Mustache from 'mustache';
import * as fs from 'fs-extra';
import config, { baseUrl } from '../config';
export interface RenderOptions {
partials?: any;
cssFiles?: string[];
jsFiles?: string[];
}
export interface View {
name: string;
path: string;
content?: any;
partials?: string[];
cssFiles?: string[];
jsFiles?: string[];
}
export function isView(o: any): boolean {
if (typeof o !== 'object' || !o) return false;
return 'path' in o && 'name' in o;
}
class MustacheService {
private get defaultLayoutPath(): string {
return `${config().layoutDir}/default.mustache`;
}
private get defaultLayoutOptions(): any {
return {
baseUrl: baseUrl(),
};
}
private async loadTemplateContent(path: string): Promise<string> {
return fs.readFile(path, 'utf8');
}
private resolvesFilePaths(type: string, paths: string[]): string[] {
const output: string[] = [];
for (const path of paths) {
output.push(`${baseUrl()}/${type}/${path}.${type}`);
}
return output;
}
public async renderView(view: View): Promise<string> {
const partials = view.partials || [];
const cssFiles = this.resolvesFilePaths('css', view.cssFiles || []);
const jsFiles = this.resolvesFilePaths('js', view.jsFiles || []);
const partialContents: any = {};
for (const partialName of partials) {
const filePath = `${config().viewDir}/partials/${partialName}.mustache`;
partialContents[partialName] = await this.loadTemplateContent(filePath);
}
const filePath = `${config().viewDir}/${view.path}.mustache`;
const contentHtml = Mustache.render(
await this.loadTemplateContent(filePath),
{
...view.content,
global: this.defaultLayoutOptions,
},
partialContents
);
const layoutView: any = Object.assign({}, this.defaultLayoutOptions, {
pageName: view.name,
contentHtml: contentHtml,
cssFiles: cssFiles,
jsFiles: jsFiles,
...view.content,
});
return Mustache.render(await this.loadTemplateContent(this.defaultLayoutPath), layoutView, partialContents);
}
// public async render(path: string, view: any, options: RenderOptions = null): Promise<string> {
// const partials = options && options.partials ? options.partials : {};
// const cssFiles = this.resolvesFilePaths('css', options && options.cssFiles ? options.cssFiles : []);
// const jsFiles = this.resolvesFilePaths('js', options && options.jsFiles ? options.jsFiles : []);
// const filePath = `${config().viewDir}/${path}.mustache`;
// const contentHtml = Mustache.render(await this.loadTemplateContent(filePath), { ...view, global: this.defaultLayoutOptions }, partials);
// const layoutView: any = Object.assign({}, this.defaultLayoutOptions, {
// contentHtml: contentHtml,
// cssFiles: cssFiles,
// jsFiles: jsFiles,
// });
// return Mustache.render(await this.loadTemplateContent(this.defaultLayoutPath), layoutView);
// }
}
const mustacheService = new MustacheService();
export default mustacheService;

27
packages/server/src/tools/db-migrate.ts Normal file
View File

@ -0,0 +1,27 @@
// import db, { dbConfig } from '../app/db';
// // require('source-map-support').install();
// const config = {
// directory: `${__dirname}/../migrations`,
// // Disable transactions because the models might open one too
// disableTransactions: true,
// };
// console.info(`Using database: ${dbConfig().connection.filename}`);
// console.info(`Running migrations in: ${config.directory}`);
// db().migrate.latest(config).then((event: any) => {
// const log: string[] = event[1];
// if (!log.length) {
// console.info('Database is already up to date');
// } else {
// console.info(`Ran migrations: ${log.join(', ')}`);
// }
// db().destroy();
// }).catch((error:any) => {
// console.error(error);
// process.exit(1);
// });

76
packages/server/src/tools/dbTools.ts Normal file
View File

@ -0,0 +1,76 @@
import { connectDb, disconnectDb, migrateDb, sqliteFilePath } from '../db';
import * as fs from 'fs-extra';
import { DatabaseConfig } from '../utils/types';
const { execCommand } = require('@joplin/tools/tool-utils');
export interface CreateDbOptions {
dropIfExists: boolean;
}
export interface DropDbOptions {
ignoreIfNotExists: boolean;
}
export async function createDb(config: DatabaseConfig, options: CreateDbOptions = null) {
options = {
dropIfExists: false,
...options,
};
if (config.client === 'pg') {
const cmd: string[] = [
'createdb',
'--host', config.host,
'--port', config.port.toString(),
'--username', config.user,
config.name,
];
if (options.dropIfExists) {
await dropDb(config, { ignoreIfNotExists: true });
}
await execCommand(cmd.join(' '));
} else if (config.client === 'sqlite3') {
const filePath = sqliteFilePath(config);
if (await fs.pathExists(filePath)) {
if (options.dropIfExists) {
await fs.remove(filePath);
} else {
throw new Error(`Database already exists: ${filePath}`);
}
}
}
const db = await connectDb(config);
await migrateDb(db);
await disconnectDb(db);
}
export async function dropDb(config: DatabaseConfig, options: DropDbOptions = null) {
options = {
ignoreIfNotExists: false,
...options,
};
if (config.client === 'pg') {
const cmd: string[] = [
'dropdb',
'--host', config.host,
'--port', config.port.toString(),
'--username', config.user,
config.name,
];
try {
await execCommand(cmd.join(' '));
} catch (error) {
if (options.ignoreIfNotExists && error.message.includes('does not exist')) return;
throw error;
}
} else if (config.client === 'sqlite3') {
await fs.remove(sqliteFilePath(config));
}
}

127
packages/server/src/tools/generate-types.ts Normal file
View File

@ -0,0 +1,127 @@
import sqlts from '@rmp135/sql-ts';
require('source-map-support').install();
const dbFilePath: string = `${__dirname}/../../src/db.ts`;
const fileReplaceWithinMarker = '// AUTO-GENERATED-TYPES';
const config = {
'client': 'sqlite3',
'connection': {
'filename': './db-buildTypes.sqlite',
},
'useNullAsDefault': true,
'excludedTables': [
'main.knex_migrations',
'main.knex_migrations_lock',
'android_metadata',
],
'interfaceNameFormat': '${table}',
'singularTableNames': true,
'tableNameCasing': 'pascal' as any,
'filename': './db',
'extends': {
'main.sessions': 'WithDates, WithUuid',
'main.users': 'WithDates, WithUuid',
'main.permissions': 'WithDates, WithUuid',
'main.files': 'WithDates, WithUuid',
'main.api_clients': 'WithDates, WithUuid',
'main.changes': 'WithDates, WithUuid',
},
};
function insertContentIntoFile(filePath: string, markerOpen: string, markerClose: string, contentToInsert: string): void {
const fs = require('fs');
if (!fs.existsSync(filePath)) throw new Error(`File not found: ${filePath}`);
let content: string = fs.readFileSync(filePath, 'utf-8');
// [^]* matches any character including new lines
const regex: RegExp = new RegExp(`${markerOpen}[^]*?${markerClose}`);
if (!content.match(regex)) throw new Error(`Could not find markers: ${markerOpen}`);
content = content.replace(regex, `${markerOpen}\n${contentToInsert}\n${markerClose}`);
fs.writeFileSync(filePath, content);
}
// To output:
//
// export interface User extends WithDates, WithUuid {
// email?: string
// password?: string
// is_admin?: number
// }
function createTypeString(table: any) {
const colStrings = [];
for (const col of table.columns) {
const name = col.propertyName as string;
let type = col.propertyType;
if (table.extends && table.extends.indexOf('WithDates') >= 0) {
if (['created_time', 'updated_time'].includes(name)) continue;
}
if (table.extends && table.extends.indexOf('WithUuid') >= 0) {
if (['id'].includes(name)) continue;
}
if (name === 'item_type') type = 'ItemType';
if (table.name === 'files' && name === 'content') type = 'Buffer';
if (table.name === 'changes' && name === 'type') type = 'ChangeType';
if ((name === 'id' || name.endsWith('_id') || name === 'uuid') && type === 'string') type = 'Uuid';
colStrings.push(`\t${name}?: ${type};`);
}
const header = ['export interface'];
header.push(table.interfaceName);
if (table.extends) header.push(`extends ${table.extends}`);
return `${header.join(' ')} {\n${colStrings.join('\n')}\n}`;
}
// To output:
//
// export const databaseSchema:DatabaseTables = {
// users: {
// id: { type: "string" },
// email: { type: "string" },
// password: { type: "string" },
// is_admin: { type: "number" },
// updated_time: { type: "number" },
// created_time: { type: "number" },
// },
// }
function createRuntimeObject(table: any) {
const colStrings = [];
for (const col of table.columns) {
const name = col.propertyName;
const type = col.propertyType;
colStrings.push(`\t\t${name}: { type: '${type}' },`);
}
return `\t${table.name}: {\n${colStrings.join('\n')}\n\t},`;
}
async function main() {
const definitions = await sqlts.toObject(config);
const typeStrings = [];
for (const table of definitions.tables) {
typeStrings.push(createTypeString(table));
}
const tableStrings = [];
for (const table of definitions.tables) {
tableStrings.push(createRuntimeObject(table));
}
let content = `// Auto-generated using \`npm run generate-types\`\n${typeStrings.join('\n\n')}`;
content += '\n\n';
content += `export const databaseSchema: DatabaseTables = {\n${tableStrings.join('\n')}\n};`;
insertContentIntoFile(dbFilePath, fileReplaceWithinMarker, fileReplaceWithinMarker, content);
}
main().catch(error => {
console.error('Fatal error', error);
process.exit(1);
});

81
packages/server/src/utils/TransactionHandler.ts Normal file
View File

@ -0,0 +1,81 @@
import * as Knex from 'knex';
import { DbConnection } from '../db';
// This transaction handler allows abstracting away the complexity of managing nested transactions
// within models.
// Any method in a model can start a transaction and, if one is already started, it
// simply won't do anything. The last active transaction commits the results. If a rollback
// happens, the following calls to rollback will be a no-op.
// Set logEnabled_ to `true` to see what happens with nested transactions.
export default class TransactionHandler {
private transactionStack_: number[] = [];
private activeTransaction_: Knex.Transaction = null;
private transactionIndex_: number = 0;
private logEnabled_: boolean = false;
private db_: Knex = null;
public constructor(db: DbConnection) {
this.db_ = db;
}
private get db(): DbConnection {
return this.db_;
}
public setDb(db: DbConnection) {
this.db_ = db;
}
private log(s: string): void {
if (!this.logEnabled_) return;
console.info(`TransactionHandler: ${s}`);
}
public get activeTransaction(): Knex.Transaction {
return this.activeTransaction_;
}
public async start(): Promise<number> {
const txIndex = ++this.transactionIndex_;
this.log(`Starting transaction: ${txIndex}`);
if (!this.transactionStack_.length) {
if (this.activeTransaction_) throw new Error('An active transaction was found when no transaction was in stack'); // Sanity check
this.log(`Trying to acquire transaction: ${txIndex}`);
this.activeTransaction_ = await this.db.transaction();
this.log(`Got transaction: ${txIndex}`);
}
this.transactionStack_.push(txIndex);
return txIndex;
}
private finishTransaction(txIndex: number): boolean {
if (!this.transactionStack_.length) throw new Error('Committing but no transaction was started');
const lastTxIndex = this.transactionStack_.pop();
if (lastTxIndex !== txIndex) throw new Error(`Committing a transaction but was not last to start one: ${txIndex}. Expected: ${lastTxIndex}`);
return !this.transactionStack_.length;
}
public async commit(txIndex: number): Promise<void> {
this.log(`Commit transaction: ${txIndex}`);
const isLastTransaction = this.finishTransaction(txIndex);
if (isLastTransaction) {
this.log(`Is last transaction - doing commit: ${txIndex}`);
await this.activeTransaction_.commit();
this.activeTransaction_ = null;
}
}
public async rollback(txIndex: number): Promise<void> {
this.log(`Rollback transaction: ${txIndex}`);
this.finishTransaction(txIndex);
if (this.activeTransaction_) {
this.log(`Transaction is active - doing rollback: ${txIndex}`);
await this.activeTransaction_.rollback();
this.activeTransaction_ = null;
}
}
}

10
packages/server/src/utils/auth.ts Normal file
View File

@ -0,0 +1,10 @@
const bcrypt = require('bcryptjs');
export function hashPassword(password: string): string {
const salt = bcrypt.genSaltSync(10);
return bcrypt.hashSync(password, salt);
}
export function checkPassword(password: string, hash: string): boolean {
return bcrypt.compareSync(password, hash);
}

7
packages/server/src/utils/base64.ts Normal file
View File

@ -0,0 +1,7 @@
export function encodeBase64(s: string): string {
return Buffer.from(s).toString('base64');
}
export function decodeBase64(s: string): string {
return Buffer.from(s, 'base64').toString('utf8');
}

53
packages/server/src/utils/cache.ts Normal file
View File

@ -0,0 +1,53 @@
interface CacheEntry {
object: any;
timestamp: number;
}
interface CacheEntries {
[key: string]: CacheEntry;
}
class Cache {
cache: CacheEntries = {};
private async setAny(key: string, o: any): Promise<void> {
this.cache[key] = {
object: JSON.stringify(o),
timestamp: Date.now(),
};
}
async setObject(key: string, object: Object): Promise<void> {
if (!object) return;
return this.setAny(key, object);
}
private async getAny(key: string): Promise<any> {
if (!this.cache[key]) return null;
try {
const output = JSON.parse(this.cache[key].object);
return output;
} catch (error) {
throw new Error(`Cannot unserialize object: ${key}: ${error.message}: ${this.cache[key].object}`);
}
}
async object(key: string): Promise<object> {
return this.getAny(key) as object;
}
async delete(key: string | string[]): Promise<void> {
const keys = typeof key === 'string' ? [key] : key;
for (const k of keys) delete this.cache[k];
}
async clearAll(): Promise<void> {
this.cache = {};
}
}
const cache: Cache = new Cache();
export default cache;

14
packages/server/src/utils/defaultView.ts Normal file
View File

@ -0,0 +1,14 @@
import { User } from '../db';
import { View } from '../services/MustacheService';
// Populate a View object with some good defaults.
export default function(name: string, owner: User = null): View {
return {
name: name,
path: `index/${name}`,
content: {
owner,
},
partials: ['navbar'],
};
}

63
packages/server/src/utils/errors.ts Normal file
View File

@ -0,0 +1,63 @@
// For explanation of the setPrototypeOf call, see:
// https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
class ApiError extends Error {
public httpCode: number;
public code: string;
public constructor(message: string, httpCode: number = 400, code: string = undefined) {
super(message);
this.httpCode = httpCode;
this.code = code;
Object.setPrototypeOf(this, ApiError.prototype);
}
}
export class ErrorMethodNotAllowed extends ApiError {
public constructor(message: string = 'Method Not Allowed') {
super(message, 405);
Object.setPrototypeOf(this, ErrorMethodNotAllowed.prototype);
}
}
export class ErrorNotFound extends ApiError {
public constructor(message: string = 'Not Found') {
super(message, 404);
Object.setPrototypeOf(this, ErrorNotFound.prototype);
}
}
export class ErrorForbidden extends ApiError {
public constructor(message: string = 'Forbidden') {
super(message, 403);
Object.setPrototypeOf(this, ErrorForbidden.prototype);
}
}
export class ErrorBadRequest extends ApiError {
public constructor(message: string = 'Bad Request') {
super(message, 400);
Object.setPrototypeOf(this, ErrorBadRequest.prototype);
}
}
export class ErrorUnprocessableEntity extends ApiError {
public constructor(message: string = 'Unprocessable Entity') {
super(message, 422);
Object.setPrototypeOf(this, ErrorUnprocessableEntity.prototype);
}
}
export class ErrorConflict extends ApiError {
public constructor(message: string = 'Conflict') {
super(message, 409);
Object.setPrototypeOf(this, ErrorConflict.prototype);
}
}
export class ErrorResyncRequired extends ApiError {
public constructor(message: string = 'Delta cursor is invalid and the complete data should be resynced') {
super(message, 400, 'resyncRequired');
Object.setPrototypeOf(this, ErrorResyncRequired.prototype);
}
}

8
packages/server/src/utils/htmlUtils.ts Normal file
View File

@ -0,0 +1,8 @@
/* eslint-disable import/prefer-default-export */
const Entities = require('html-entities').AllHtmlEntities;
const htmlentities = new Entities().encode;
export function escapeHtml(s: string) {
return htmlentities(s);
}

13
packages/server/src/utils/koaIf.ts Normal file
View File

@ -0,0 +1,13 @@
import { Context } from 'koa';
export default function koaIf(middleware: Function, condition: any = null) {
return async (ctx: Context, next: Function) => {
if (typeof condition === 'function' && condition(ctx)) {
await middleware(ctx, next);
} else if (typeof condition === 'boolean' && condition) {
await middleware(ctx, next);
} else {
await next();
}
};
}

45
packages/server/src/utils/requestUtils.ts Normal file
View File

@ -0,0 +1,45 @@
import { ErrorBadRequest, ErrorForbidden } from './errors';
import { AppContext } from './types';
const formidable = require('formidable');
export type BodyFields = Record<string, any>;
interface FormParseResult {
fields: BodyFields;
files: any;
}
// Input should be Koa ctx.req, which corresponds to the native Node request
export async function formParse(req: any): Promise<FormParseResult> {
return new Promise((resolve: Function, reject: Function) => {
const form = formidable({ multiples: true });
form.parse(req, (error: any, fields: any, files: any) => {
if (error) {
reject(error);
return;
}
resolve({ fields, files });
});
});
}
export async function bodyFields(req: any): Promise<BodyFields> {
if (req.headers['content-type'] !== 'application/json') {
throw new ErrorBadRequest(`Unsupported Content-Type: "${req.headers['content-type']}". Expected: "application/json"`);
}
const form = await formParse(req);
return form.fields;
}
export function headerSessionId(headers: any): string {
return headers['x-api-auth'] ? headers['x-api-auth'] : '';
}
export function contextSessionId(ctx: AppContext): string {
const id = ctx.cookies.get('sessionId');
if (!id) throw new ErrorForbidden('Invalid or missing session');
return id;
}

Some files were not shown because too many files have changed in this diff Show More