self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-10-31 00:07:48 +02:00
Code Issues Releases Activity

All: Refactored REST API to make it testable and to allow further extension

Browse Source
This commit is contained in:
Laurent Cozic
2018-09-27 09:14:05 +01:00
parent bc09d2c640
commit 8a619e4b8b
8 changed files with 1208 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CliClient/.gitignore vendored
View File

@@ -13,6 +13,7 @@ tests/fuzzing.*
tests/fuzzing -*
tests/logs/*
tests/cli-integration/
tests/tmp/
*.mo
*.*~
tests/sync

874
CliClient/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
CliClient/package.json
View File

@@ -37,6 +37,7 @@
"fs-extra": "^5.0.0",
"html-entities": "^1.2.1",
"html-minifier": "^3.5.15",
"image-data-uri": "^2.0.0",
"image-type": "^3.0.0",
"joplin-turndown": "^4.0.8",
"joplin-turndown-plugin-gfm": "^1.0.7",
@@ -57,7 +58,7 @@
"redux": "^3.7.2",
"sax": "^1.2.2",
"server-destroy": "^1.0.1",
"sharp": "^0.18.4",
"sharp": "^0.20.8",
"sprintf-js": "^1.1.1",
"sqlite3": "^4.0.1",
"string-padding": "^1.0.2",

118
CliClient/tests/services_rest_Api.js Normal file
View File

@@ -0,0 +1,118 @@
require('app-module-path').addPath(__dirname);
const { time } = require('lib/time-utils.js');
const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
const markdownUtils = require('lib/markdownUtils.js');
const Api = require('lib/services/rest/Api');
const Folder = require('lib/models/Folder');
const Resource = require('lib/models/Resource');
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
let api = null;
describe('services_rest_Api', function() {
beforeEach(async (done) => {
api = new Api();
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
done();
});
it('should ping', async (done) => {
const response = await api.route('GET', 'ping');
expect(response).toBe('JoplinClipperServer');
done();
});
it('should handle Not Found errors', async (done) => {
const hasThrown = await checkThrowAsync(async () => await api.route('GET', 'pong'));
expect(hasThrown).toBe(true);
done();
});
it('should get folders', async (done) => {
let f1 = await Folder.save({ title: "mon carnet" });
const response = await api.route('GET', 'folders');
expect(response.length).toBe(1);
expect(response[0].title).toBe('mon carnet');
done();
});
it('should create notes', async (done) => {
let response = null;
const f = await Folder.save({ title: "mon carnet" });
response = await api.route('POST', 'notes', null, JSON.stringify({
title: 'testing',
parent_id: f.id,
}));
expect(response.title).toBe('testing');
expect(!!response.id).toBe(true);
response = await api.route('POST', 'notes', null, JSON.stringify({
title: 'testing',
parent_id: f.id,
}));
expect(response.title).toBe('testing');
expect(!!response.id).toBe(true);
done();
});
it('should create notes with images', async (done) => {
let response = null;
const f = await Folder.save({ title: "mon carnet" });
response = await api.route('POST', 'notes', null, JSON.stringify({
title: 'testing image',
parent_id: f.id,
image_data_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAANZJREFUeNoAyAA3/wFwtO3K6gUB/vz2+Prw9fj/+/r+/wBZKAAExOgF4/MC9ff+MRH6Ui4E+/0Bqc/zutj6AgT+/Pz7+vv7++nu82c4DlMqCvLs8goA/gL8/fz09fb59vXa6vzZ6vjT5fbn6voD/fwC8vX4UiT9Zi//APHyAP8ACgUBAPv5APz7BPj2+DIaC2o3E+3o6ywaC5fT6gD6/QD9/QEVf9kD+/dcLQgJA/7v8vqfwOf18wA1IAIEVycAyt//v9XvAPv7APz8LhoIAPz9Ri4OAgwARgx4W/6fVeEAAAAASUVORK5CYII="
}));
const resources = await Resource.all();
expect(resources.length).toBe(1);
const resource = resources[0];
expect(response.body.indexOf(resource.id) >= 0).toBe(true);
done();
});
it('should create notes from HTML', async (done) => {
let response = null;
const f = await Folder.save({ title: "mon carnet" });
response = await api.route('POST', 'notes', null, JSON.stringify({
title: 'testing HTML',
parent_id: f.id,
body_html: '<b>Bold text</b>',
}));
expect(response.body).toBe('**Bold text**');
done();
});
it('should filter fields', async (done) => {
let f = api.fields_({ query: { fields: 'one,two' } }, []);
expect(f.length).toBe(2);
expect(f[0]).toBe('one');
expect(f[1]).toBe('two');
f = api.fields_({ query: { fields: 'one ,, two ' } }, []);
expect(f.length).toBe(2);
expect(f[0]).toBe('one');
expect(f[1]).toBe('two');
f = api.fields_({ query: { fields: ' ' } }, ['def']);
expect(f.length).toBe(1);
expect(f[0]).toBe('def');
done();
});
});

3
CliClient/tests/test-utils.js
View File

@@ -48,7 +48,9 @@ EncryptionService.fsDriver_ = fsDriver;
FileApiDriverLocal.fsDriver_ = fsDriver;
const logDir = __dirname + '/../tests/logs';
const tempDir = __dirname + '/../tests/tmp';
fs.mkdirpSync(logDir, 0o755);
fs.mkdirpSync(tempDir, 0o755);
SyncTargetRegistry.addClass(SyncTargetMemory);
SyncTargetRegistry.addClass(SyncTargetFilesystem);
@@ -80,6 +82,7 @@ BaseItem.loadClass('MasterKey', MasterKey);
Setting.setConstant('appId', 'net.cozic.joplin-cli');
Setting.setConstant('appType', 'cli');
Setting.setConstant('tempDir', tempDir);
BaseService.logger_ = logger;

231
ReactNativeClient/lib/ClipperServer.js
View File

@@ -1,19 +1,10 @@
const { netUtils } = require('lib/net-utils');
const urlParser = require("url");
const Note = require('lib/models/Note');
const Folder = require('lib/models/Folder');
const Resource = require('lib/models/Resource');
const Tag = require('lib/models/Tag');
const Setting = require('lib/models/Setting');
const { shim } = require('lib/shim');
const md5 = require('md5');
const { fileExtension, safeFileExtension, safeFilename, filename } = require('lib/path-utils');
const HtmlToMd = require('lib/HtmlToMd');
const { Logger } = require('lib/logger.js');
const markdownUtils = require('lib/markdownUtils');
const mimeUtils = require('lib/mime-utils.js').mime;
const randomClipperPort = require('lib/randomClipperPort');
const enableServerDestroy = require('server-destroy');
const Api = require('lib/services/rest/Api');
class ClipperServer {
@@ -22,6 +13,7 @@ class ClipperServer {
this.startState_ = 'idle';
this.server_ = null;
this.port_ = null;
this.api_ = new Api();
}
static instance() {
@@ -32,6 +24,7 @@ class ClipperServer {
setLogger(l) {
this.logger_ = l;
this.api_.setLogger(l);
}
logger() {
@@ -65,140 +58,6 @@ class ClipperServer {
});
}
htmlToMdParser() {
if (this.htmlToMdParser_) return this.htmlToMdParser_;
this.htmlToMdParser_ = new HtmlToMd();
return this.htmlToMdParser_;
}
async requestNoteToNote(requestNote) {
const output = {
title: requestNote.title ? requestNote.title : '',
body: requestNote.body ? requestNote.body : '',
};
if (requestNote.body_html) {
// Parsing will not work if the HTML is not wrapped in a top level tag, which is not guaranteed
// when getting the content from elsewhere. So here wrap it - it won't change anything to the final
// rendering but it makes sure everything will be parsed.
output.body = await this.htmlToMdParser().parse('<div>' + requestNote.body_html + '</div>', {
baseUrl: requestNote.base_url ? requestNote.base_url : '',
});
}
if (requestNote.parent_id) {
output.parent_id = requestNote.parent_id;
} else {
const folder = await Folder.defaultFolder();
if (!folder) throw new Error('Cannot find folder for note');
output.parent_id = folder.id;
}
if (requestNote.source_url) output.source_url = requestNote.source_url;
if (requestNote.author) output.author = requestNote.author;
return output;
}
// Note must have been saved first
async attachImageFromDataUrl_(note, imageDataUrl, cropRect) {
const tempDir = Setting.value('tempDir');
const mime = mimeUtils.fromDataUrl(imageDataUrl);
let ext = mimeUtils.toFileExtension(mime) || '';
if (ext) ext = '.' + ext;
const tempFilePath = tempDir + '/' + md5(Math.random() + '_' + Date.now()) + ext;
const imageConvOptions = {};
if (cropRect) imageConvOptions.cropRect = cropRect;
await shim.imageFromDataUrl(imageDataUrl, tempFilePath, imageConvOptions);
return await shim.attachFileToNote(note, tempFilePath);
}
async downloadImage_(url) {
const tempDir = Setting.value('tempDir');
const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0;
const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url);
let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase());
if (fileExt) fileExt = '.' + fileExt;
let imagePath = tempDir + '/' + safeFilename(name) + fileExt;
if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt;
try {
if (isDataUrl) {
await shim.imageFromDataUrl(url, imagePath);
} else {
await shim.fetchBlob(url, { path: imagePath });
}
return imagePath;
} catch (error) {
this.logger().warn('Cannot download image at ' + url, error);
return '';
}
}
async downloadImages_(urls) {
const PromisePool = require('es6-promise-pool')
const output = {};
let urlIndex = 0;
const promiseProducer = () => {
if (urlIndex >= urls.length) return null;
const url = urls[urlIndex++];
return new Promise(async (resolve, reject) => {
const imagePath = await this.downloadImage_(url);
if (imagePath) output[url] = { path: imagePath };
resolve();
});
}
const concurrency = 3
const pool = new PromisePool(promiseProducer, concurrency)
await pool.start()
return output;
}
async createResourcesFromPaths_(urls) {
for (let url in urls) {
if (!urls.hasOwnProperty(url)) continue;
const urlInfo = urls[url];
try {
const resource = await shim.createResourceFromPath(urlInfo.path);
urlInfo.resource = resource;
} catch (error) {
this.logger().warn('Cannot create resource for ' + url, error);
}
}
return urls;
}
async removeTempFiles_(urls) {
for (let url in urls) {
if (!urls.hasOwnProperty(url)) continue;
const urlInfo = urls[url];
try {
await shim.fsDriver().remove(urlInfo.path);
} catch (error) {
this.logger().warn('Cannot remove ' + urlInfo.path, error);
}
}
}
replaceImageUrlsByResources_(md, urls) {
let output = md.replace(/(!\[.*?\]\()([^\s\)]+)(.*?\))/g, (match, before, imageUrl, after) => {
const urlInfo = urls[imageUrl];
if (!urlInfo || !urlInfo.resource) return before + imageUrl + after;
const resourceUrl = Resource.internalUrl(urlInfo.resource);
return before + resourceUrl + after;
});
return output;
}
async findAvailablePort() {
const tcpPortUsed = require('tcp-port-used');
@@ -251,26 +110,33 @@ class ClipperServer {
response.end();
}
const requestId = Date.now();
this.logger().info('Request (' + requestId + '): ' + request.method + ' ' + request.url);
const writeResponse = (code, response) => {
if (typeof response === 'string') {
writeResponseText(code, response);
} else {
writeResponseJson(code, response);
}
}
this.logger().info('Request: ' + request.method + ' ' + request.url);
const url = urlParser.parse(request.url, true);
if (request.method === 'GET') {
if (url.pathname === '/ping') {
return writeResponseText(200, 'JoplinClipperServer');
const execRequest = async (request, body = '') => {
try {
const response = await this.api_.route(request.method, url.pathname, url.query, body);
writeResponse(200, response);
} catch (error) {
console.error(error);
writeResponse(error.httpCode ? error.httpCode : 500, error.message);
}
}
if (url.pathname === '/folders') {
const structure = await Folder.allAsTree({ fields: ['id', 'parent_id', 'title'] });
return writeResponseJson(200, structure);
}
if (url.pathname === '/tags') {
return writeResponseJson(200, await Tag.all({ fields: ['id', 'title'] }));
}
} else if (request.method === 'POST') {
if (url.pathname === '/notes') {
if (request.method === 'OPTIONS') {
writeCorsHeaders(200);
response.end();
} else {
if (request.method === 'POST') {
let body = '';
request.on('data', (data) => {
@@ -278,57 +144,14 @@ class ClipperServer {
});
request.on('end', async () => {
try {
const requestNote = JSON.parse(body);
let note = await this.requestNoteToNote(requestNote);
const imageUrls = markdownUtils.extractImageUrls(note.body);
this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length);
let result = await this.downloadImages_(imageUrls);
this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length);
result = await this.createResourcesFromPaths_(result);
await this.removeTempFiles_(result);
note.body = this.replaceImageUrlsByResources_(note.body, result);
this.logger().info('Request (' + requestId + '): Saving note...');
note = await Note.save(note);
if (requestNote.tags) {
const tagTitles = requestNote.tags.split(',');
await Tag.setNoteTagsByTitles(note.id, tagTitles);
}
if (requestNote.image_data_url) {
await this.attachImageFromDataUrl_(note, requestNote.image_data_url, requestNote.crop_rect);
}
this.logger().info('Request (' + requestId + '): Created note ' + note.id);
return writeResponseJson(200, note);
} catch (error) {
this.logger().error(error);
return writeResponseJson(400, { errorCode: 'exception', errorMessage: error.message });
}
execRequest(request, body);
});
} else {
return writeResponseJson(404, { errorCode: 'not_found' });
execRequest(request);
}
} else if (request.method === 'OPTIONS') {
writeCorsHeaders(200);
response.end();
} else {
return writeResponseJson(405, { errorCode: 'method_not_allowed' });
}
});
this.server_.on('close', () => {
});
enableServerDestroy(this.server_);
this.logger().info('Starting Clipper server on port ' + this.port_);

288
ReactNativeClient/lib/services/rest/Api.js Normal file
View File

@@ -0,0 +1,288 @@
const { ltrimSlashes } = require('lib/path-utils.js');
const Folder = require('lib/models/Folder');
const Note = require('lib/models/Note');
const Tag = require('lib/models/Tag');
const Setting = require('lib/models/Setting');
const markdownUtils = require('lib/markdownUtils');
const mimeUtils = require('lib/mime-utils.js').mime;
const { Logger } = require('lib/logger.js');
const md5 = require('md5');
const { shim } = require('lib/shim');
const HtmlToMd = require('lib/HtmlToMd');
const { fileExtension, safeFileExtension, safeFilename, filename } = require('lib/path-utils');
class ApiError extends Error {
constructor(message, httpCode = 400) {
super(message);
this.httpCode_ = httpCode;
}
get httpCode() {
return this.httpCode_;
}
}
class MethodNotAllowedError extends ApiError {
constructor() {
super('Method Not Allowed', 405);
}
}
class NotFoundError extends ApiError {
constructor() {
super('Not Found', 404);
}
}
class Api {
constructor() {
this.logger_ = new Logger();
}
async route(method, path, query = null, body = null) {
path = ltrimSlashes(path);
const callName = 'action_' + path;
if (!this[callName]) throw new NotFoundError();
try {
return this[callName]({
method: method,
query: query ? query : {},
body: body,
});
} catch (error) {
if (!error.httpCode) error.httpCode = 500;
throw error;
}
}
setLogger(l) {
this.logger_ = l;
}
logger() {
return this.logger_;
}
fields_(request, defaultFields) {
const query = request.query;
if (!query || !query.fields) return defaultFields;
const fields = query.fields.split(',').map(f => f.trim()).filter(f => !!f);
return fields.length ? fields : defaultFields;
}
async action_ping(request) {
if (request.method === 'GET') {
return 'JoplinClipperServer';
}
throw new MethodNotAllowedError();
}
async action_folders(request) {
if (request.method === 'GET') {
return await Folder.allAsTree({ fields: this.fields_(request, ['id', 'parent_id', 'title']) });
}
throw new MethodNotAllowedError();
}
async action_tags(request) {
if (request.method === 'GET') {
return await Tag.all({ fields: this.fields_(request, ['id', 'title']) })
}
throw new MethodNotAllowedError();
}
async action_notes(request) {
if (request.method === 'POST') {
const requestId = Date.now();
const requestNote = JSON.parse(request.body);
let note = await this.requestNoteToNote(requestNote);
const imageUrls = markdownUtils.extractImageUrls(note.body);
this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length);
let result = await this.downloadImages_(imageUrls);
this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length);
result = await this.createResourcesFromPaths_(result);
await this.removeTempFiles_(result);
note.body = this.replaceImageUrlsByResources_(note.body, result);
this.logger().info('Request (' + requestId + '): Saving note...');
note = await Note.save(note);
if (requestNote.tags) {
const tagTitles = requestNote.tags.split(',');
await Tag.setNoteTagsByTitles(note.id, tagTitles);
}
if (requestNote.image_data_url) {
note = await this.attachImageFromDataUrl_(note, requestNote.image_data_url, requestNote.crop_rect);
}
this.logger().info('Request (' + requestId + '): Created note ' + note.id);
return note;
}
throw new MethodNotAllowedError();
}
// ========================================================================================================================
// UTILIY FUNCTIONS
// ========================================================================================================================
htmlToMdParser() {
if (this.htmlToMdParser_) return this.htmlToMdParser_;
this.htmlToMdParser_ = new HtmlToMd();
return this.htmlToMdParser_;
}
async requestNoteToNote(requestNote) {
const output = {
title: requestNote.title ? requestNote.title : '',
body: requestNote.body ? requestNote.body : '',
};
if (requestNote.body_html) {
// Parsing will not work if the HTML is not wrapped in a top level tag, which is not guaranteed
// when getting the content from elsewhere. So here wrap it - it won't change anything to the final
// rendering but it makes sure everything will be parsed.
output.body = await this.htmlToMdParser().parse('<div>' + requestNote.body_html + '</div>', {
baseUrl: requestNote.base_url ? requestNote.base_url : '',
});
}
if (requestNote.parent_id) {
output.parent_id = requestNote.parent_id;
} else {
const folder = await Folder.defaultFolder();
if (!folder) throw new Error('Cannot find folder for note');
output.parent_id = folder.id;
}
if (requestNote.source_url) output.source_url = requestNote.source_url;
if (requestNote.author) output.author = requestNote.author;
return output;
}
// Note must have been saved first
async attachImageFromDataUrl_(note, imageDataUrl, cropRect) {
const tempDir = Setting.value('tempDir');
const mime = mimeUtils.fromDataUrl(imageDataUrl);
let ext = mimeUtils.toFileExtension(mime) || '';
if (ext) ext = '.' + ext;
const tempFilePath = tempDir + '/' + md5(Math.random() + '_' + Date.now()) + ext;
const imageConvOptions = {};
if (cropRect) imageConvOptions.cropRect = cropRect;
await shim.imageFromDataUrl(imageDataUrl, tempFilePath, imageConvOptions);
return await shim.attachFileToNote(note, tempFilePath);
}
async downloadImage_(url) {
const tempDir = Setting.value('tempDir');
const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0;
const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url);
let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase());
if (fileExt) fileExt = '.' + fileExt;
let imagePath = tempDir + '/' + safeFilename(name) + fileExt;
if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt;
try {
if (isDataUrl) {
await shim.imageFromDataUrl(url, imagePath);
} else {
await shim.fetchBlob(url, { path: imagePath });
}
return imagePath;
} catch (error) {
this.logger().warn('Cannot download image at ' + url, error);
return '';
}
}
async downloadImages_(urls) {
const PromisePool = require('es6-promise-pool')
const output = {};
let urlIndex = 0;
const promiseProducer = () => {
if (urlIndex >= urls.length) return null;
const url = urls[urlIndex++];
return new Promise(async (resolve, reject) => {
const imagePath = await this.downloadImage_(url);
if (imagePath) output[url] = { path: imagePath };
resolve();
});
}
const concurrency = 3
const pool = new PromisePool(promiseProducer, concurrency)
await pool.start()
return output;
}
async createResourcesFromPaths_(urls) {
for (let url in urls) {
if (!urls.hasOwnProperty(url)) continue;
const urlInfo = urls[url];
try {
const resource = await shim.createResourceFromPath(urlInfo.path);
urlInfo.resource = resource;
} catch (error) {
this.logger().warn('Cannot create resource for ' + url, error);
}
}
return urls;
}
async removeTempFiles_(urls) {
for (let url in urls) {
if (!urls.hasOwnProperty(url)) continue;
const urlInfo = urls[url];
try {
await shim.fsDriver().remove(urlInfo.path);
} catch (error) {
this.logger().warn('Cannot remove ' + urlInfo.path, error);
}
}
}
replaceImageUrlsByResources_(md, urls) {
let output = md.replace(/(!\[.*?\]\()([^\s\)]+)(.*?\))/g, (match, before, imageUrl, after) => {
const urlInfo = urls[imageUrl];
if (!urlInfo || !urlInfo.resource) return before + imageUrl + after;
const resourceUrl = Resource.internalUrl(urlInfo.resource);
return before + resourceUrl + after;
});
return output;
}
}
module.exports = Api;

6
ReactNativeClient/lib/shim-init-node.js
View File

@@ -186,7 +186,11 @@ function shimInit() {
const mime = mimeUtils.fromDataUrl(imageDataUrl);
await shim.writeImageToFile(image, mime, filePath);
} else {
throw new Error('Node support not implemented');
if (options.cropRect) throw new Error('Crop rect not supported in Node');
const imageDataURI = require('image-data-uri');
const result = imageDataURI.decode(imageDataUrl);
await shim.fsDriver().writeFile(filePath, result.dataBuffer, 'buffer');
}
}