You've already forked joplin
							
							
				mirror of
				https://github.com/laurent22/joplin.git
				synced 2025-10-31 00:07:48 +02:00 
			
		
		
		
	All: Refactored REST API to make it testable and to allow further extension
This commit is contained in:
		
							
								
								
									
										1
									
								
								CliClient/.gitignore
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										1
									
								
								CliClient/.gitignore
									
									
									
									
										vendored
									
									
								
							| @@ -13,6 +13,7 @@ tests/fuzzing.* | |||||||
| tests/fuzzing -* | tests/fuzzing -* | ||||||
| tests/logs/* | tests/logs/* | ||||||
| tests/cli-integration/ | tests/cli-integration/ | ||||||
|  | tests/tmp/ | ||||||
| *.mo | *.mo | ||||||
| *.*~ | *.*~ | ||||||
| tests/sync | tests/sync | ||||||
|   | |||||||
							
								
								
									
										874
									
								
								CliClient/package-lock.json
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										874
									
								
								CliClient/package-lock.json
									
									
									
										generated
									
									
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							| @@ -37,6 +37,7 @@ | |||||||
|     "fs-extra": "^5.0.0", |     "fs-extra": "^5.0.0", | ||||||
|     "html-entities": "^1.2.1", |     "html-entities": "^1.2.1", | ||||||
|     "html-minifier": "^3.5.15", |     "html-minifier": "^3.5.15", | ||||||
|  |     "image-data-uri": "^2.0.0", | ||||||
|     "image-type": "^3.0.0", |     "image-type": "^3.0.0", | ||||||
|     "joplin-turndown": "^4.0.8", |     "joplin-turndown": "^4.0.8", | ||||||
|     "joplin-turndown-plugin-gfm": "^1.0.7", |     "joplin-turndown-plugin-gfm": "^1.0.7", | ||||||
| @@ -57,7 +58,7 @@ | |||||||
|     "redux": "^3.7.2", |     "redux": "^3.7.2", | ||||||
|     "sax": "^1.2.2", |     "sax": "^1.2.2", | ||||||
|     "server-destroy": "^1.0.1", |     "server-destroy": "^1.0.1", | ||||||
|     "sharp": "^0.18.4", |     "sharp": "^0.20.8", | ||||||
|     "sprintf-js": "^1.1.1", |     "sprintf-js": "^1.1.1", | ||||||
|     "sqlite3": "^4.0.1", |     "sqlite3": "^4.0.1", | ||||||
|     "string-padding": "^1.0.2", |     "string-padding": "^1.0.2", | ||||||
|   | |||||||
							
								
								
									
										118
									
								
								CliClient/tests/services_rest_Api.js
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										118
									
								
								CliClient/tests/services_rest_Api.js
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,118 @@ | |||||||
|  | require('app-module-path').addPath(__dirname); | ||||||
|  |  | ||||||
|  | const { time } = require('lib/time-utils.js'); | ||||||
|  | const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); | ||||||
|  | const markdownUtils = require('lib/markdownUtils.js'); | ||||||
|  | const Api = require('lib/services/rest/Api'); | ||||||
|  | const Folder = require('lib/models/Folder'); | ||||||
|  | const Resource = require('lib/models/Resource'); | ||||||
|  |  | ||||||
|  | process.on('unhandledRejection', (reason, p) => { | ||||||
|  | 	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); | ||||||
|  | }); | ||||||
|  |  | ||||||
|  | let api = null; | ||||||
|  |  | ||||||
|  | describe('services_rest_Api', function() { | ||||||
|  |  | ||||||
|  | 	beforeEach(async (done) => { | ||||||
|  | 		api = new Api(); | ||||||
|  | 		await setupDatabaseAndSynchronizer(1); | ||||||
|  | 		await switchClient(1); | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should ping', async (done) => { | ||||||
|  | 		const response = await api.route('GET', 'ping'); | ||||||
|  | 		expect(response).toBe('JoplinClipperServer'); | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should handle Not Found errors', async (done) => { | ||||||
|  | 		const hasThrown = await checkThrowAsync(async () => await api.route('GET', 'pong')); | ||||||
|  | 		expect(hasThrown).toBe(true); | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should get folders', async (done) => { | ||||||
|  | 		let f1 = await Folder.save({ title: "mon carnet" }); | ||||||
|  | 		const response = await api.route('GET', 'folders'); | ||||||
|  | 		expect(response.length).toBe(1); | ||||||
|  | 		expect(response[0].title).toBe('mon carnet'); | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should create notes', async (done) => { | ||||||
|  | 		let response = null; | ||||||
|  | 		const f = await Folder.save({ title: "mon carnet" }); | ||||||
|  | 		 | ||||||
|  | 		response = await api.route('POST', 'notes', null, JSON.stringify({ | ||||||
|  | 			title: 'testing', | ||||||
|  | 			parent_id: f.id, | ||||||
|  | 		})); | ||||||
|  | 		expect(response.title).toBe('testing'); | ||||||
|  | 		expect(!!response.id).toBe(true); | ||||||
|  |  | ||||||
|  | 		response = await api.route('POST', 'notes', null, JSON.stringify({ | ||||||
|  | 			title: 'testing', | ||||||
|  | 			parent_id: f.id, | ||||||
|  | 		})); | ||||||
|  | 		expect(response.title).toBe('testing'); | ||||||
|  | 		expect(!!response.id).toBe(true); | ||||||
|  |  | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should create notes with images', async (done) => { | ||||||
|  | 		let response = null; | ||||||
|  | 		const f = await Folder.save({ title: "mon carnet" }); | ||||||
|  | 		 | ||||||
|  | 		response = await api.route('POST', 'notes', null, JSON.stringify({ | ||||||
|  | 			title: 'testing image', | ||||||
|  | 			parent_id: f.id, | ||||||
|  | 			image_data_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAANZJREFUeNoAyAA3/wFwtO3K6gUB/vz2+Prw9fj/+/r+/wBZKAAExOgF4/MC9ff+MRH6Ui4E+/0Bqc/zutj6AgT+/Pz7+vv7++nu82c4DlMqCvLs8goA/gL8/fz09fb59vXa6vzZ6vjT5fbn6voD/fwC8vX4UiT9Zi//APHyAP8ACgUBAPv5APz7BPj2+DIaC2o3E+3o6ywaC5fT6gD6/QD9/QEVf9kD+/dcLQgJA/7v8vqfwOf18wA1IAIEVycAyt//v9XvAPv7APz8LhoIAPz9Ri4OAgwARgx4W/6fVeEAAAAASUVORK5CYII=" | ||||||
|  | 		})); | ||||||
|  |  | ||||||
|  | 		const resources = await Resource.all(); | ||||||
|  | 		expect(resources.length).toBe(1); | ||||||
|  |  | ||||||
|  | 		const resource = resources[0]; | ||||||
|  | 		expect(response.body.indexOf(resource.id) >= 0).toBe(true); | ||||||
|  |  | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should create notes from HTML', async (done) => { | ||||||
|  | 		let response = null; | ||||||
|  | 		const f = await Folder.save({ title: "mon carnet" }); | ||||||
|  | 		 | ||||||
|  | 		response = await api.route('POST', 'notes', null, JSON.stringify({ | ||||||
|  | 			title: 'testing HTML', | ||||||
|  | 			parent_id: f.id, | ||||||
|  | 			body_html: '<b>Bold text</b>', | ||||||
|  | 		})); | ||||||
|  |  | ||||||
|  | 		expect(response.body).toBe('**Bold text**'); | ||||||
|  |  | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | 	it('should filter fields', async (done) => { | ||||||
|  | 		let f = api.fields_({ query: { fields: 'one,two' } }, []); | ||||||
|  | 		expect(f.length).toBe(2); | ||||||
|  | 		expect(f[0]).toBe('one'); | ||||||
|  | 		expect(f[1]).toBe('two'); | ||||||
|  |  | ||||||
|  | 		f = api.fields_({ query: { fields: 'one  ,, two  ' } }, []); | ||||||
|  | 		expect(f.length).toBe(2); | ||||||
|  | 		expect(f[0]).toBe('one'); | ||||||
|  | 		expect(f[1]).toBe('two'); | ||||||
|  |  | ||||||
|  | 		f = api.fields_({ query: { fields: '  ' } }, ['def']); | ||||||
|  | 		expect(f.length).toBe(1); | ||||||
|  | 		expect(f[0]).toBe('def'); | ||||||
|  |  | ||||||
|  | 		done(); | ||||||
|  | 	}); | ||||||
|  |  | ||||||
|  | }); | ||||||
| @@ -48,7 +48,9 @@ EncryptionService.fsDriver_ = fsDriver; | |||||||
| FileApiDriverLocal.fsDriver_ = fsDriver; | FileApiDriverLocal.fsDriver_ = fsDriver; | ||||||
|  |  | ||||||
| const logDir = __dirname + '/../tests/logs'; | const logDir = __dirname + '/../tests/logs'; | ||||||
|  | const tempDir = __dirname + '/../tests/tmp'; | ||||||
| fs.mkdirpSync(logDir, 0o755); | fs.mkdirpSync(logDir, 0o755); | ||||||
|  | fs.mkdirpSync(tempDir, 0o755); | ||||||
|  |  | ||||||
| SyncTargetRegistry.addClass(SyncTargetMemory); | SyncTargetRegistry.addClass(SyncTargetMemory); | ||||||
| SyncTargetRegistry.addClass(SyncTargetFilesystem); | SyncTargetRegistry.addClass(SyncTargetFilesystem); | ||||||
| @@ -80,6 +82,7 @@ BaseItem.loadClass('MasterKey', MasterKey); | |||||||
|  |  | ||||||
| Setting.setConstant('appId', 'net.cozic.joplin-cli'); | Setting.setConstant('appId', 'net.cozic.joplin-cli'); | ||||||
| Setting.setConstant('appType', 'cli'); | Setting.setConstant('appType', 'cli'); | ||||||
|  | Setting.setConstant('tempDir', tempDir); | ||||||
|  |  | ||||||
| BaseService.logger_ = logger; | BaseService.logger_ = logger; | ||||||
|  |  | ||||||
|   | |||||||
| @@ -1,19 +1,10 @@ | |||||||
| const { netUtils } = require('lib/net-utils'); | const { netUtils } = require('lib/net-utils'); | ||||||
| const urlParser = require("url"); | const urlParser = require("url"); | ||||||
| const Note = require('lib/models/Note'); |  | ||||||
| const Folder = require('lib/models/Folder'); |  | ||||||
| const Resource = require('lib/models/Resource'); |  | ||||||
| const Tag = require('lib/models/Tag'); |  | ||||||
| const Setting = require('lib/models/Setting'); | const Setting = require('lib/models/Setting'); | ||||||
| const { shim } = require('lib/shim'); |  | ||||||
| const md5 = require('md5'); |  | ||||||
| const { fileExtension, safeFileExtension, safeFilename, filename } = require('lib/path-utils'); |  | ||||||
| const HtmlToMd = require('lib/HtmlToMd'); |  | ||||||
| const { Logger } = require('lib/logger.js'); | const { Logger } = require('lib/logger.js'); | ||||||
| const markdownUtils = require('lib/markdownUtils'); |  | ||||||
| const mimeUtils = require('lib/mime-utils.js').mime; |  | ||||||
| const randomClipperPort = require('lib/randomClipperPort'); | const randomClipperPort = require('lib/randomClipperPort'); | ||||||
| const enableServerDestroy = require('server-destroy'); | const enableServerDestroy = require('server-destroy'); | ||||||
|  | const Api = require('lib/services/rest/Api'); | ||||||
|  |  | ||||||
| class ClipperServer { | class ClipperServer { | ||||||
|  |  | ||||||
| @@ -22,6 +13,7 @@ class ClipperServer { | |||||||
| 		this.startState_ = 'idle'; | 		this.startState_ = 'idle'; | ||||||
| 		this.server_ = null; | 		this.server_ = null; | ||||||
| 		this.port_ = null; | 		this.port_ = null; | ||||||
|  | 		this.api_ = new Api(); | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	static instance() { | 	static instance() { | ||||||
| @@ -32,6 +24,7 @@ class ClipperServer { | |||||||
|  |  | ||||||
| 	setLogger(l) { | 	setLogger(l) { | ||||||
| 		this.logger_ = l; | 		this.logger_ = l; | ||||||
|  | 		this.api_.setLogger(l); | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	logger() { | 	logger() { | ||||||
| @@ -65,140 +58,6 @@ class ClipperServer { | |||||||
| 		}); | 		}); | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	htmlToMdParser() { |  | ||||||
| 		if (this.htmlToMdParser_) return this.htmlToMdParser_; |  | ||||||
| 		this.htmlToMdParser_ = new HtmlToMd(); |  | ||||||
| 		return this.htmlToMdParser_; |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async requestNoteToNote(requestNote) { |  | ||||||
| 		const output = { |  | ||||||
| 			title: requestNote.title ? requestNote.title : '', |  | ||||||
| 			body: requestNote.body ? requestNote.body : '', |  | ||||||
| 		}; |  | ||||||
|  |  | ||||||
| 		if (requestNote.body_html) { |  | ||||||
| 			// Parsing will not work if the HTML is not wrapped in a top level tag, which is not guaranteed |  | ||||||
| 			// when getting the content from elsewhere. So here wrap it - it won't change anything to the final |  | ||||||
| 			// rendering but it makes sure everything will be parsed. |  | ||||||
| 			output.body = await this.htmlToMdParser().parse('<div>' + requestNote.body_html + '</div>', { |  | ||||||
| 				baseUrl: requestNote.base_url ? requestNote.base_url : '', |  | ||||||
| 			}); |  | ||||||
| 		} |  | ||||||
|  |  | ||||||
| 		if (requestNote.parent_id) { |  | ||||||
| 			output.parent_id = requestNote.parent_id; |  | ||||||
| 		} else { |  | ||||||
| 			const folder = await Folder.defaultFolder(); |  | ||||||
| 			if (!folder) throw new Error('Cannot find folder for note'); |  | ||||||
| 			output.parent_id = folder.id; |  | ||||||
| 		} |  | ||||||
|  |  | ||||||
| 		if (requestNote.source_url) output.source_url = requestNote.source_url; |  | ||||||
| 		if (requestNote.author) output.author = requestNote.author; |  | ||||||
|  |  | ||||||
| 		return output; |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	// Note must have been saved first |  | ||||||
| 	async attachImageFromDataUrl_(note, imageDataUrl, cropRect) { |  | ||||||
| 		const tempDir = Setting.value('tempDir'); |  | ||||||
| 		const mime = mimeUtils.fromDataUrl(imageDataUrl); |  | ||||||
| 		let ext = mimeUtils.toFileExtension(mime) || ''; |  | ||||||
| 		if (ext) ext = '.' + ext; |  | ||||||
| 		const tempFilePath = tempDir + '/' + md5(Math.random() + '_' + Date.now()) + ext; |  | ||||||
| 		const imageConvOptions = {}; |  | ||||||
| 		if (cropRect) imageConvOptions.cropRect = cropRect; |  | ||||||
| 		await shim.imageFromDataUrl(imageDataUrl, tempFilePath, imageConvOptions); |  | ||||||
| 		return await shim.attachFileToNote(note, tempFilePath); |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async downloadImage_(url) { |  | ||||||
| 		const tempDir = Setting.value('tempDir'); |  | ||||||
|  |  | ||||||
| 		const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0; |  | ||||||
|  |  | ||||||
| 		const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url); |  | ||||||
| 		let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase()); |  | ||||||
| 		if (fileExt) fileExt = '.' + fileExt; |  | ||||||
| 		let imagePath = tempDir + '/' + safeFilename(name) + fileExt; |  | ||||||
| 		if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt; |  | ||||||
|  |  | ||||||
| 		try { |  | ||||||
| 			if (isDataUrl) { |  | ||||||
| 				await shim.imageFromDataUrl(url, imagePath); |  | ||||||
| 			} else { |  | ||||||
| 				await shim.fetchBlob(url, { path: imagePath }); |  | ||||||
| 			} |  | ||||||
| 			return imagePath; |  | ||||||
| 		} catch (error) { |  | ||||||
| 			this.logger().warn('Cannot download image at ' + url, error); |  | ||||||
| 			return ''; |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async downloadImages_(urls) { |  | ||||||
| 		const PromisePool = require('es6-promise-pool') |  | ||||||
|  |  | ||||||
| 		const output = {}; |  | ||||||
|  |  | ||||||
| 		let urlIndex = 0; |  | ||||||
| 		const promiseProducer = () => { |  | ||||||
| 			if (urlIndex >= urls.length) return null; |  | ||||||
|  |  | ||||||
| 			const url = urls[urlIndex++]; |  | ||||||
|  |  | ||||||
| 			return new Promise(async (resolve, reject) => { |  | ||||||
| 				const imagePath = await this.downloadImage_(url); |  | ||||||
| 				if (imagePath) output[url] = { path: imagePath }; |  | ||||||
| 				resolve(); |  | ||||||
| 			}); |  | ||||||
| 		} |  | ||||||
|  |  | ||||||
| 		const concurrency = 3 |  | ||||||
| 		const pool = new PromisePool(promiseProducer, concurrency) |  | ||||||
| 		await pool.start() |  | ||||||
|  |  | ||||||
| 		return output; |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async createResourcesFromPaths_(urls) { |  | ||||||
| 		for (let url in urls) { |  | ||||||
| 			if (!urls.hasOwnProperty(url)) continue; |  | ||||||
| 			const urlInfo = urls[url]; |  | ||||||
| 			try { |  | ||||||
| 				const resource = await shim.createResourceFromPath(urlInfo.path); |  | ||||||
| 				urlInfo.resource = resource; |  | ||||||
| 			} catch (error) { |  | ||||||
| 				this.logger().warn('Cannot create resource for ' + url, error); |  | ||||||
| 			} |  | ||||||
| 		} |  | ||||||
| 		return urls; |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async removeTempFiles_(urls) { |  | ||||||
| 		for (let url in urls) { |  | ||||||
| 			if (!urls.hasOwnProperty(url)) continue; |  | ||||||
| 			const urlInfo = urls[url]; |  | ||||||
| 			try { |  | ||||||
| 				await shim.fsDriver().remove(urlInfo.path); |  | ||||||
| 			} catch (error) { |  | ||||||
| 				this.logger().warn('Cannot remove ' + urlInfo.path, error); |  | ||||||
| 			} |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	replaceImageUrlsByResources_(md, urls) { |  | ||||||
| 		let output = md.replace(/(!\[.*?\]\()([^\s\)]+)(.*?\))/g, (match, before, imageUrl, after) => { |  | ||||||
| 			const urlInfo = urls[imageUrl]; |  | ||||||
| 			if (!urlInfo || !urlInfo.resource) return before + imageUrl + after; |  | ||||||
| 			const resourceUrl = Resource.internalUrl(urlInfo.resource); |  | ||||||
| 			return before + resourceUrl + after; |  | ||||||
| 		}); |  | ||||||
|  |  | ||||||
| 		return output; |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	async findAvailablePort() { | 	async findAvailablePort() { | ||||||
| 		const tcpPortUsed = require('tcp-port-used'); | 		const tcpPortUsed = require('tcp-port-used'); | ||||||
|  |  | ||||||
| @@ -251,26 +110,33 @@ class ClipperServer { | |||||||
| 				response.end(); | 				response.end(); | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
| 			const requestId = Date.now(); | 			const writeResponse = (code, response) => { | ||||||
| 			this.logger().info('Request (' + requestId + '): ' + request.method + ' ' + request.url); | 				if (typeof response === 'string') { | ||||||
|  | 					writeResponseText(code, response); | ||||||
|  | 				} else { | ||||||
|  | 					writeResponseJson(code, response); | ||||||
|  | 				} | ||||||
|  | 			} | ||||||
|  |  | ||||||
|  | 			this.logger().info('Request: ' + request.method + ' ' + request.url); | ||||||
|  |  | ||||||
| 			const url = urlParser.parse(request.url, true); | 			const url = urlParser.parse(request.url, true); | ||||||
|  |  | ||||||
| 			if (request.method === 'GET') { | 			const execRequest = async (request, body = '') => { | ||||||
| 				if (url.pathname === '/ping') { | 				try { | ||||||
| 					return writeResponseText(200, 'JoplinClipperServer'); | 					const response = await this.api_.route(request.method, url.pathname, url.query, body); | ||||||
|  | 					writeResponse(200, response); | ||||||
|  | 				} catch (error) { | ||||||
|  | 					console.error(error); | ||||||
|  | 					writeResponse(error.httpCode ? error.httpCode : 500, error.message); | ||||||
| 				} | 				} | ||||||
|  | 			} | ||||||
|  |  | ||||||
| 				if (url.pathname === '/folders') { | 			if (request.method === 'OPTIONS') { | ||||||
| 					const structure = await Folder.allAsTree({ fields: ['id', 'parent_id', 'title'] }); | 				writeCorsHeaders(200); | ||||||
| 					return writeResponseJson(200, structure); | 				response.end(); | ||||||
| 				} | 			} else { | ||||||
|  | 				if (request.method === 'POST') { | ||||||
| 				if (url.pathname === '/tags') { |  | ||||||
| 					return writeResponseJson(200, await Tag.all({ fields: ['id', 'title'] })); |  | ||||||
| 				} |  | ||||||
| 			} else if (request.method === 'POST') { |  | ||||||
| 				if (url.pathname === '/notes') { |  | ||||||
| 					let body = ''; | 					let body = ''; | ||||||
|  |  | ||||||
| 					request.on('data', (data) => { | 					request.on('data', (data) => { | ||||||
| @@ -278,57 +144,14 @@ class ClipperServer { | |||||||
| 					}); | 					}); | ||||||
|  |  | ||||||
| 					request.on('end', async () => { | 					request.on('end', async () => { | ||||||
| 						try { | 						execRequest(request, body); | ||||||
| 							const requestNote = JSON.parse(body); |  | ||||||
| 							let note = await this.requestNoteToNote(requestNote); |  | ||||||
|  |  | ||||||
| 							const imageUrls = markdownUtils.extractImageUrls(note.body); |  | ||||||
|  |  | ||||||
| 							this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length); |  | ||||||
|  |  | ||||||
| 							let result = await this.downloadImages_(imageUrls); |  | ||||||
|  |  | ||||||
| 							this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length); |  | ||||||
|  |  | ||||||
| 							result = await this.createResourcesFromPaths_(result); |  | ||||||
| 							await this.removeTempFiles_(result); |  | ||||||
| 							note.body = this.replaceImageUrlsByResources_(note.body, result); |  | ||||||
|  |  | ||||||
| 							this.logger().info('Request (' + requestId + '): Saving note...'); |  | ||||||
|  |  | ||||||
| 							note = await Note.save(note); |  | ||||||
|  |  | ||||||
| 							if (requestNote.tags) { |  | ||||||
| 								const tagTitles = requestNote.tags.split(','); |  | ||||||
| 								await Tag.setNoteTagsByTitles(note.id, tagTitles); |  | ||||||
| 							} |  | ||||||
|  |  | ||||||
| 							if (requestNote.image_data_url) { |  | ||||||
| 								await this.attachImageFromDataUrl_(note, requestNote.image_data_url, requestNote.crop_rect); |  | ||||||
| 							} |  | ||||||
|  |  | ||||||
| 							this.logger().info('Request (' + requestId + '): Created note ' + note.id); |  | ||||||
| 							return writeResponseJson(200, note); |  | ||||||
| 						} catch (error) { |  | ||||||
| 							this.logger().error(error); |  | ||||||
| 							return writeResponseJson(400, { errorCode: 'exception', errorMessage: error.message }); |  | ||||||
| 						} |  | ||||||
| 					}); | 					}); | ||||||
| 				} else { | 				} else { | ||||||
| 					return writeResponseJson(404, { errorCode: 'not_found' }); | 					execRequest(request); | ||||||
| 				} | 				} | ||||||
| 			} else if (request.method === 'OPTIONS') { |  | ||||||
| 				writeCorsHeaders(200); |  | ||||||
| 				response.end(); |  | ||||||
| 			} else { |  | ||||||
| 				return writeResponseJson(405, { errorCode: 'method_not_allowed' }); |  | ||||||
| 			} | 			} | ||||||
| 		}); | 		}); | ||||||
|  |  | ||||||
| 		this.server_.on('close', () => { |  | ||||||
|  |  | ||||||
| 		}); |  | ||||||
|  |  | ||||||
| 		enableServerDestroy(this.server_); | 		enableServerDestroy(this.server_); | ||||||
|  |  | ||||||
| 		this.logger().info('Starting Clipper server on port ' + this.port_); | 		this.logger().info('Starting Clipper server on port ' + this.port_); | ||||||
|   | |||||||
							
								
								
									
										288
									
								
								ReactNativeClient/lib/services/rest/Api.js
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										288
									
								
								ReactNativeClient/lib/services/rest/Api.js
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,288 @@ | |||||||
|  | const { ltrimSlashes } = require('lib/path-utils.js'); | ||||||
|  | const Folder = require('lib/models/Folder'); | ||||||
|  | const Note = require('lib/models/Note'); | ||||||
|  | const Tag = require('lib/models/Tag'); | ||||||
|  | const Setting = require('lib/models/Setting'); | ||||||
|  | const markdownUtils = require('lib/markdownUtils'); | ||||||
|  | const mimeUtils = require('lib/mime-utils.js').mime; | ||||||
|  | const { Logger } = require('lib/logger.js'); | ||||||
|  | const md5 = require('md5'); | ||||||
|  | const { shim } = require('lib/shim'); | ||||||
|  | const HtmlToMd = require('lib/HtmlToMd'); | ||||||
|  | const { fileExtension, safeFileExtension, safeFilename, filename } = require('lib/path-utils'); | ||||||
|  |  | ||||||
|  | class ApiError extends Error { | ||||||
|  |  | ||||||
|  | 	constructor(message, httpCode = 400) { | ||||||
|  | 		super(message); | ||||||
|  | 		this.httpCode_ = httpCode; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	get httpCode() { | ||||||
|  | 		return this.httpCode_; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | } | ||||||
|  |  | ||||||
|  | class MethodNotAllowedError extends ApiError { | ||||||
|  |  | ||||||
|  | 	constructor() { | ||||||
|  | 		super('Method Not Allowed', 405); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | } | ||||||
|  |  | ||||||
|  | class NotFoundError extends ApiError { | ||||||
|  |  | ||||||
|  | 	constructor() { | ||||||
|  | 		super('Not Found', 404); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | } | ||||||
|  |  | ||||||
|  | class Api { | ||||||
|  |  | ||||||
|  | 	constructor() { | ||||||
|  | 		this.logger_ = new Logger(); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async route(method, path, query = null, body = null) { | ||||||
|  | 		path = ltrimSlashes(path); | ||||||
|  | 		const callName = 'action_' + path; | ||||||
|  | 		if (!this[callName]) throw new NotFoundError(); | ||||||
|  |  | ||||||
|  | 		try { | ||||||
|  | 			return this[callName]({ | ||||||
|  | 				method: method, | ||||||
|  | 				query: query ? query : {}, | ||||||
|  | 				body: body, | ||||||
|  | 			}); | ||||||
|  | 		} catch (error) { | ||||||
|  | 			if (!error.httpCode) error.httpCode = 500; | ||||||
|  | 			throw error; | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	setLogger(l) { | ||||||
|  | 		this.logger_ = l; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	logger() { | ||||||
|  | 		return this.logger_; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	fields_(request, defaultFields) { | ||||||
|  | 		const query = request.query; | ||||||
|  | 		if (!query || !query.fields) return defaultFields; | ||||||
|  | 		const fields = query.fields.split(',').map(f => f.trim()).filter(f => !!f); | ||||||
|  | 		return fields.length ? fields : defaultFields; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async action_ping(request) { | ||||||
|  | 		if (request.method === 'GET') { | ||||||
|  | 			return 'JoplinClipperServer'; | ||||||
|  | 		} | ||||||
|  | 		throw new MethodNotAllowedError(); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async action_folders(request) { | ||||||
|  | 		if (request.method === 'GET') { | ||||||
|  | 			return await Folder.allAsTree({ fields: this.fields_(request, ['id', 'parent_id', 'title']) }); | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		throw new MethodNotAllowedError(); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async action_tags(request) { | ||||||
|  | 		if (request.method === 'GET') { | ||||||
|  | 			return await Tag.all({ fields: this.fields_(request, ['id', 'title']) }) | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		throw new MethodNotAllowedError(); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async action_notes(request) { | ||||||
|  | 		if (request.method === 'POST') { | ||||||
|  | 			const requestId = Date.now(); | ||||||
|  | 			const requestNote = JSON.parse(request.body); | ||||||
|  | 			let note = await this.requestNoteToNote(requestNote); | ||||||
|  |  | ||||||
|  | 			const imageUrls = markdownUtils.extractImageUrls(note.body); | ||||||
|  |  | ||||||
|  | 			this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length); | ||||||
|  |  | ||||||
|  | 			let result = await this.downloadImages_(imageUrls); | ||||||
|  |  | ||||||
|  | 			this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length); | ||||||
|  |  | ||||||
|  | 			result = await this.createResourcesFromPaths_(result); | ||||||
|  | 			await this.removeTempFiles_(result); | ||||||
|  | 			note.body = this.replaceImageUrlsByResources_(note.body, result); | ||||||
|  |  | ||||||
|  | 			this.logger().info('Request (' + requestId + '): Saving note...'); | ||||||
|  |  | ||||||
|  | 			note = await Note.save(note); | ||||||
|  |  | ||||||
|  | 			if (requestNote.tags) { | ||||||
|  | 				const tagTitles = requestNote.tags.split(','); | ||||||
|  | 				await Tag.setNoteTagsByTitles(note.id, tagTitles); | ||||||
|  | 			} | ||||||
|  |  | ||||||
|  | 			if (requestNote.image_data_url) { | ||||||
|  | 				note = await this.attachImageFromDataUrl_(note, requestNote.image_data_url, requestNote.crop_rect); | ||||||
|  | 			} | ||||||
|  |  | ||||||
|  | 			this.logger().info('Request (' + requestId + '): Created note ' + note.id); | ||||||
|  | 			 | ||||||
|  | 			return note; | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		throw new MethodNotAllowedError(); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  |  | ||||||
|  |  | ||||||
|  |  | ||||||
|  |  | ||||||
|  | 	// ======================================================================================================================== | ||||||
|  | 	// UTILIY FUNCTIONS | ||||||
|  | 	// ======================================================================================================================== | ||||||
|  |  | ||||||
|  | 	htmlToMdParser() { | ||||||
|  | 		if (this.htmlToMdParser_) return this.htmlToMdParser_; | ||||||
|  | 		this.htmlToMdParser_ = new HtmlToMd(); | ||||||
|  | 		return this.htmlToMdParser_; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async requestNoteToNote(requestNote) { | ||||||
|  | 		const output = { | ||||||
|  | 			title: requestNote.title ? requestNote.title : '', | ||||||
|  | 			body: requestNote.body ? requestNote.body : '', | ||||||
|  | 		}; | ||||||
|  |  | ||||||
|  | 		if (requestNote.body_html) { | ||||||
|  | 			// Parsing will not work if the HTML is not wrapped in a top level tag, which is not guaranteed | ||||||
|  | 			// when getting the content from elsewhere. So here wrap it - it won't change anything to the final | ||||||
|  | 			// rendering but it makes sure everything will be parsed. | ||||||
|  | 			output.body = await this.htmlToMdParser().parse('<div>' + requestNote.body_html + '</div>', { | ||||||
|  | 				baseUrl: requestNote.base_url ? requestNote.base_url : '', | ||||||
|  | 			}); | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		if (requestNote.parent_id) { | ||||||
|  | 			output.parent_id = requestNote.parent_id; | ||||||
|  | 		} else { | ||||||
|  | 			const folder = await Folder.defaultFolder(); | ||||||
|  | 			if (!folder) throw new Error('Cannot find folder for note'); | ||||||
|  | 			output.parent_id = folder.id; | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		if (requestNote.source_url) output.source_url = requestNote.source_url; | ||||||
|  | 		if (requestNote.author) output.author = requestNote.author; | ||||||
|  |  | ||||||
|  | 		return output; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	// Note must have been saved first | ||||||
|  | 	async attachImageFromDataUrl_(note, imageDataUrl, cropRect) { | ||||||
|  | 		const tempDir = Setting.value('tempDir'); | ||||||
|  | 		const mime = mimeUtils.fromDataUrl(imageDataUrl); | ||||||
|  | 		let ext = mimeUtils.toFileExtension(mime) || ''; | ||||||
|  | 		if (ext) ext = '.' + ext; | ||||||
|  | 		const tempFilePath = tempDir + '/' + md5(Math.random() + '_' + Date.now()) + ext; | ||||||
|  | 		const imageConvOptions = {}; | ||||||
|  | 		if (cropRect) imageConvOptions.cropRect = cropRect; | ||||||
|  | 		await shim.imageFromDataUrl(imageDataUrl, tempFilePath, imageConvOptions); | ||||||
|  | 		return await shim.attachFileToNote(note, tempFilePath); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async downloadImage_(url) { | ||||||
|  | 		const tempDir = Setting.value('tempDir'); | ||||||
|  |  | ||||||
|  | 		const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0; | ||||||
|  |  | ||||||
|  | 		const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url); | ||||||
|  | 		let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase()); | ||||||
|  | 		if (fileExt) fileExt = '.' + fileExt; | ||||||
|  | 		let imagePath = tempDir + '/' + safeFilename(name) + fileExt; | ||||||
|  | 		if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt; | ||||||
|  |  | ||||||
|  | 		try { | ||||||
|  | 			if (isDataUrl) { | ||||||
|  | 				await shim.imageFromDataUrl(url, imagePath); | ||||||
|  | 			} else { | ||||||
|  | 				await shim.fetchBlob(url, { path: imagePath }); | ||||||
|  | 			} | ||||||
|  | 			return imagePath; | ||||||
|  | 		} catch (error) { | ||||||
|  | 			this.logger().warn('Cannot download image at ' + url, error); | ||||||
|  | 			return ''; | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async downloadImages_(urls) { | ||||||
|  | 		const PromisePool = require('es6-promise-pool') | ||||||
|  |  | ||||||
|  | 		const output = {}; | ||||||
|  |  | ||||||
|  | 		let urlIndex = 0; | ||||||
|  | 		const promiseProducer = () => { | ||||||
|  | 			if (urlIndex >= urls.length) return null; | ||||||
|  |  | ||||||
|  | 			const url = urls[urlIndex++]; | ||||||
|  |  | ||||||
|  | 			return new Promise(async (resolve, reject) => { | ||||||
|  | 				const imagePath = await this.downloadImage_(url); | ||||||
|  | 				if (imagePath) output[url] = { path: imagePath }; | ||||||
|  | 				resolve(); | ||||||
|  | 			}); | ||||||
|  | 		} | ||||||
|  |  | ||||||
|  | 		const concurrency = 3 | ||||||
|  | 		const pool = new PromisePool(promiseProducer, concurrency) | ||||||
|  | 		await pool.start() | ||||||
|  |  | ||||||
|  | 		return output; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async createResourcesFromPaths_(urls) { | ||||||
|  | 		for (let url in urls) { | ||||||
|  | 			if (!urls.hasOwnProperty(url)) continue; | ||||||
|  | 			const urlInfo = urls[url]; | ||||||
|  | 			try { | ||||||
|  | 				const resource = await shim.createResourceFromPath(urlInfo.path); | ||||||
|  | 				urlInfo.resource = resource; | ||||||
|  | 			} catch (error) { | ||||||
|  | 				this.logger().warn('Cannot create resource for ' + url, error); | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 		return urls; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	async removeTempFiles_(urls) { | ||||||
|  | 		for (let url in urls) { | ||||||
|  | 			if (!urls.hasOwnProperty(url)) continue; | ||||||
|  | 			const urlInfo = urls[url]; | ||||||
|  | 			try { | ||||||
|  | 				await shim.fsDriver().remove(urlInfo.path); | ||||||
|  | 			} catch (error) { | ||||||
|  | 				this.logger().warn('Cannot remove ' + urlInfo.path, error); | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	replaceImageUrlsByResources_(md, urls) { | ||||||
|  | 		let output = md.replace(/(!\[.*?\]\()([^\s\)]+)(.*?\))/g, (match, before, imageUrl, after) => { | ||||||
|  | 			const urlInfo = urls[imageUrl]; | ||||||
|  | 			if (!urlInfo || !urlInfo.resource) return before + imageUrl + after; | ||||||
|  | 			const resourceUrl = Resource.internalUrl(urlInfo.resource); | ||||||
|  | 			return before + resourceUrl + after; | ||||||
|  | 		}); | ||||||
|  |  | ||||||
|  | 		return output; | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  |  | ||||||
|  | } | ||||||
|  |  | ||||||
|  | module.exports = Api; | ||||||
| @@ -186,7 +186,11 @@ function shimInit() { | |||||||
| 			const mime = mimeUtils.fromDataUrl(imageDataUrl); | 			const mime = mimeUtils.fromDataUrl(imageDataUrl); | ||||||
| 			await shim.writeImageToFile(image, mime, filePath); | 			await shim.writeImageToFile(image, mime, filePath); | ||||||
| 		} else { | 		} else { | ||||||
| 			throw new Error('Node support not implemented'); | 			if (options.cropRect) throw new Error('Crop rect not supported in Node'); | ||||||
|  |  | ||||||
|  | 			const imageDataURI = require('image-data-uri'); | ||||||
|  | 			const result = imageDataURI.decode(imageDataUrl); | ||||||
|  | 			await shim.fsDriver().writeFile(filePath, result.dataBuffer, 'buffer');	 | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user