From a8da469523956228c4a20cb07362baa79045b256 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 23 May 2018 14:25:59 +0100 Subject: [PATCH] Clipper: Improved download of images and conversion to resources --- CliClient/package-lock.json | 27 +++ CliClient/package.json | 3 + .../tests/html_to_md/anchor_with_js.html | 1 + CliClient/tests/html_to_md/anchor_with_js.md | 1 + CliClient/tests/markdownUtils.js | 17 ++ Clipper/joplin-webclipper/popup/src/App.js | 2 +- ElectronClient/app/package-lock.json | 158 ++++++++++-------- ElectronClient/app/package.json | 5 +- ReactNativeClient/lib/BaseApplication.js | 15 +- ReactNativeClient/lib/ClipperServer.js | 101 ++++++----- ReactNativeClient/lib/logger.js | 2 +- ReactNativeClient/lib/markdownUtils.js | 13 ++ ReactNativeClient/lib/path-utils.js | 8 +- ReactNativeClient/lib/shim-init-node.js | 49 ++---- 14 files changed, 255 insertions(+), 147 deletions(-) create mode 100644 CliClient/tests/html_to_md/anchor_with_js.html create mode 100644 CliClient/tests/html_to_md/anchor_with_js.md diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index 7b111f11b..d561022fd 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -457,6 +457,11 @@ "iconv-lite": "~0.4.13" } }, + "es6-promise-pool": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz", + "integrity": "sha1-FHxhKza0fxBQJ/nSv1SlmKmdnMs=" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -536,6 +541,11 @@ "format": "^0.2.2" } }, + "file-type": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz", + "integrity": "sha1-G2AOX8ofvcboDApwxxyNul95BsU=" + }, "follow-redirects": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.2.5.tgz", @@ -756,6 +766,14 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" }, + "image-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/image-type/-/image-type-3.0.0.tgz", + "integrity": "sha1-FQKvMTX5BuEiyHfDHpSve3qRRsU=", + "requires": { + "file-type": "^4.1.0" + } + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1361,6 +1379,15 @@ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-1.0.0.tgz", "integrity": "sha1-YoYkIRLFtxL6ZU5SZlK/ahP/Bcs=" }, + "read-chunk": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz", + "integrity": "sha1-agTAkoAF7Z1C4aasVgDhnLx/9lU=", + "requires": { + "pify": "^3.0.0", + "safe-buffer": "^5.1.1" + } + }, "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index be7399b28..f218ec7fb 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -31,11 +31,13 @@ "async-mutex": "^0.1.3", "base-64": "^0.1.0", "compare-version": "^0.1.2", + "es6-promise-pool": "^2.5.0", "follow-redirects": "^1.2.4", "form-data": "^2.1.4", "fs-extra": "^5.0.0", "html-entities": "^1.2.1", "html-minifier": "^3.5.15", + "image-type": "^3.0.0", "joplin-turndown": "^4.0.3", "joplin-turndown-plugin-gfm": "^1.0.2", "jssha": "^2.3.0", @@ -50,6 +52,7 @@ "promise": "^7.1.1", "proper-lockfile": "^2.0.1", "query-string": "4.3.4", + "read-chunk": "^2.1.0", "redux": "^3.7.2", "sax": "^1.2.2", "server-destroy": "^1.0.1", diff --git a/CliClient/tests/html_to_md/anchor_with_js.html b/CliClient/tests/html_to_md/anchor_with_js.html new file mode 100644 index 000000000..da25e3f33 --- /dev/null +++ b/CliClient/tests/html_to_md/anchor_with_js.html @@ -0,0 +1 @@ +Some text \ No newline at end of file diff --git a/CliClient/tests/html_to_md/anchor_with_js.md b/CliClient/tests/html_to_md/anchor_with_js.md new file mode 100644 index 000000000..a1584d3d7 --- /dev/null +++ b/CliClient/tests/html_to_md/anchor_with_js.md @@ -0,0 +1 @@ +[Some text]() \ No newline at end of file diff --git a/CliClient/tests/markdownUtils.js b/CliClient/tests/markdownUtils.js index 32f944183..5a68ed6e2 100644 --- a/CliClient/tests/markdownUtils.js +++ b/CliClient/tests/markdownUtils.js @@ -34,4 +34,21 @@ describe('markdownUtils', function() { done(); }); + it('should extract image URLs', async (done) => { + const testCases = [ + ['![something](http://test.com/img.png)', ['http://test.com/img.png']], + ['![something](http://test.com/img.png) ![something2](http://test.com/img2.png)', ['http://test.com/img.png', 'http://test.com/img2.png']], + ['![something](http://test.com/img.png "Some description")', ['http://test.com/img.png']], + ]; + + for (let i = 0; i < testCases.length; i++) { + const md = testCases[i][0]; + const expected = testCases[i][1]; + + expect(markdownUtils.extractImageUrls(md).join('')).toBe(expected.join('')); + } + + done(); + }); + }); \ No newline at end of file diff --git a/Clipper/joplin-webclipper/popup/src/App.js b/Clipper/joplin-webclipper/popup/src/App.js index 6db6920ec..35241547e 100644 --- a/Clipper/joplin-webclipper/popup/src/App.js +++ b/Clipper/joplin-webclipper/popup/src/App.js @@ -67,7 +67,7 @@ class AppComponent extends Component { let msg = ''; if (operation.uploading) { - msg = 'Sending to Joplin...'; + msg = 'Processing note... The note will be available in Joplin as soon as the web page and images have been downloaded and converted. In the meantime you may close this popup.'; } else if (operation.success) { msg = 'Note was successfully created!'; } else { diff --git a/ElectronClient/app/package-lock.json b/ElectronClient/app/package-lock.json index da624707b..167f6f810 100644 --- a/ElectronClient/app/package-lock.json +++ b/ElectronClient/app/package-lock.json @@ -2256,6 +2256,11 @@ "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.1.1.tgz", "integrity": "sha512-OaU1hHjgJf+b0NzsxCg7NdIYERD6Hy/PEmFLTjw+b65scuisG3Kt4QoTvJ66BBkPZ581gr0kpoVzKnxniM8nng==" }, + "es6-promise-pool": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz", + "integrity": "sha1-FHxhKza0fxBQJ/nSv1SlmKmdnMs=" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -2446,6 +2451,11 @@ "pend": "1.2.0" } }, + "file-type": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz", + "integrity": "sha1-G2AOX8ofvcboDApwxxyNul95BsU=" + }, "filename-regex": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz", @@ -2612,8 +2622,8 @@ "dev": true, "optional": true, "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" + "delegates": "1.0.0", + "readable-stream": "2.2.9" } }, "asn1": { @@ -2834,10 +2844,10 @@ "bundled": true, "dev": true, "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" + "graceful-fs": "4.1.11", + "inherits": "2.0.3", + "mkdirp": "0.5.1", + "rimraf": "2.6.1" } }, "fstream-ignore": { @@ -2857,14 +2867,14 @@ "dev": true, "optional": true, "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" + "aproba": "1.1.1", + "console-control-strings": "1.1.0", + "has-unicode": "2.0.1", + "object-assign": "4.1.1", + "signal-exit": "3.0.2", + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wide-align": "1.1.2" } }, "getpass": { @@ -2889,12 +2899,12 @@ "bundled": true, "dev": true, "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" } }, "graceful-fs": { @@ -2946,9 +2956,9 @@ "dev": true, "optional": true, "requires": { - "assert-plus": "^0.2.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" + "assert-plus": "0.2.0", + "jsprim": "1.4.0", + "sshpk": "1.13.0" } }, "inflight": { @@ -2956,8 +2966,8 @@ "bundled": true, "dev": true, "requires": { - "once": "^1.3.0", - "wrappy": "1" + "once": "1.4.0", + "wrappy": "1.0.2" } }, "inherits": { @@ -3132,10 +3142,10 @@ "dev": true, "optional": true, "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" + "are-we-there-yet": "1.1.4", + "console-control-strings": "1.1.0", + "gauge": "2.7.4", + "set-blocking": "2.0.0" } }, "number-is-nan": { @@ -3160,7 +3170,7 @@ "bundled": true, "dev": true, "requires": { - "wrappy": "1" + "wrappy": "1.0.2" } }, "os-homedir": { @@ -3238,13 +3248,13 @@ "bundled": true, "dev": true, "requires": { - "buffer-shims": "~1.0.0", - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "~1.0.0", - "process-nextick-args": "~1.0.6", - "string_decoder": "~1.0.0", - "util-deprecate": "~1.0.1" + "buffer-shims": "1.0.0", + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "1.0.7", + "string_decoder": "1.0.1", + "util-deprecate": "1.0.2" } }, "request": { @@ -3253,28 +3263,28 @@ "dev": true, "optional": true, "requires": { - "aws-sign2": "~0.6.0", - "aws4": "^1.2.1", - "caseless": "~0.12.0", - "combined-stream": "~1.0.5", - "extend": "~3.0.0", - "forever-agent": "~0.6.1", - "form-data": "~2.1.1", - "har-validator": "~4.2.1", - "hawk": "~3.1.3", - "http-signature": "~1.1.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.7", - "oauth-sign": "~0.8.1", - "performance-now": "^0.2.0", - "qs": "~6.4.0", - "safe-buffer": "^5.0.1", - "stringstream": "~0.0.4", - "tough-cookie": "~2.3.0", - "tunnel-agent": "^0.6.0", - "uuid": "^3.0.0" + "aws-sign2": "0.6.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.1.4", + "har-validator": "4.2.1", + "hawk": "3.1.3", + "http-signature": "1.1.1", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.15", + "oauth-sign": "0.8.2", + "performance-now": "0.2.0", + "qs": "6.4.0", + "safe-buffer": "5.0.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.2", + "tunnel-agent": "0.6.0", + "uuid": "3.0.1" } }, "rimraf": { @@ -3282,7 +3292,7 @@ "bundled": true, "dev": true, "requires": { - "glob": "^7.0.5" + "glob": "7.1.2" } }, "safe-buffer": { @@ -3461,7 +3471,7 @@ "dev": true, "optional": true, "requires": { - "string-width": "^1.0.2" + "string-width": "1.0.2" } }, "wrappy": { @@ -3726,6 +3736,14 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" }, + "image-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/image-type/-/image-type-3.0.0.tgz", + "integrity": "sha1-FQKvMTX5BuEiyHfDHpSve3qRRsU=", + "requires": { + "file-type": "4.4.0" + } + }, "import-lazy": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", @@ -4009,9 +4027,9 @@ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "joplin-turndown": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.3.tgz", - "integrity": "sha512-WbAXje8wq4/ZLNtPDUFBEtG5zKEbz7Wth5N3vB4Nw7k+PUs3mMF49LVEPP7Kc6H4Ui671qdjpSShvdsmiLY2gA==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.4.tgz", + "integrity": "sha512-Qgi9DvLGT2r86yiQjKO83tvGYF9FabjVSnP6S9ts/+jaVWvwmBGhcGklFfMArlxY3doKZmdIPspiESsLcpn2Jg==", "requires": { "jsdom": "11.10.0" } @@ -4860,8 +4878,7 @@ "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" }, "pinkie": { "version": "2.0.4", @@ -5150,6 +5167,15 @@ "prop-types": "15.6.0" } }, + "read-chunk": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz", + "integrity": "sha1-agTAkoAF7Z1C4aasVgDhnLx/9lU=", + "requires": { + "pify": "3.0.0", + "safe-buffer": "5.1.1" + } + }, "read-config-file": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-config-file/-/read-config-file-3.0.0.tgz", diff --git a/ElectronClient/app/package.json b/ElectronClient/app/package.json index 667feb744..bae24ec7f 100644 --- a/ElectronClient/app/package.json +++ b/ElectronClient/app/package.json @@ -82,12 +82,14 @@ "electron-context-menu": "^0.9.1", "electron-is-dev": "^0.3.0", "electron-window-state": "^4.1.1", + "es6-promise-pool": "^2.5.0", "follow-redirects": "^1.2.5", "form-data": "^2.3.1", "fs-extra": "^5.0.0", "highlight.js": "^9.12.0", "html-entities": "^1.2.1", - "joplin-turndown": "^4.0.3", + "image-type": "^3.0.0", + "joplin-turndown": "^4.0.4", "joplin-turndown-plugin-gfm": "^1.0.5", "jssha": "^2.3.1", "katex": "^0.9.0-beta1", @@ -109,6 +111,7 @@ "react-datetime": "^2.11.0", "react-dom": "^16.0.0", "react-redux": "^5.0.6", + "read-chunk": "^2.1.0", "readability-node": "^0.1.0", "redux": "^3.7.2", "smalltalk": "^2.5.1", diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js index 8dc0136e1..8dd5e43e5 100644 --- a/ReactNativeClient/lib/BaseApplication.js +++ b/ReactNativeClient/lib/BaseApplication.js @@ -387,20 +387,21 @@ class BaseApplication { } async testing() { + const markdownUtils = require('lib/markdownUtils'); const ClipperServer = require('lib/ClipperServer'); const server = new ClipperServer(); const HtmlToMd = require('lib/HtmlToMd'); const service = new HtmlToMd(); const html = await shim.fsDriver().readFile('/mnt/d/test.html'); - let markdown = service.parse(html); + let markdown = service.parse(html, { baseUrl: 'https://duckduckgo.com/' }); console.info(markdown); console.info('--------------------------------------------------'); - const imageUrls = server.extractImageUrls(markdown); - let result = await server.downloadImages(imageUrls); - result = await server.createResourcesFromPaths(result); + const imageUrls = markdownUtils.extractImageUrls(markdown); + let result = await server.downloadImages_(imageUrls); + result = await server.createResourcesFromPaths_(result); console.info(result); - markdown = server.replaceImageUrlByResources(markdown, result); + markdown = server.replaceImageUrlsByResources_(markdown, result); console.info('--------------------------------------------------'); console.info(markdown); console.info('--------------------------------------------------'); @@ -492,7 +493,11 @@ class BaseApplication { // await this.testing();process.exit(); + const clipperLogger = new Logger(); + clipperLogger.addTarget('file', { path: profileDir + '/log-clipper.txt' }); + clipperLogger.addTarget('console'); this.clipperServer_ = new ClipperServer(); + this.clipperServer_.setLogger(clipperLogger); this.clipperServer_.start(); return argv; diff --git a/ReactNativeClient/lib/ClipperServer.js b/ReactNativeClient/lib/ClipperServer.js index 6181ac420..cb943525d 100644 --- a/ReactNativeClient/lib/ClipperServer.js +++ b/ReactNativeClient/lib/ClipperServer.js @@ -6,9 +6,10 @@ const Resource = require('lib/models/Resource'); const Setting = require('lib/models/Setting'); const { shim } = require('lib/shim'); const md5 = require('md5'); -const { fileExtension, safeFileExtension, filename } = require('lib/path-utils'); +const { fileExtension, safeFileExtension, safeFilename, filename } = require('lib/path-utils'); const HtmlToMd = require('lib/HtmlToMd'); const { Logger } = require('lib/logger.js'); +const markdownUtils = require('lib/markdownUtils'); class ClipperServer { @@ -36,9 +37,7 @@ class ClipperServer { body: requestNote.body ? requestNote.body : '', }; - if (requestNote.bodyHtml) { - console.info(requestNote.bodyHtml); - + if (requestNote.bodyHtml) { // Parsing will not work if the HTML is not wrapped in a top level tag, which is not guaranteed // when getting the content from elsewhere. So here wrap it - it won't change anything to the final // rendering but it makes sure everything will be parsed. @@ -58,39 +57,45 @@ class ClipperServer { return output; } - extractImageUrls_(md) { - // ![some text](http://path/to/image) - const regex = new RegExp(/!\[.*?\]\((http[s]?:\/\/.*?)\)/, 'g') - let match = regex.exec(md); - const output = []; - while (match) { - const url = match[1]; - if (output.indexOf(url) < 0) output.push(url); - match = regex.exec(md); + async downloadImage_(url) { + const tempDir = Setting.value('tempDir'); + const name = filename(url); + let fileExt = safeFileExtension(fileExtension(url).toLowerCase()); + if (fileExt) fileExt = '.' + fileExt; + let imagePath = tempDir + '/' + safeFilename(name) + fileExt; + if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt; + + try { + const result = await shim.fetchBlob(url, { path: imagePath }); + return imagePath; + } catch (error) { + this.logger().warn('Cannot download image at ' + url, error); + return ''; } - return output; } async downloadImages_(urls) { - const tempDir = Setting.value('tempDir'); + const PromisePool = require('es6-promise-pool') + const output = {}; - for (let i = 0; i < urls.length; i++) { - const url = urls[i]; - const name = filename(url); - let fileExt = safeFileExtension(fileExtension(url).toLowerCase()); - if (fileExt) fileExt = '.' + fileExt; - let imagePath = tempDir + '/' + name + fileExt; - if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + name + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt; + let urlIndex = 0; + const promiseProducer = () => { + if (urlIndex >= urls.length) return null; - try { - const result = await shim.fetchBlob(url, { path: imagePath }); - output[url] = { path: imagePath }; - } catch (error) { - this.logger().warn('ClipperServer: Cannot download image at ' + url, error); - } + const url = urls[urlIndex++]; + + return new Promise(async (resolve, reject) => { + const imagePath = await this.downloadImage_(url); + if (imagePath) output[url] = { path: imagePath }; + resolve(); + }); } + const concurrency = 3 + const pool = new PromisePool(promiseProducer, concurrency) + await pool.start() + return output; } @@ -102,16 +107,28 @@ class ClipperServer { const resource = await shim.createResourceFromPath(urlInfo.path); urlInfo.resource = resource; } catch (error) { - this.logger().warn('ClipperServer: Cannot create resource for ' + url, error); + this.logger().warn('Cannot create resource for ' + url, error); + } + } + return urls; + } + + async removeTempFiles_(urls) { + for (let url in urls) { + if (!urls.hasOwnProperty(url)) continue; + const urlInfo = urls[url]; + try { + await shim.fsDriver().remove(urlInfo.path); + } catch (error) { + this.logger().warn('Cannot remove ' + urlInfo.path, error); } } - return urls; } replaceImageUrlsByResources_(md, urls) { - let output = md.replace(/(!\[.*?\]\()(http[s]?:\/\/.*?)(\))/g, (match, before, imageUrl, after) => { + let output = md.replace(/(!\[.*?\]\()([^\s\)]+)(.*?\))/g, (match, before, imageUrl, after) => { const urlInfo = urls[imageUrl]; - if (!urlInfo || !urlInfo.resource) return imageUrl; + if (!urlInfo || !urlInfo.resource) return before + imageUrl + after; const resourceUrl = Resource.internalUrl(urlInfo.resource); return before + resourceUrl + after; }); @@ -121,7 +138,10 @@ class ClipperServer { async start() { const port = await netUtils.findAvailablePort([9967, 8967, 8867], 0); // TODO: Make it shared with OneDrive server - if (!port) throw new Error('All potential ports are in use or not available.'); + if (!port) { + this.logger().error('All potential ports are in use or not available.'); + return; + } const server = require('http').createServer(); @@ -142,7 +162,8 @@ class ClipperServer { response.end(); } - console.info('GOT REQUEST', request.method + ' ' + request.url); + const requestId = Date.now(); + this.logger().info('Request (' + requestId + '): ' + request.method + ' ' + request.url); if (request.method === 'POST') { const url = urlParser.parse(request.url, true); @@ -159,19 +180,17 @@ class ClipperServer { const requestNote = JSON.parse(body); let note = await this.requestNoteToNote(requestNote); - // TODO: Provide way to check status (importing image x/y) - // TODO: Delete temp file after import - // TODO: Download multiple images at once - - const imageUrls = this.extractImageUrls_(note.body); + const imageUrls = markdownUtils.extractImageUrls(note.body); let result = await this.downloadImages_(imageUrls); result = await this.createResourcesFromPaths_(result); + await this.removeTempFiles_(result); note.body = this.replaceImageUrlsByResources_(note.body, result); note = await Note.save(note); + this.logger().info('Request (' + requestId + '): Created note ' + note.id); return writeResponseJson(200, note); } catch (error) { - console.warn(error); + this.logger().error(error); return writeResponseJson(400, { errorCode: 'exception', errorMessage: error.message }); } }); @@ -190,7 +209,7 @@ class ClipperServer { }); - console.info('Starting Clipper server on port ' + port); + this.logger().info('Starting Clipper server on port ' + port); server.listen(port); } diff --git a/ReactNativeClient/lib/logger.js b/ReactNativeClient/lib/logger.js index 6f4ab3c30..a370fd72c 100644 --- a/ReactNativeClient/lib/logger.js +++ b/ReactNativeClient/lib/logger.js @@ -7,7 +7,7 @@ class Logger { constructor() { this.targets_ = []; - this.level_ = Logger.LEVEL_ERROR; + this.level_ = Logger.LEVEL_INFO; this.fileAppendQueue_ = [] this.lastDbCleanup_ = time.unixMs(); } diff --git a/ReactNativeClient/lib/markdownUtils.js b/ReactNativeClient/lib/markdownUtils.js index c53ce7eca..9837dc27c 100644 --- a/ReactNativeClient/lib/markdownUtils.js +++ b/ReactNativeClient/lib/markdownUtils.js @@ -19,6 +19,19 @@ const markdownUtils = { }); }, + extractImageUrls(md) { + // ![some text](http://path/to/image) + const regex = new RegExp(/!\[.*?\]\(([^\s\)]+).*?\)/, 'g') + let match = regex.exec(md); + const output = []; + while (match) { + const url = match[1]; + if (output.indexOf(url) < 0) output.push(url); + match = regex.exec(md); + } + return output; + }, + }; module.exports = markdownUtils; \ No newline at end of file diff --git a/ReactNativeClient/lib/path-utils.js b/ReactNativeClient/lib/path-utils.js index 526a9b277..a35012cd8 100644 --- a/ReactNativeClient/lib/path-utils.js +++ b/ReactNativeClient/lib/path-utils.js @@ -40,6 +40,12 @@ function safeFileExtension(e) { return e.replace(/[^a-zA-Z0-9]/g, '') } +function safeFilename(e, maxLength = 32) { + if (!e || !e.replace) return ''; + let output = e.replace(/[^a-zA-Z0-9\-_\(\)\.]/g, '_') + return output.substr(0, maxLength); +} + function toSystemSlashes(path, os = null) { if (os === null) os = process.platform; if (os === 'win32') return path.replace(/\//g, "\\"); @@ -54,4 +60,4 @@ function ltrimSlashes(path) { return path.replace(/^\/+/, ''); } -module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes }; \ No newline at end of file +module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes }; \ No newline at end of file diff --git a/ReactNativeClient/lib/shim-init-node.js b/ReactNativeClient/lib/shim-init-node.js index ba1a7ca90..5047df1c0 100644 --- a/ReactNativeClient/lib/shim-init-node.js +++ b/ReactNativeClient/lib/shim-init-node.js @@ -97,6 +97,9 @@ function shimInit() { } shim.createResourceFromPath = async function(filePath) { + const readChunk = require('read-chunk'); + const imageType = require('image-type'); + const Resource = require('lib/models/Resource.js'); const { uuid } = require('lib/uuid.js'); const { basename, fileExtension, safeFileExtension } = require('lib/path-utils.js'); @@ -109,9 +112,22 @@ function shimInit() { resource.id = uuid.create(); resource.mime = mime.getType(filePath); resource.title = basename(filePath); - resource.file_extension = safeFileExtension(fileExtension(filePath)); - if (!resource.mime) resource.mime = 'application/octet-stream'; + let fileExt = safeFileExtension(fileExtension(filePath)); + + if (!resource.mime) { + const buffer = await readChunk(filePath, 0, 64); + const detectedType = imageType(buffer); + + if (detectedType) { + fileExt = detectedType.ext; + resource.mime = detectedType.mime; + } else { + resource.mime = 'application/octet-stream'; + } + } + + resource.file_extension = fileExt; let targetPath = Resource.fullPath(resource); @@ -130,35 +146,6 @@ function shimInit() { } shim.attachFileToNote = async function(note, filePath, position = null) { - // const Resource = require('lib/models/Resource.js'); - // const { uuid } = require('lib/uuid.js'); - // const { basename, fileExtension, safeFileExtension } = require('lib/path-utils.js'); - // const mime = require('mime/lite'); - // const Note = require('lib/models/Note.js'); - - // if (!(await fs.pathExists(filePath))) throw new Error(_('Cannot access %s', filePath)); - - // let resource = Resource.new(); - // resource.id = uuid.create(); - // resource.mime = mime.getType(filePath); - // resource.title = basename(filePath); - // resource.file_extension = safeFileExtension(fileExtension(filePath)); - - // if (!resource.mime) resource.mime = 'application/octet-stream'; - - // let targetPath = Resource.fullPath(resource); - - // if (resource.mime == 'image/jpeg' || resource.mime == 'image/jpg' || resource.mime == 'image/png') { - // const result = await resizeImage_(filePath, targetPath, resource.mime); - // } else { - // const stat = await shim.fsDriver().stat(filePath); - // if (stat.size >= 10000000) throw new Error('Resources larger than 10 MB are not currently supported as they may crash the mobile applications. The issue is being investigated and will be fixed at a later time.'); - - // await fs.copy(filePath, targetPath, { overwrite: true }); - // } - - // await Resource.save(resource, { isNew: true }); - const resource = shim.createResourceFromPath(filePath); const newBody = [];