Clipper: Fixed importing certain images with sources that contain brackets

CliClient/package-lock.json

@@ -70,6 +70,21 @@
         "readable-stream": "^2.0.6"
       }
     },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      },
+      "dependencies": {
+        "sprintf-js": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+          "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
+        }
+      }
+    },
     "array-equal": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
@@ -493,6 +508,11 @@
         "iconv-lite": "~0.4.13"
       }
     },
+    "entities": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
+      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
+    },
     "es6-promise-pool": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz",
@@ -1117,6 +1137,14 @@
         "type-check": "~0.3.2"
       }
     },
+    "linkify-it": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.0.3.tgz",
+      "integrity": "sha1-2UpGSPmxwXnWT6lykSaL22zpQ08=",
+      "requires": {
+        "uc.micro": "^1.0.1"
+      }
+    },
     "lodash": {
       "version": "4.17.4",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz",
@@ -1159,6 +1187,18 @@
         "highlight.js": "~9.12.0"
       }
     },
+    "markdown-it": {
+      "version": "8.4.2",
+      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-8.4.2.tgz",
+      "integrity": "sha512-GcRz3AWTqSUphY3vsUqQSFMbgR38a4Lh3GWlHRh/7MRwz8mcu9n2IO7HOh+bXHrR9kOPDl5RNCaEsrneb+xhHQ==",
+      "requires": {
+        "argparse": "^1.0.7",
+        "entities": "~1.1.1",
+        "linkify-it": "^2.0.0",
+        "mdurl": "^1.0.1",
+        "uc.micro": "^1.0.5"
+      }
+    },
     "md5": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz",
@@ -1169,6 +1209,11 @@
         "is-buffer": "~1.1.1"
       }
     },
+    "mdurl": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
+      "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4="
+    },
     "mime": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/mime/-/mime-2.0.3.tgz",
@@ -2039,6 +2084,11 @@
         "prelude-ls": "~1.1.2"
       }
     },
+    "uc.micro": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.5.tgz",
+      "integrity": "sha512-JoLI4g5zv5qNyT09f4YAvEZIIV1oOjqnewYg5D38dkQljIzpPT296dbIGvKro3digYI1bkb7W6EP1y4uDlmzLg=="
+    },
     "uglify-js": {
       "version": "3.3.25",
       "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.25.tgz",

CliClient/package.json

@@ -43,6 +43,7 @@
     "jssha": "^2.3.0",
     "levenshtein": "^1.0.5",
     "lodash": "^4.17.4",
+    "markdown-it": "^8.4.2",
     "md5": "^2.2.1",
     "mime": "^2.0.3",
     "moment": "^2.18.1",

CliClient/tests/markdownUtils.js

@@ -39,6 +39,7 @@ describe('markdownUtils', function() {
 			['![something](http://test.com/img.png)', ['http://test.com/img.png']],
 			['![something](http://test.com/img.png) ![something2](http://test.com/img2.png)', ['http://test.com/img.png', 'http://test.com/img2.png']],
 			['![something](http://test.com/img.png "Some description")', ['http://test.com/img.png']],
+			['![something](https://test.com/ohoh_(123).png)', ['https://test.com/ohoh_(123).png']],
 		];
 
 		for (let i = 0; i < testCases.length; i++) {

ReactNativeClient/lib/ClipperServer.js

@@ -115,14 +115,21 @@ class ClipperServer {
 
 	async downloadImage_(url) {
 		const tempDir = Setting.value('tempDir');
-		const name = filename(url);
-		let fileExt = safeFileExtension(fileExtension(url).toLowerCase());
+
+		const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0;
+
+		const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url);
+		let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase());
 		if (fileExt) fileExt = '.' + fileExt;
 		let imagePath = tempDir + '/' + safeFilename(name) + fileExt;
 		if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt;
 
 		try {
-			const result = await shim.fetchBlob(url, { path: imagePath });
+			if (isDataUrl) {
+				await shim.imageFromDataUrl(url, imagePath);
+			} else {
+				await shim.fetchBlob(url, { path: imagePath });
+			}
 			return imagePath;
 		} catch (error) {
 			this.logger().warn('Cannot download image at ' + url, error);
@@ -276,11 +283,19 @@ class ClipperServer {
 							let note = await this.requestNoteToNote(requestNote);
 
 							const imageUrls = markdownUtils.extractImageUrls(note.body);
+
+							this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length);
+
 							let result = await this.downloadImages_(imageUrls);
+
+							this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length);
+
 							result = await this.createResourcesFromPaths_(result);
 							await this.removeTempFiles_(result);
 							note.body = this.replaceImageUrlsByResources_(note.body, result);
 
+							this.logger().info('Request (' + requestId + '): Saving note...');
+
 							note = await Note.save(note);
 
 							if (requestNote.tags) {

ReactNativeClient/lib/markdownUtils.js

@@ -1,4 +1,5 @@
 const urlUtils = require('lib/urlUtils');
+const MarkdownIt = require('markdown-it');
 
 const markdownUtils = {
 
@@ -20,15 +21,32 @@ const markdownUtils = {
 	},
 
 	extractImageUrls(md) {
-		// ![some text](http://path/to/image)
-		const regex = new RegExp(/!\[.*?\]\(([^\s\)]+).*?\)/, 'g')
-		let match = regex.exec(md);
+		const markdownIt = new MarkdownIt();
+		const env = {};
+		const tokens = markdownIt.parse(md, env);
 		const output = [];
-		while (match) {
-			const url = match[1];
-			if (output.indexOf(url) < 0) output.push(url);
-			match = regex.exec(md);
+
+		const searchUrls = (tokens) => {
+			for (let i = 0; i < tokens.length; i++) {
+				const token = tokens[i];
+
+				if (token.type === 'image') {
+					for (let j = 0; j < token.attrs.length; j++) {
+						const a = token.attrs[j];
+						if (a[0] === 'src' && a.length >= 2 && a[1]) {
+							output.push(a[1]);
+						}
+					}
+				}
+				
+				if (token.children && token.children.length) {
+					searchUrls(token.children);
+				}
+			}
 		}
+
+		searchUrls(tokens);
+
 		return output;
 	},
 

ReactNativeClient/lib/shim-init-node.js

@@ -173,7 +173,7 @@ function shimInit() {
 		if (shim.isElectron()) {
 			const nativeImage = require('electron').nativeImage;
 			let image = nativeImage.createFromDataURL(imageDataUrl);
-			if (image.isEmpty()) throw new Error('Could not convert data URL to image');
+			if (image.isEmpty()) throw new Error('Could not convert data URL to image'); // Would throw for example if the image format is no supported (eg. image/gif)
 			if (options.cropRect) {
 				// Crop rectangle values need to be rounded or the crop() call will fail
 				const c = options.cropRect;