From a0dd0702fb020d2cce3e1b8a8a5f303e284c047a Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 14 Jul 2019 16:00:02 +0100 Subject: [PATCH 01/18] Clipper: Adding support for clipping page as HTML --- CliClient/package-lock.json | 317 +++++++++++--- CliClient/package.json | 1 + CliClient/tests/htmlUtils.js | 33 ++ .../content_scripts/index.js | 38 +- Clipper/joplin-webclipper/popup/src/App.js | 27 +- Clipper/joplin-webclipper/popup/src/bridge.js | 3 + ElectronClient/app/gui/note-viewer/index.html | 3 + ElectronClient/app/package-lock.json | 412 +++++++++++++++--- ElectronClient/app/package.json | 2 + .../lib/MdToHtml/rules/html_image.js | 2 +- ReactNativeClient/lib/htmlUtils.js | 45 ++ ReactNativeClient/lib/joplin-database.js | 6 +- ReactNativeClient/lib/markupLanguageUtils.js | 21 + ReactNativeClient/lib/models/Note.js | 3 + ReactNativeClient/lib/services/rest/Api.js | 99 +++-- 15 files changed, 866 insertions(+), 146 deletions(-) create mode 100644 CliClient/tests/htmlUtils.js create mode 100644 ReactNativeClient/lib/htmlUtils.js create mode 100644 ReactNativeClient/lib/markupLanguageUtils.js diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index 6d3718096..62dc0c167 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -15,9 +15,9 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "acorn": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", - "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==" + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.2.0.tgz", + "integrity": "sha512-8oe72N3WPMjA+2zVG71Ia0nXZ8DpQH+QyyHO+p06jT8eg8FGG3FbcUIi8KziHlAfheJQZeoqbvq1mQSQHXKYLw==" }, "acorn-globals": { "version": "4.3.2", @@ -26,13 +26,6 @@ "requires": { "acorn": "^6.0.1", "acorn-walk": "^6.0.1" - }, - "dependencies": { - "acorn": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.2.0.tgz", - "integrity": "sha512-8oe72N3WPMjA+2zVG71Ia0nXZ8DpQH+QyyHO+p06jT8eg8FGG3FbcUIi8KziHlAfheJQZeoqbvq1mQSQHXKYLw==" - } } }, "acorn-walk": { @@ -518,18 +511,6 @@ "abab": "^2.0.0", "whatwg-mimetype": "^2.2.0", "whatwg-url": "^7.0.0" - }, - "dependencies": { - "whatwg-url": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", - "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", - "requires": { - "lodash.sortby": "^4.7.0", - "tr46": "^1.0.1", - "webidl-conversions": "^4.0.2" - } - } } }, "debug": { @@ -1374,6 +1355,11 @@ "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", "integrity": "sha1-ge9X/l0FgUzVjCSDYyqZwwoOgIc=" }, + "ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=" + }, "is-absolute": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-0.2.6.tgz", @@ -1472,6 +1458,193 @@ "css": "^2.2.4", "html-entities": "^1.2.1", "jsdom": "^11.9.0" + }, + "dependencies": { + "acorn": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", + "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==" + }, + "ajv": { + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.1.tgz", + "integrity": "sha512-w1YQaVGNC6t2UCPjEawK/vo/dG8OOrVtUmhBT1uJJYxbl5kU2Tj3v6LGqBcsysN1yhuCStJCCA3GqdvKY8sqXQ==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "jsdom": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", + "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", + "requires": { + "abab": "^2.0.0", + "acorn": "^5.5.3", + "acorn-globals": "^4.1.0", + "array-equal": "^1.0.0", + "cssom": ">= 0.3.2 < 0.4.0", + "cssstyle": "^1.0.0", + "data-urls": "^1.0.0", + "domexception": "^1.0.1", + "escodegen": "^1.9.1", + "html-encoding-sniffer": "^1.0.2", + "left-pad": "^1.3.0", + "nwsapi": "^2.0.7", + "parse5": "4.0.0", + "pn": "^1.1.0", + "request": "^2.87.0", + "request-promise-native": "^1.0.5", + "sax": "^1.2.4", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.3.4", + "w3c-hr-time": "^1.0.1", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.3", + "whatwg-mimetype": "^2.1.0", + "whatwg-url": "^6.4.1", + "ws": "^5.2.0", + "xml-name-validator": "^3.0.0" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "parse5": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", + "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } + } + } + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "whatwg-url": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", + "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "ws": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", + "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", + "requires": { + "async-limiter": "~1.0.0" + } + } } }, "joplin-turndown-plugin-gfm": { @@ -1496,35 +1669,35 @@ "optional": true }, "jsdom": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", - "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-15.1.1.tgz", + "integrity": "sha512-cQZRBB33arrDAeCrAEWn1U3SvrvC8XysBua9Oqg1yWrsY/gYcusloJC3RZJXuY5eehSCmws8f2YeliCqGSkrtQ==", "requires": { "abab": "^2.0.0", - "acorn": "^5.5.3", - "acorn-globals": "^4.1.0", + "acorn": "^6.1.1", + "acorn-globals": "^4.3.2", "array-equal": "^1.0.0", - "cssom": ">= 0.3.2 < 0.4.0", - "cssstyle": "^1.0.0", - "data-urls": "^1.0.0", + "cssom": "^0.3.6", + "cssstyle": "^1.2.2", + "data-urls": "^1.1.0", "domexception": "^1.0.1", - "escodegen": "^1.9.1", + "escodegen": "^1.11.1", "html-encoding-sniffer": "^1.0.2", - "left-pad": "^1.3.0", - "nwsapi": "^2.0.7", - "parse5": "4.0.0", + "nwsapi": "^2.1.4", + "parse5": "5.1.0", "pn": "^1.1.0", - "request": "^2.87.0", - "request-promise-native": "^1.0.5", - "sax": "^1.2.4", + "request": "^2.88.0", + "request-promise-native": "^1.0.7", + "saxes": "^3.1.9", "symbol-tree": "^3.2.2", - "tough-cookie": "^2.3.4", + "tough-cookie": "^3.0.1", "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.1.2", "webidl-conversions": "^4.0.2", - "whatwg-encoding": "^1.0.3", - "whatwg-mimetype": "^2.1.0", - "whatwg-url": "^6.4.1", - "ws": "^5.2.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^7.0.0", "xml-name-validator": "^3.0.0" }, "dependencies": { @@ -1647,6 +1820,23 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" }, + "tough-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==", + "requires": { + "ip-regex": "^2.1.0", + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "dependencies": { + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + } + } + }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", @@ -2200,9 +2390,9 @@ } }, "parse5": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", - "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" }, "path-exists": { "version": "3.0.0", @@ -2503,6 +2693,14 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "saxes": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", + "integrity": "sha512-Ydydq3zC+WYDJK1+gRxRapLIED9PWeSuuS41wqyoRmzvhhh9nc+QQrVMKJYzJFULazeGhzSV0QleN2wD3boh2g==", + "requires": { + "xmlchars": "^2.1.1" + } + }, "semver": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", @@ -3212,6 +3410,16 @@ "browser-process-hrtime": "^0.1.2" } }, + "w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "requires": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, "webidl-conversions": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", @@ -3241,9 +3449,9 @@ "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" }, "whatwg-url": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", - "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", + "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", "requires": { "lodash.sortby": "^4.7.0", "tr46": "^1.0.1", @@ -3297,11 +3505,11 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", - "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.1.0.tgz", + "integrity": "sha512-Swie2C4fs7CkwlHu1glMePLYJJsWjzhl1vm3ZaLplD0h7OMkZyZ6kLTB/OagiU923bZrPFXuDTeEqaEN4NWG4g==", "requires": { - "async-limiter": "~1.0.0" + "async-limiter": "^1.0.0" } }, "xml-name-validator": { @@ -3323,6 +3531,11 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz", "integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8=" }, + "xmlchars": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.1.1.tgz", + "integrity": "sha512-7hew1RPJ1iIuje/Y01bGD/mXokXxegAgVS+e+E0wSi2ILHQkYAH1+JXARwTjZSM4Z4Z+c73aKspEcqj+zPPL/w==" + }, "xregexp": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-3.2.0.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index 99e2311c8..1918b607c 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -45,6 +45,7 @@ "image-type": "^3.0.0", "joplin-turndown": "^4.0.17", "joplin-turndown-plugin-gfm": "^1.0.8", + "jsdom": "^15.1.1", "jssha": "^2.3.0", "levenshtein": "^1.0.5", "lodash": "^4.17.4", diff --git a/CliClient/tests/htmlUtils.js b/CliClient/tests/htmlUtils.js new file mode 100644 index 000000000..0181025f3 --- /dev/null +++ b/CliClient/tests/htmlUtils.js @@ -0,0 +1,33 @@ +require('app-module-path').addPath(__dirname); + +const { time } = require('lib/time-utils.js'); +const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); +const htmlUtils = require('lib/htmlUtils.js'); + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +describe('htmlUtils', function() { + + beforeEach(async (done) => { + done(); + }); + + it('should extract image URLs', async (done) => { + const testCases = [ + ['', ['http://test.com/img.png']], + [' ', ['http://test.com/img.png', 'http://test.com/img2.png']], + ]; + + for (let i = 0; i < testCases.length; i++) { + const md = testCases[i][0]; + const expected = testCases[i][1]; + + expect(htmlUtils.extractImageUrls(md).join(' ')).toBe(expected.join(' ')); + } + + done(); + }); + +}); \ No newline at end of file diff --git a/Clipper/joplin-webclipper/content_scripts/index.js b/Clipper/joplin-webclipper/content_scripts/index.js index 5b5324802..943530de8 100644 --- a/Clipper/joplin-webclipper/content_scripts/index.js +++ b/Clipper/joplin-webclipper/content_scripts/index.js @@ -61,7 +61,7 @@ const output = {}; for (let i = 0; i < images.length; i++) { const img = images[i]; - const src = forceAbsoluteUrls ? absoluteUrl(img.src) : img.src; + const src = forceAbsoluteUrls ? absoluteUrl(img.currentSrc) : img.currentSrc; output[src] = { width: img.width, height: img.height, @@ -112,7 +112,7 @@ } if (nodeName === 'img') { - node.src = absoluteUrl(node.src); + node.src = absoluteUrl(node.currentSrc); const imageSize = imageSizes[node.src]; if (imageSize) { node.width = imageSize.width; @@ -135,6 +135,8 @@ for (let i = 0; i < childNodes.length; i++) { const node = childNodes[i]; const nodeName = node.nodeName.toLowerCase(); + const nodeParent = node.parentNode; + const nodeParentName = nodeParent ? nodeParent.nodeName.toLowerCase() : ''; let isVisible = node.nodeType === 1 ? window.getComputedStyle(node).display !== 'none' : true; if (isVisible && ['script', 'noscript', 'style', 'select', 'option', 'button'].indexOf(nodeName) >= 0) isVisible = false; @@ -153,6 +155,10 @@ if (a && a.toLowerCase().indexOf('math/tex') >= 0) isVisible = true; } + if (nodeName === 'source' && nodeParentName === 'picture') { + isVisible = false + } + if (!isVisible) { node.classList.add('joplin-clipper-hidden'); } else { @@ -179,19 +185,25 @@ // Given a document, return a '; + return output; } function documentForReadability() { @@ -223,7 +235,7 @@ async function prepareCommandResponse(command) { console.info('Got command: ' + command.name); - const clippedContentResponse = (title, html, imageSizes, anchorNames) => { + const clippedContentResponse = (title, html, imageSizes, anchorNames, stylesheets) => { return { name: 'clippedContent', title: title, @@ -234,6 +246,9 @@ tags: command.tags || '', image_sizes: imageSizes, anchor_names: anchorNames, + source_command: Object.assign({}, command), + convert_to: command.preProcessFor ? command.preProcessFor : 'markdown', + stylesheets: stylesheets, }; } @@ -255,7 +270,6 @@ } else if (command.name === "isProbablyReaderable") { const ok = isProbablyReaderable(documentForReadability()); - console.info('isProbablyReaderable', ok); return { name: 'isProbablyReaderable', value: ok }; } else if (command.name === "completePageHtml") { @@ -267,7 +281,9 @@ const cleanDocument = document.body.cloneNode(true); const imageSizes = getImageSizes(document, true); cleanUpElement(cleanDocument, imageSizes); - return clippedContentResponse(pageTitle(), cleanDocument.innerHTML, imageSizes, getAnchorNames(document)); + + const stylesheets = command.preProcessFor === 'html' ? getStyleSheets(document) : null; + return clippedContentResponse(pageTitle(), cleanDocument.innerHTML, imageSizes, getAnchorNames(document), stylesheets); } else if (command.name === "selectedHtml") { diff --git a/Clipper/joplin-webclipper/popup/src/App.js b/Clipper/joplin-webclipper/popup/src/App.js index 109bf8ace..fb2ef624b 100644 --- a/Clipper/joplin-webclipper/popup/src/App.js +++ b/Clipper/joplin-webclipper/popup/src/App.js @@ -7,6 +7,20 @@ import led_orange from './led_orange.png'; const { connect } = require('react-redux'); const { bridge } = require('./bridge'); +function commandUserString(command) { + const s = []; + + if (command.name === 'simplifiedPageHtml') s.push('Simplified page'); + if (command.name === 'completePageHtml') s.push('Complete page'); + if (command.name === 'selectedHtml') s.push('Selection'); + if (command.name === 'pageUrl') s.push('URL only'); + + const p = command.preProcessFor ? command.preProcessFor : 'markdown'; + s.push('(' + p + ')'); + + return s.join(' '); +} + class PreviewComponent extends React.PureComponent { constructor() { @@ -17,7 +31,7 @@ class PreviewComponent extends React.PureComponent { componentDidMount() { if (!this.bodyRef.current) return; - + // Because the text size is made twice smaller with CSS, we need // to also reduce the size of the images const imgs = this.bodyRef.current.getElementsByTagName('img'); @@ -32,6 +46,7 @@ class PreviewComponent extends React.PureComponent {

Title:

+

Type: {commandUserString(this.props.command)}

Confirm
); @@ -84,6 +99,14 @@ class AppComponent extends Component { this.clipComplete_click = () => { bridge().sendCommandToActiveTab({ name: 'completePageHtml', + preProcessFor: 'markdown', + }); + } + + this.clipCompleteHtml_click = () => { + bridge().sendCommandToActiveTab({ + name: 'completePageHtml', + preProcessFor: 'html', }); } @@ -259,6 +282,7 @@ class AppComponent extends Component { title={content.title} body_html={content.body_html} onTitleChange={this.contentTitle_change} + command={content.source_command} /> } @@ -360,6 +384,7 @@ class AppComponent extends Component {