Clipper: Resolves #1379: Improved: Display warning icon when a page might not render well in simplified mode

Clipper/joplin-webclipper/content_scripts/Readability-readerable.js

@@ -0,0 +1,97 @@
+/* eslint-env es6:false */
+/* globals exports */
+/*
+ * Copyright (c) 2010 Arc90 Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This code is heavily based on Arc90's readability.js (1.7.1) script
+ * available at: http://code.google.com/p/arc90labs-readability
+ */
+
+var REGEXPS = {
+  // NOTE: These two regular expressions are duplicated in
+  // Readability.js. Please keep both copies in sync.
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
+  okMaybeItsACandidate: /and|article|body|column|main|shadow/i,
+};
+
+function isNodeVisible(node) {
+  // Have to null-check node.style to deal with SVG and MathML nodes.
+  return (!node.style || node.style.display != "none") && !node.hasAttribute("hidden");
+}
+
+/**
+ * Decides whether or not the document is reader-able without parsing the whole thing.
+ *
+ * @return boolean Whether or not we suspect Readability.parse() will suceeed at returning an article object.
+ */
+function isProbablyReaderable(doc, isVisible) {
+  if (!isVisible) {
+    isVisible = isNodeVisible;
+  }
+
+  var nodes = doc.querySelectorAll("p, pre");
+
+  // Get <div> nodes which have <br> node(s) and append them into the `nodes` variable.
+  // Some articles' DOM structures might look like
+  // <div>
+  //   Sentences<br>
+  //   <br>
+  //   Sentences<br>
+  // </div>
+  var brNodes = doc.querySelectorAll("div > br");
+  if (brNodes.length) {
+    var set = new Set(nodes);
+    [].forEach.call(brNodes, function(node) {
+      set.add(node.parentNode);
+    });
+    nodes = Array.from(set);
+  }
+
+  var score = 0;
+  // This is a little cheeky, we use the accumulator 'score' to decide what to return from
+  // this callback:
+  return [].some.call(nodes, function(node) {
+    if (!isVisible(node))
+      return false;
+
+    var matchString = node.className + " " + node.id;
+    if (REGEXPS.unlikelyCandidates.test(matchString) &&
+        !REGEXPS.okMaybeItsACandidate.test(matchString)) {
+      return false;
+    }
+
+    if (node.matches("li p")) {
+      return false;
+    }
+
+    var textContentLength = node.textContent.trim().length;
+    if (textContentLength < 140) {
+      return false;
+    }
+
+    score += Math.sqrt(textContentLength - 140);
+
+    if (score > 20) {
+      return true;
+    }
+    return false;
+  });
+}
+
+if (typeof exports === "object") {
+  exports.isProbablyReaderable = isProbablyReaderable;
+}

Clipper/joplin-webclipper/content_scripts/index.js

@@ -63,6 +63,12 @@
 		}
 	}
 
+	function documentForReadability() {
+		// Readability directly change the passed document so clone it so as
+		// to preserve the original web page.
+		return document.cloneNode(true);
+	}
+
 	function readabilityProcess() {
 		var uri = {
 			spec: location.href,
@@ -72,10 +78,7 @@
 			pathBase: location.protocol + "//" + location.host + location.pathname.substr(0, location.pathname.lastIndexOf("/") + 1)
 		};
 
-		// Readability directly change the passed document so clone it so as
-		// to preserve the original web page.
-		const documentClone = document.cloneNode(true);
-		const readability = new Readability(documentClone); // new window.Readability(uri, documentClone);
+		const readability = new Readability(documentForReadability());
 		const article = readability.parse();
 
 		if (!article) throw new Error('Could not parse HTML document with Readability');
@@ -117,6 +120,12 @@
 			}
 			return clippedContentResponse(article.title, article.body, getImageSizes(document));
 
+		} else if (command.name === "isProbablyReaderable") {
+
+			const ok = isProbablyReaderable(documentForReadability());
+			console.info('isProbablyReaderable', ok);
+			return { name: 'isProbablyReaderable', value: ok };
+
 		} else if (command.name === "completePageHtml") {
 
 			const cleanDocument = document.body.cloneNode(true);
@@ -250,6 +259,7 @@
 			return {};
 
 		} else if (command.name === "pageUrl") {
+
 			let url = location.origin + location.pathname + location.search;
 			return clippedContentResponse(pageTitle(), url, getImageSizes(document));
 

Clipper/joplin-webclipper/popup/src/App.js

@@ -123,6 +123,7 @@ class AppComponent extends Component {
 	async loadContentScripts() {
 		await bridge().tabsExecuteScript({file: "/content_scripts/JSDOMParser.js"});
 		await bridge().tabsExecuteScript({file: "/content_scripts/Readability.js"});
+		await bridge().tabsExecuteScript({file: "/content_scripts/Readability-readerable.js"});
 		await bridge().tabsExecuteScript({file: "/content_scripts/index.js"});
 	}
 
@@ -158,6 +159,8 @@ class AppComponent extends Component {
 				id: newFolderId,
 			});
 		}
+
+		bridge().sendCommandToActiveTab({ name: 'isProbablyReaderable' });
 	}
 
 	componentDidUpdate() {
@@ -278,11 +281,10 @@ class AppComponent extends Component {
 		const tagsComp = () => {
 			const comps = [];
 			for (let i = 0; i < this.state.selectedTags.length; i++) {
-				comps.push(<div>
+				comps.push(<div key={i}>
 					<input
 						ref={'tagSelector' + i}
 						data-index={i}
-						key={i}
 						type="text"
 						list="tags"
 						value={this.state.selectedTags[i]}
@@ -306,11 +308,18 @@ class AppComponent extends Component {
 			tagDataListOptions.push(<option key={tag.id}>{tag.title}</option>);
 		}
 
+		let simplifiedPageButtonLabel = 'Clip simplified page';
+		let simplifiedPageButtonTooltip = '';
+		if (!this.props.isProbablyReaderable) {
+			simplifiedPageButtonLabel += ' ⚠️';
+			simplifiedPageButtonTooltip = 'It might not be possible to create a good simplified version of this page.\nYou may want to clip the complete page instead.';
+		}
+
 		return (
 			<div className="App">
 				<div className="Controls">			
 					<ul>
-						<li><a className="Button" onClick={this.clipSimplified_click}>Clip simplified page</a></li>
+						<li><a className="Button" onClick={this.clipSimplified_click} title={simplifiedPageButtonTooltip}>{simplifiedPageButtonLabel}</a></li>
 						<li><a className="Button" onClick={this.clipComplete_click}>Clip complete page</a></li>
 						<li><a className="Button" onClick={this.clipSelection_click}>Clip selection</a></li>
 						<li><a className="Button" onClick={this.clipScreenshot_click}>Clip screenshot</a></li>
@@ -343,6 +352,7 @@ const mapStateToProps = (state) => {
 		folders: state.folders,
 		tags: state.tags,
 		selectedFolderId: state.selectedFolderId,
+		isProbablyReaderable: state.isProbablyReaderable,
 	};
 };
 

Clipper/joplin-webclipper/popup/src/bridge.js

@@ -34,6 +34,10 @@ class Bridge {
 
 				this.dispatch({ type: 'CLIPPED_CONTENT_SET', content: content });
 			}
+
+			if (command.name === 'isProbablyReaderable') {
+				this.dispatch({ type: 'IS_PROBABLY_READERABLE', value: command.value });
+			}
 		}
 
 		this.browser_.runtime.onMessage.addListener(this.browser_notify);

Clipper/joplin-webclipper/popup/src/index.js

@@ -19,6 +19,7 @@ const defaultState = {
 	tags: [],
 	selectedFolderId: null,
 	env: 'prod',
+	isProbablyReaderable: true,
 };
 
 const reduxMiddleware = store => next => async (action) => {
@@ -40,6 +41,11 @@ function reducer(state = defaultState, action) {
 		newState = Object.assign({}, state);
 		newState.warning = action.text;
 
+	} else if (action.type === 'IS_PROBABLY_READERABLE') {
+
+		newState = Object.assign({}, state);
+		newState.isProbablyReaderable = action.value;
+
 	} else if (action.type === 'CLIPPED_CONTENT_SET') {
 
 		newState = Object.assign({}, state);

docs/faq/index.html

@@ -251,7 +251,7 @@
 </ul>
 </div>
 <h1 id="clicking-edit-in-external-editor-does-nothing-i-want-to-change-the-editor-">Clicking &#39;Edit in External Editor&#39; does nothing! / I want to change the editor!</h1>
-<p>The editor command (may include arguments) defines which editor will be used to open a note. If none is provided it will try to auto-detect the default editor. If this does nothing or you want to change it for Joplin, you need to configure it in Settings -&gt; Text editor command.</p>
+<p>The editor command (may include arguments) defines which editor will be used to open a note. If none is provided it will try to auto-detect the default editor. If this does nothing or you want to change it for Joplin, you need to configure it in the Preferences -&gt; Text editor command.</p>
 <p>Some example configurations are: (comments after #)</p>
 <p>Linux/Mac:</p>
 <pre><code class="lang-bash">subl -n       # Opens Sublime (subl) in a new window (-n)

readme/faq.md

@@ -1,6 +1,6 @@
 # Clicking 'Edit in External Editor' does nothing! / I want to change the editor!
 
-The editor command (may include arguments) defines which editor will be used to open a note. If none is provided it will try to auto-detect the default editor. If this does nothing or you want to change it for Joplin, you need to configure it in Settings -> Text editor command.
+The editor command (may include arguments) defines which editor will be used to open a note. If none is provided it will try to auto-detect the default editor. If this does nothing or you want to change it for Joplin, you need to configure it in the Preferences -> Text editor command.
 
 Some example configurations are: (comments after #)