CLI v1.0.159

* added toggleLayout logic

* added id

* improved code

.eslintignore

@@ -6,6 +6,7 @@ _releases/
 Assets/
 CliClient/build
 CliClient/locales
+CliClient/locales-build
 CliClient/node_modules
 CliClient/tests-build
 CliClient/tests/enex_to_md
@@ -14,20 +15,20 @@ CliClient/tests/logs
 CliClient/tests/support
 CliClient/tests/sync
 CliClient/tests/tmp
-Clipper/joplin-webclipper/content_scripts/JSDOMParser.js
-Clipper/joplin-webclipper/content_scripts/Readability-readerable.js
-Clipper/joplin-webclipper/content_scripts/Readability.js
-Clipper/joplin-webclipper/dist
-Clipper/joplin-webclipper/icons
-Clipper/joplin-webclipper/popup/build
-Clipper/joplin-webclipper/popup/node_modules
+Clipper/content_scripts/JSDOMParser.js
+Clipper/content_scripts/Readability-readerable.js
+Clipper/content_scripts/Readability.js
+Clipper/dist
+Clipper/icons
+Clipper/popup/build
+Clipper/popup/node_modules
 docs/
-ElectronClient/app/dist
-ElectronClient/app/lib
-ElectronClient/app/lib/vendor/sjcl-rn.js
-ElectronClient/app/lib/vendor/sjcl.js
-ElectronClient/app/locales
-ElectronClient/app/node_modules
+ElectronClient/dist
+ElectronClient/lib
+ElectronClient/lib/vendor/sjcl-rn.js
+ElectronClient/lib/vendor/sjcl.js
+ElectronClient/locales
+ElectronClient/node_modules
 highlight.pack.js
 node_modules/
 ReactNativeClient/android
@@ -45,8 +46,19 @@ Server/docs/
 Server/dist/
 Server/bin/
 Server/node_modules/
-ElectronClient/app/packageInfo.js
+ElectronClient/packageInfo.js
+ReactNativeClient/pluginAssets/
+ReactNativeClient/lib/joplin-renderer/vendor/fountain.min.js
+ReactNativeClient/lib/joplin-renderer/assets/
+ReactNativeClient/lib/rnInjectedJs/
 
-# Ignore files generated from TypeScript files
-ElectronClient/app/gui/ShareNoteDialog.js
+# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
+ElectronClient/gui/NoteContentPropertiesDialog.js
+ElectronClient/gui/ResourceScreen.js
+ElectronClient/gui/ShareNoteDialog.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/PluginAssetsLoader.js
+ReactNativeClient/setUpQuickActions.js
+# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

.eslintrc.js

@@ -6,6 +6,11 @@ module.exports = {
 	},
 	"parser": "@typescript-eslint/parser",
 	'extends': ['eslint:recommended'],
+	"settings": {
+		'react': {
+			'version': '16.12',
+		},
+	},
 	'globals': {
 		'Atomics': 'readonly',
 		'SharedArrayBuffer': 'readonly',
@@ -14,7 +19,10 @@ module.exports = {
 		'expect': 'readonly',
 		'describe': 'readonly',
 		'it': 'readonly',
+		'beforeAll': 'readonly',
+		'afterAll': 'readonly',
 		'beforeEach': 'readonly',
+		'afterEach': 'readonly',
 		'jasmine': 'readonly',
 
 		// React Native variables
@@ -44,7 +52,11 @@ module.exports = {
 		// This error is always a false positive so far since it detects
 		// possible race conditions in contexts where we know it cannot happen.
 		"require-atomic-updates": 0,
-		// "no-lonely-if": "error",
+
+		// Checks rules of Hooks
+		"react-hooks/rules-of-hooks": "error",
+		// Checks effect dependencies
+		"react-hooks/exhaustive-deps": "error",
 
 		// -------------------------------
 		// Formatting
@@ -59,6 +71,8 @@ module.exports = {
 		"linebreak-style": ["error", "unix"],
 		"prefer-template": ["error"],
 		"template-curly-spacing": ["error", "never"],
+		"object-curly-spacing": ["error", "always"],
+		"array-bracket-spacing": ["error", "never"],
 		"key-spacing": ["error", {
 			"beforeColon": false,
 			"afterColon": true,
@@ -81,5 +95,6 @@ module.exports = {
 	"plugins": [
 		"react",
 		"@typescript-eslint",
+		"react-hooks"
 	],
-};
+};

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,24 +19,26 @@ labels: 'bug'
 
 ## Environment
 
-Joplin version:
-Platform:
-OS specifcs:
+Joplin version: 
+Platform: 
+OS specifics: 
+
 <!--
   Platform can be one of: macOS, Linux, Windows, Android, iOS, terminal (or a combination)
-  OS specifcs: e.g. OS version, Linux distribution, Android/iOS version, ...
+  OS specifics: e.g. OS version, Linux distribution, Android/iOS version...
 -->
 
-## Steps To Reproduce
+## Steps to reproduce
 
-1.
-2.
+1. 
+2. 
+3. 
 
 <!--
   Issues without reproduction steps are likely to stall.
 -->
 
-Describe what you expected to happen:
+## Describe what you expected to happen
 
 
 

.github/stale.yml

@@ -12,6 +12,7 @@ exemptLabels:
   - "nice to have"
   - "upstream"
   - "backlog"
+  - "high"
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.gitignore

@@ -40,12 +40,22 @@ Tools/github_oauth_token.txt
 _releases
 ReactNativeClient/lib/csstojs/
 ReactNativeClient/lib/rnInjectedJs/
-ElectronClient/app/gui/note-viewer/fonts/
-ElectronClient/app/gui/note-viewer/lib.js
+ElectronClient/gui/note-viewer/fonts/
+ElectronClient/gui/note-viewer/lib.js
+Clipper-source/
+Clipper/joplin-webclipper-source.zip
+joplin-webclipper-source.zip
 Tools/commit_hook.txt
 .vscode/*
 *.map
 
-# Ignore files generated from TypeScript files
-ElectronClient/app/gui/ShareNoteDialog.js
+# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
+ElectronClient/gui/NoteContentPropertiesDialog.js
+ElectronClient/gui/ResourceScreen.js
+ElectronClient/gui/ShareNoteDialog.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/PluginAssetsLoader.js
+ReactNativeClient/setUpQuickActions.js
+# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

.travis.yml

@@ -55,12 +55,8 @@ before_install:
 
 script:
   - |
-    # Copy lib
-    rsync -aP --delete ReactNativeClient/lib/ ElectronClient/app/lib/
-
     # Install tools
     npm install
-    npm run tsc
     cd Tools
     npm install
     cd ..
@@ -70,8 +66,7 @@ script:
     # and that would break the desktop release.
     if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
       cd CliClient
-      npm install
-      ./run_test.sh
+      npm run test
       testResult=$?
       if [ $testResult -ne 0 ]; then
         exit $testResult
@@ -113,5 +108,5 @@ script:
     fi
 
     # Prepare the Electron app and build it
-    cd ElectronClient/app
-    npm install && USE_HARD_LINKS=false yarn dist
+    cd ElectronClient
+    USE_HARD_LINKS=false yarn dist

BUILD.md

@@ -1,77 +1,104 @@
  [![Travis Build Status](https://travis-ci.org/laurent22/joplin.svg?branch=master)](https://travis-ci.org/laurent22/joplin) [![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/github/laurent22/joplin?branch=master&passingText=master%20-%20OK&svg=true)](https://ci.appveyor.com/project/laurent22/joplin)
 
-# General information
+# Building the applications
 
-- All the applications share the same library, which, for historical reasons, is in ReactNativeClient/lib. This library is copied to the relevant directories when building each app.
-- In general, most of the backend (anything to do with the database, synchronisation, data import or export, etc.) is shared across all the apps, so when making a change please consider how it will affect all the apps.
+Note that all the applications share the same library, which, for historical reasons, is in `ReactNativeClient/lib`. This library is copied to the relevant directories when building each app.
 
-# TypeScript
-
-Most of the application is written in JavaScript, however new classes and files should generally be written in [TypeScript](https://www.typescriptlang.org/). Even if you don't write TypeScript code, you will need to build the existing .ts and .tsx files. This is done from the root of the project, by running `npm run tsc`.
-
-If you are modifying TypeScript code, the best is to have the compiler watch for changes from a terminal. To do so, run `npm run tsc-watch`.
-
-All TypeScript files are generated next to the .ts or .tsx file. So for example, if there's a file "lib/MyClass.ts", there will be a generated "lib/MyClass.js" next to it. If you create a new TypeScript file, make sure you add the generated .js file to .gitignore. It is implemented that way as it requires minimal changes to integrate TypeScript in the existing JavaScript code base.
-
-## macOS dependencies
-
-	brew install yarn node
-	echo 'export PATH="/usr/local/opt/gettext/bin:$PATH"' >> ~/.bash_profile
-	source ~/.bash_profile
-
-## Linux and Windows (WSL) dependencies
+## Required dependencies
 
 - Install yarn - https://yarnpkg.com/lang/en/docs/install/
-- Install node v10.x (check with `node --version`) - https://nodejs.org/en/
-- If you get a node-gyp related error you might need to manually install it: `npm install -g node-gyp`
+- Install node - https://nodejs.org/en/
 
-# Building the tools
+## Building
 
-Before building any of the applications, you need to build the tools and pre-commit hooks:
+Before doing anything else, from the root of the project, run:
 
-```
-npm install && cd Tools && npm install
+	npm install
+
+Then you can test the various applications:
+
+## Testing the desktop application
+
+	cd ElectronClient
+	npm start
+
+If you'd like to auto-reload the app on changes rather than having to quit and restart it manually each time, you can use [watchman-make](https://facebook.github.io/watchman/docs/watchman-make.html):
+
+```sh
+cd ElectronClient
+watchman-make -p '**/*.js' '**/*.jsx' --run "npm start"
 ```
 
-# Building the Electron application
+It still requires you to quit the application each time you want it to rebuild, but at least you don't have to re-run `"npm start"` each time. Here's what the workflow loop looks like in practice:
 
-```
-npm run copyLib
-npm run tsc
-cd ElectronClient/app
-npm install
-yarn dist
-```
+1. Edit and save files in your text editor.
+2. Switch to the Electron app and <kbd>cmd</kbd>+<kbd>Q</kbd> to quit it.
+3. `watchman` immediately restarts the app for you (whereas usually you'd have to switch back to the terminal, type `"npm start"`, and hit enter).
+
+## Testing the Terminal application
+
+	cd CliClient
+	npm start
+
+## Testing the Mobile application
+
+First you need to setup React Native to build projects with native code. For this, follow the instructions on the [Get Started](https://facebook.github.io/react-native/docs/getting-started.html) tutorial, in the "React Native CLI Quickstart" tab.
+
+Then:
+
+	cd ReactNativeClient
+	npm start-android
+	# Or: npm start-ios
+
+To run the iOS application, it might be easier to open the file `ios/Joplin.xcworkspace` on XCode and run the app from there.
+
+Normally the bundler should start automatically with the application. If it doesn't run `npm start`.
+
+## Building the clipper
+
+	cd Clipper/popup
+	npm install
+	npm run watch # To watch for changes
+
+To test the extension please refer to the relevant pages for each browser: [Firefox](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Your_first_WebExtension#Trying_it_out) / [Chrome](https://developer.chrome.com/extensions/faq#faq-dev-01). Please note that the extension in dev mode will only connect to a dev instance of the desktop app (and vice-versa).
+
+## Watching files
+
+To make changes to the application, you'll need to rebuild any TypeScript file you've changed, and rebuild the lib. The simplest way to do all this is to watch for changes from the root of the project. Simply run this command, and it should take care of the rest:
+
+	npm run watch
+
+Running `npm run build` would have the same effect, but without watching.
+
+## Running an application with additional parameters
+
+You can specify additional parameters when running the desktop or CLI application. To do so, add `--` to the `npm start` command, followed by your flags. For example:
+
+	npm start -- --profile ~/MyTestProfile
+
+## TypeScript
+
+Most of the application is written in JavaScript, however new classes and files should generally be written in [TypeScript](https://www.typescriptlang.org/). All TypeScript files are generated next to the .ts or .tsx file. So for example, if there's a file "lib/MyClass.ts", there will be a generated "lib/MyClass.js" next to it. It is implemented that way as it requires minimal changes to integrate TypeScript in the existing JavaScript code base.
+
+# Troubleshooting desktop application
+
+## On Linux and macOS
 
 If there's an error `while loading shared libraries: libgconf-2.so.4: cannot open shared object file: No such file or directory`, run `sudo apt-get install libgconf-2-4`
 
-If you get a node-gyp related error you might need to manually install it: `npm install -g node-gyp`.
+If you get a node-gyp related error, you might need to manually install it: `npm install -g node-gyp`.
 
 If you get the error `libtool: unrecognized option '-static'`, follow the instructions [in this post](https://stackoverflow.com/a/38552393/561309) to use the correct libtool version.
 
-That will create the executable file in the `dist` directory.
+## On Windows
 
-From `/ElectronClient` you can also run `run.sh` to run the app for testing.
-
-## Building Electron application on Windows
-
-```
-xcopy /C /I /H /R /Y /S ReactNativeClient\lib ElectronClient\app\lib
-npm run tsc
-cd ElectronClient\app
-npm install
-yarn dist
-```
-
-If node-gyp does not works (MSBUILD: error MSB3428: Could not load the Visual C++ component "VCBuild.exe"), you might need to install the `windows-build-tools` using `npm install --global windows-build-tools`.
+If node-gyp does not work (MSBUILD: error MSB3428: Could not load the Visual C++ component "VCBuild.exe"), you might need to install `windows-build-tools` using `npm install --global windows-build-tools`.
 
 If `yarn dist` fails, it may need administrative rights.
 
 If you get an `error MSB8020: The build tools for v140 cannot be found.` try to run with a different toolset version, eg `npm install --toolset=v141` (See [here](https://github.com/mapbox/node-sqlite3/issues/1124) for more info).
 
-The [building\_win32\_tips on this page](./readme/building_win32_tips.md) might be helpful.
-
-## Troubleshooting desktop application
+## Other issues
 
 > The application window doesn't open or is white
 
@@ -80,34 +107,10 @@ This is an indication that there's an early initialisation error. Try this:
 - In ElectronAppWrapper, set `debugEarlyBugs` to `true`. This will force the window to show up and should open the console next to it, which should display any error.
 - In more rare cases, an already open instance of Joplin can create strange low-level bugs that will display no error but will result in this white window. A non-dev instance of Joplin, or a dev instance that wasn't properly closed might cause this. So make sure you close everything and try again. Perhaps even other Electron apps running (Skype, Slack, etc.) could cause this?
 - Also try to delete node_modules and rebuild.
-- If all else fail, switch your computer off and on again, to make sure you start clean.
+- If all else fails, switch your computer off and on again, to make sure you start clean.
 
 > How to work on the app from Windows?
 
-You should not use WSL at all because this is a GUI app that lives outside of WSL, and the WSL layer can cause all kind of very hard to debug issues. It can also lock files in node_modules that cannot be unlocked when the app crashes (you need to restart your computer). Likewise, don't run the TypeScript watch command from WSL.
+**You should not use WSL at all** because this is a GUI app that lives outside of WSL, and the WSL layer can cause all kind of very hard to debug issues. It can also lock files in node_modules that cannot be unlocked when the app crashes. (You need to restart your computer.) Likewise, don't run the TypeScript watch command from WSL.
 
-So everything should be done from a Windows Command prompt running as Administrator. You can use `run.bat` to run the app in dev mode.
-
-# Building the Mobile application
-
-First you need to setup React Native to build projects with native code. For this, follow the instructions on the [Get Started](https://facebook.github.io/react-native/docs/getting-started.html) tutorial, in the "React Native CLI Quickstart" tab.
-
-Then:
-
-```
-npm run tsc
-cd ReactNativeClient
-npm install
-react-native run-ios
-# Or: react-native run-android
-```
-
-# Building the Terminal application
-
-```
-cd CliClient
-npm install
-./build.sh
-```
-
-Run `run.sh` to start the application for testing.
+So everything should be done from a Windows Command prompt or Windows PowerShell running as Administrator. All build and start commands are designed to work cross-platform, including on Windows.

CONTRIBUTING.md

@@ -7,18 +7,19 @@ The [Joplin Forum](https://discourse.joplinapp.org/) is the community driven pla
 File bugs in the [Github Issue Tracker](https://github.com/laurent22/joplin/issues?utf8=%E2%9C%93&q=is%3Aissue). Please follow these guidelines:
 
 - Search existing issues first, make sure yours hasn't already been reported.
+- Please follow the template.
 - Consider [enabling debug mode](https://joplinapp.org/debugging/) so that you can provide as much details as possible when reporting the issue.
-- Stay on topic, but describe the issue in detail so that others can reproduce it.
+- Stay on topic, but describe the issue in detail so that others can **reproduce** it.
 - **Provide a screenshot** if possible. A screenshot showing the problem is often more useful than a paragraph describing it.
 - For web clipper bugs, **please provide the URL causing the issue**. Sometimes the clipper works in one page but not in another so it is important to know what URL has a problem.
 
 # Feature requests
 
-Please check that your request has not already been posted in the [Github Issue Tracker](https://github.com/laurent22/joplin/issues?utf8=%E2%9C%93&q=is%3Aissue). If it has, **up-voting the issue** increases the chances it'll be noticed and implemented in the future. "+1" comments are not tracked.
+Feature requests **must be opened and discussed on the [forum](https://discourse.joplinapp.org/c/features)**. After they have been accepted, they can be added to the GitHub tracker.
 
-As a general rule, suggestions to *improve Joplin* should be posted first in the [Joplin Forum](https://discourse.joplinapp.org/) for discussion.
+Please check that your request has not already been posted on the forum or the [Github Issue Tracker](https://github.com/laurent22/joplin/issues?utf8=%E2%9C%93&q=is%3Aissue). If it has, **up-voting the issue or topic** increases the chances it'll be noticed and implemented in the future. "+1" comments are not tracked.
 
-Avoid listing multiple requests in one report in the [Github Issue Tracker](https://github.com/laurent22/joplin/issues?utf8=%E2%9C%93&q=is%3Aissue). One issue per request makes it easier to track and discuss it.
+Avoid listing multiple requests in one topic. One topic per request makes it easier to track and discuss it.
 
 Finally, when submitting a pull request, don't forget to [test your code](#unit-tests).
 
@@ -36,39 +37,56 @@ If you want to start contributing to the project's code, please follow these gui
 - A good way to easily start contributing is to pick and work on a [good first issue](https://github.com/laurent22/joplin/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). We try to make these issues as clear as possible and provide basic info on how the code should be changed, and if something is unclear feel free to ask for more information on the issue.
 - Before adding a new feature, ask about it in the [Github Issue Tracker](https://github.com/laurent22/joplin/issues?utf8=%E2%9C%93&q=is%3Aissue) or the [Joplin Forum](https://discourse.joplinapp.org/), or check if existing discussions exist to make sure the new functionality is desired.
 - **Changes that will consist in more than 50 lines of code should be discussed the [Joplin Forum](https://discourse.joplinapp.org/)**, so that you don't spend too much time implementing something that might not be accepted.
+- All the applications share the same backend (database, synchronisation, settings, models, business logic, etc.) so if you change something in the backend in one app, makes sure it still work in the other apps. Usually it does, but keep this in mind.
 
 Building the apps is relatively easy - please [see the build instructions](https://github.com/laurent22/joplin/blob/master/BUILD.md) for more details.
 
-## Coding style
+### Coding style
 
 Coding style is enforced by a pre-commit hook that runs eslint. This hook is installed whenever running `npm install` on any of the application directory. If for some reason the pre-commit hook didn't get installed, you can manually install it by running `npm install` at the root of the repository.
 
-## Unit tests
+For new React components, please use [React Hooks](https://reactjs.org/docs/hooks-intro.html). For new code in general, please use TypeScript (unless you are modifying a file that was originally in JavaScript).
+
+For changes made to the Desktop client that affect the user interface, refer to `ElectronClient/app/theme.js` for all styling information. The goal is to create a consistent user interface to allow for easy navigation of Joplin's various features and improve the overall user experience.
+
+### Unit tests
 
 When submitting a pull request for a new feature or bug fix, please add unit tests for your code. Unit testing GUI changes is not always possible so it is not required, but any change in a file under /lib for example should be unit tested.
 
 The tests are under CliClient/tests. To get them running, you first need to build the CLI app:
 
-    cd CliClient
-    npm i
-
-To run the test units, you must have an instance of the cli app running. In a first window navigate into `CliClient` and run:
-
 ```sh
-./run.sh
+npm run tsc # Build the .ts and .tsx files
+cd CliClient
+npm install
 ```
 
-> If you get an error like `Error: Cannot find module '../locales/index.js'`, this means you must (a) rebuild translations or (b) take > them from one of the other apps. To do option b, you can run the following command to copy them from the `ReactNativeClient` directory:> 
->
-> ```sh
-> cd .. # Return to the root of the project
-> rsync -aP ./ReactNativeClient/locales/ ./CliClient/build/locales/
-> ```
+To run all the test units:
 
-Then run the tests in a second window. To run all the test units:
-
-    ./run_test.sh
+```sh
+npm run test
+```
 
 To run just one particular file:
 
-    ./run_test.sh markdownUtils # Don't add the .js extension
+```sh
+npm run test -- --filter=markdownUtils # Don't add the .js extension
+```
+
+To filter tests. For example, to run all the test units that contain "should handle conflict" in their description:
+
+```sh
+npm run test -- --filter="should handle conflict"
+```
+
+If you get the error `Cannot find module '/joplin/CliClient/node_modules/sqlite3/lib/binding/node-v79-darwin-x64/node_sqlite3.node'`, you may need to run `npm rebuild`.
+
+## About abandoned pull requests
+
+It happens that a pull request is started but not finished and despite our attempts to contact the contributor, we don’t hear from them again.
+
+In that case we will not merge the pull request, even if only small changes are missing. Our policy is simply to close the pull request. Why? Because an unfinished pull request essentially means giving us work and moving on. We would rather not encourage this behaviour.
+
+Also, please note that since we have spent time reviewing the pull request and proposing solutions, we reserve the right to re-use that knowledge to create a new pull request, potentially based on your changes.
+
+We’d much prefer that you complete the pull request though, so we’ll be sure to ping you a few times before that!

CliClient/.gitignore

@@ -1,4 +1,3 @@
-build/
 node_modules/
 app/src
 tests-build/
@@ -20,4 +19,5 @@ tests/sync
 out.txt
 linkToLocal.sh
 yarn-error.log
-tests/support/dropbox-auth.txt
+tests/support/dropbox-auth.txt
+build/

CliClient/app/command-apidoc.js

@@ -131,6 +131,24 @@ class Command extends BaseCommand {
 		lines.push('');
 		lines.push('Call **GET /search?query=YOUR_QUERY** to search for notes. This end-point supports the `field` parameter which is recommended to use so that you only get the data that you need. The query syntax is as described in the main documentation: https://joplinapp.org/#searching');
 		lines.push('');
+		lines.push('To retrieve non-notes items, such as notebooks or tags, add a `type` parameter and set it to the required [item type name](#item-type-id). In that case, full text search will not be used - instead it will be a simple case-insensitive search. You can also use `*` as a wildcard. This is convenient for example to retrieve notebooks or tags by title.');
+		lines.push('');
+		lines.push('For example, to retrieve the notebook named `recipes`: **GET /search?query=recipes&type=folder**');
+		lines.push('');
+		lines.push('To retrieve all the tags that start with `project-`: **GET /search?query=project-*&type=tag**');
+		lines.push('');
+
+		lines.push('# Item type IDs');
+		lines.push('');
+		lines.push('Item type IDs might be refered to in certain object you will retrieve from the API. This is the correspondance between name and ID:');
+		lines.push('');
+		lines.push('Name | Value');
+		lines.push('---- | -----');
+		for (const t of BaseModel.typeEnum_) {
+			const value = t[1];
+			lines.push(`${BaseModel.modelTypeToName(value)} | ${value}   `);
+		}
+		lines.push('');
 
 		for (let i = 0; i < models.length; i++) {
 			const model = models[i];

CliClient/app/command-config.js

@@ -1,6 +1,7 @@
 const { BaseCommand } = require('./base-command.js');
 const { _, setLocale } = require('lib/locale.js');
 const { app } = require('./app.js');
+const fs = require('fs-extra');
 const Setting = require('lib/models/Setting.js');
 
 class Command extends BaseCommand {
@@ -13,11 +14,60 @@ class Command extends BaseCommand {
 	}
 
 	options() {
-		return [['-v, --verbose', _('Also displays unset and hidden config variables.')]];
+		return [
+			['-v, --verbose', _('Also displays unset and hidden config variables.')],
+			['--export', 'Writes all settings to STDOUT as JSON including secure variables.'],
+			['--import', 'Reads in JSON formatted settings from STDIN.'],
+			['--import-file <file>', 'Reads in settings from <file>. <file> must contain valid JSON.'],
+		];
 	}
+	async __importSettings(inputStream) {
+		return new Promise((resolve, reject) => {
+			// being defensive and not attempting to settle twice
+			let isSettled = false;
+			const chunks = [];
 
+			inputStream.on('readable', () => {
+				let chunk;
+				while ((chunk = inputStream.read()) !== null) {
+					chunks.push(chunk);
+				}
+			});
+
+			inputStream.on('end', () => {
+				let json = chunks.join('');
+				let settingsObj;
+				try {
+					settingsObj = JSON.parse(json);
+				} catch (err) {
+					isSettled = true;
+					return reject(new Error(`Invalid JSON passed to config --import: \n${err.message}.`));
+				}
+				if (settingsObj) {
+					Object.entries(settingsObj)
+						.forEach(([key, value]) => {
+							Setting.setValue(key, value);
+						});
+				}
+				if (!isSettled) {
+					isSettled = true;
+					resolve();
+				}
+			});
+
+			inputStream.on('error', (error) => {
+				if (!isSettled) {
+					isSettled = true;
+					reject(error);
+				}
+			});
+		});
+	}
 	async action(args) {
 		const verbose = args.options.verbose;
+		const isExport = args.options.export;
+		const isImport = args.options.import || args.options.importFile;
+		const importFile = args.options.importFile;
 
 		const renderKeyValue = name => {
 			const md = Setting.settingMetadata(name);
@@ -32,35 +82,45 @@ class Command extends BaseCommand {
 			}
 		};
 
-		if (!args.name && !args.value) {
+		if (isExport || (!isImport && !args.value)) {
 			let keys = Setting.keys(!verbose, 'cli');
 			keys.sort();
-			for (let i = 0; i < keys.length; i++) {
-				const value = Setting.value(keys[i]);
-				if (!verbose && !value) continue;
-				this.stdout(renderKeyValue(keys[i]));
+
+			if (isExport) {
+				const resultObj = keys.reduce((acc, key) => {
+					const value = Setting.value(key);
+					if (!verbose && !value) return acc;
+					acc[key] = value;
+					return acc;
+				}, {});
+				// Printing the object in "pretty" format so it's easy to read/edit
+				this.stdout(JSON.stringify(resultObj, null, 2));
+			} else if (!args.name) {
+				for (let i = 0; i < keys.length; i++) {
+					const value = Setting.value(keys[i]);
+					if (!verbose && !value) continue;
+					this.stdout(renderKeyValue(keys[i]));
+				}
+			} else {
+				this.stdout(renderKeyValue(args.name));
 			}
-			app()
-				.gui()
-				.showConsole();
-			app()
-				.gui()
-				.maximizeConsole();
+
+			app().gui().showConsole();
+			app().gui().maximizeConsole();
+
 			return;
 		}
 
-		if (args.name && !args.value) {
-			this.stdout(renderKeyValue(args.name));
-			app()
-				.gui()
-				.showConsole();
-			app()
-				.gui()
-				.maximizeConsole();
-			return;
+		if (isImport) {
+			let fileStream = process.stdin;
+			if (importFile) {
+				fileStream = fs.createReadStream(importFile, { autoClose: true });
+			}
+			await this.__importSettings(fileStream);
+		} else {
+			Setting.setValue(args.name, args.value);
 		}
 
-		Setting.setValue(args.name, args.value);
 
 		if (args.name == 'locale') {
 			setLocale(Setting.value('locale'));

CliClient/app/command-server.js

@@ -25,7 +25,7 @@ class Command extends BaseCommand {
 			info: stdoutFn,
 			warn: stdoutFn,
 			error: stdoutFn,
-		}});
+		} });
 		ClipperServer.instance().setDispatch(() => {});
 		ClipperServer.instance().setLogger(clipperLogger);
 

CliClient/app/gui/FolderListWidget.js

@@ -40,7 +40,7 @@ class FolderListWidget extends ListWidget {
 		let output = 0;
 		while (true) {
 			const folder = BaseModel.byId(folders, folderId);
-			if (!folder.parent_id) return output;
+			if (!folder || !folder.parent_id) return output;
 			output++;
 			folderId = folder.parent_id;
 		}

CliClient/build-doc.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-./build.sh && NODE_PATH=build node build/build-doc.js
+npm run build && NODE_PATH=build node build/build-doc.js

CliClient/build.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-BUILD_DIR="$ROOT_DIR/build"
-
-rsync -a --exclude "node_modules/" "$ROOT_DIR/app/" "$BUILD_DIR/"
-rsync -a --delete "$ROOT_DIR/../ReactNativeClient/lib/" "$BUILD_DIR/lib/"
-rsync -a --delete "$ROOT_DIR/../ReactNativeClient/locales/" "$BUILD_DIR/locales/"
-cp "$ROOT_DIR/package.json" "$BUILD_DIR"
-
-# Don't add TypeScript here or make it silent as output of Cli app must be clean
-# cd $ROOT_DIR/..
-# npm run tsc
-# cd $ROOT_DIR
-
-chmod 755 "$BUILD_DIR/main.js"

CliClient/cli-integration.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-./build.sh && NODE_PATH="build/" node build/cli-integration-tests.js
+npm run build && NODE_PATH="build/" node build/cli-integration-tests.js

CliClient/fuzzing.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-./build.sh && NODE_PATH="build/" node build/fuzzing.js
+npm run build && NODE_PATH="build/" node build/fuzzing.js

CliClient/gulpfile.js

@@ -0,0 +1,48 @@
+const gulp = require('gulp');
+const fs = require('fs-extra');
+const utils = require('../Tools/gulp/utils');
+const tasks = {
+	copyLib: require('../Tools/gulp/tasks/copyLib'),
+};
+
+tasks.build = {
+	fn: async () => {
+		const buildDir = `${__dirname}/build`;
+		await utils.copyDir(`${__dirname}/app`, buildDir, {
+			excluded: ['node_modules'],
+		});
+		await utils.copyDir(`${__dirname}/locales-build`, `${buildDir}/locales`);
+		await utils.copyDir(`${__dirname}/../patches`, `${buildDir}/patches`);
+		await tasks.copyLib.fn();
+		await utils.copyFile(`${__dirname}/package.json`, `${buildDir}/package.json`);
+		await utils.copyFile(`${__dirname}/package-lock.json`, `${buildDir}/package-lock.json`);
+
+		// const packageRaw = await fs.readFile(`${buildDir}/package.json`);
+		// const package = JSON.parse(packageRaw.toString());
+		// package.scripts.postinstall = package.scripts.postinstall.replace(/\.\.\/patches/, './patches');
+		// await fs.writeFile(`${buildDir}/package.json`, JSON.stringify(package, null, 2), 'utf8');
+
+		fs.chmodSync(`${buildDir}/main.js`, 0o755);
+	},
+};
+
+tasks.buildTests = {
+	fn: async () => {
+		const testBuildDir = `${__dirname}/tests-build`;
+
+		await utils.copyDir(`${__dirname}/tests`, testBuildDir, {
+			excluded: [
+				'lib/',
+				'locales/',
+				'node_modules/',
+			],
+		});
+
+		await utils.copyDir(`${__dirname}/../ReactNativeClient/lib`, `${testBuildDir}/lib`);
+		await utils.copyDir(`${__dirname}/../ReactNativeClient/locales`, `${testBuildDir}/locales`);
+		await fs.mkdirp(`${testBuildDir}/data`);
+	},
+};
+
+gulp.task('build', tasks.build.fn);
+gulp.task('buildTests', tasks.buildTests.fn);

CliClient/locales-build/index.js

@@ -12,6 +12,7 @@ locales['el_GR'] = require('./el_GR.json');
 locales['en_US'] = require('./en_US.json');
 locales['eo'] = require('./eo.json');
 locales['es_ES'] = require('./es_ES.json');
+locales['et_EE'] = require('./et_EE.json');
 locales['eu'] = require('./eu.json');
 locales['fa'] = require('./fa.json');
 locales['fr_FR'] = require('./fr_FR.json');
@@ -34,38 +35,39 @@ locales['sv'] = require('./sv.json');
 locales['tr_TR'] = require('./tr_TR.json');
 locales['zh_CN'] = require('./zh_CN.json');
 locales['zh_TW'] = require('./zh_TW.json');
-stats['ar'] = {"percentDone":75};
-stats['eu'] = {"percentDone":42};
-stats['bs_BA'] = {"percentDone":94};
-stats['bg_BG'] = {"percentDone":83};
-stats['ca'] = {"percentDone":66};
-stats['hr_HR'] = {"percentDone":35};
-stats['cs_CZ'] = {"percentDone":96};
-stats['da_DK'] = {"percentDone":93};
-stats['de_DE'] = {"percentDone":96};
+stats['ar'] = {"percentDone":92};
+stats['eu'] = {"percentDone":39};
+stats['bs_BA'] = {"percentDone":85};
+stats['bg_BG'] = {"percentDone":77};
+stats['ca'] = {"percentDone":61};
+stats['hr_HR'] = {"percentDone":32};
+stats['cs_CZ'] = {"percentDone":94};
+stats['da_DK'] = {"percentDone":85};
+stats['de_DE'] = {"percentDone":100};
+stats['et_EE'] = {"percentDone":76};
 stats['en_GB'] = {"percentDone":100};
 stats['en_US'] = {"percentDone":100};
-stats['es_ES'] = {"percentDone":96};
-stats['eo'] = {"percentDone":48};
-stats['fr_FR'] = {"percentDone":96};
-stats['gl_ES'] = {"percentDone":54};
-stats['it_IT'] = {"percentDone":91};
-stats['nl_BE'] = {"percentDone":43};
-stats['nl_NL'] = {"percentDone":91};
-stats['nb_NO'] = {"percentDone":96};
-stats['fa'] = {"percentDone":41};
-stats['pl_PL'] = {"percentDone":82};
-stats['pt_PT'] = {"percentDone":96};
-stats['pt_BR'] = {"percentDone":95};
-stats['ro'] = {"percentDone":42};
-stats['sl_SI'] = {"percentDone":53};
-stats['sv'] = {"percentDone":73};
-stats['tr_TR'] = {"percentDone":89};
-stats['el_GR'] = {"percentDone":96};
-stats['ru_RU'] = {"percentDone":92};
-stats['sr_RS'] = {"percentDone":82};
-stats['zh_CN'] = {"percentDone":90};
-stats['zh_TW'] = {"percentDone":66};
-stats['ja_JP'] = {"percentDone":96};
-stats['ko'] = {"percentDone":93};
+stats['es_ES'] = {"percentDone":95};
+stats['eo'] = {"percentDone":44};
+stats['fr_FR'] = {"percentDone":95};
+stats['gl_ES'] = {"percentDone":50};
+stats['it_IT'] = {"percentDone":97};
+stats['nl_NL'] = {"percentDone":97};
+stats['nl_BE'] = {"percentDone":39};
+stats['nb_NO'] = {"percentDone":89};
+stats['fa'] = {"percentDone":38};
+stats['pl_PL'] = {"percentDone":75};
+stats['pt_PT'] = {"percentDone":91};
+stats['pt_BR'] = {"percentDone":88};
+stats['ro'] = {"percentDone":39};
+stats['sl_SI'] = {"percentDone":49};
+stats['sv'] = {"percentDone":68};
+stats['tr_TR'] = {"percentDone":92};
+stats['el_GR'] = {"percentDone":93};
+stats['ru_RU'] = {"percentDone":95};
+stats['sr_RS'] = {"percentDone":75};
+stats['zh_CN'] = {"percentDone":97};
+stats['zh_TW'] = {"percentDone":91};
+stats['ja_JP'] = {"percentDone":97};
+stats['ko'] = {"percentDone":97};
 module.exports = { locales: locales, stats: stats };

CliClient/package.json

@@ -3,6 +3,12 @@
   "description": "Joplin CLI Client",
   "license": "MIT",
   "author": "Laurent Cozic",
+  "scripts": {
+    "test": "gulp buildTests -L && jasmine --config=tests/support/jasmine.json",
+    "postinstall": "patch-package --patch-dir ./patches && npm run build",
+    "build": "gulp build",
+    "start": "gulp build -L && node 'build/main.js' --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev"
+  },
   "bugs": {
     "url": "https://github.com/laurent22/joplin/issues"
   },
@@ -16,11 +22,12 @@
       2016,
       2017,
       2018,
-      2019
+      2019,
+      2020
     ],
     "owner": "Laurent Cozic"
   },
-  "version": "1.0.150",
+  "version": "1.0.159",
   "bin": {
     "joplin": "./main.js"
   },
@@ -31,6 +38,7 @@
     "app-module-path": "^2.2.0",
     "async-mutex": "^0.1.3",
     "base-64": "^0.1.0",
+    "base64-stream": "^1.0.0",
     "clean-html": "^1.5.0",
     "compare-version": "^0.1.2",
     "diacritics": "^1.3.0",
@@ -38,35 +46,55 @@
     "es6-promise-pool": "^2.5.0",
     "file-uri-to-path": "^1.0.0",
     "follow-redirects": "^1.2.4",
+    "font-awesome-filetypes": "^2.1.0",
     "form-data": "^2.1.4",
     "fs-extra": "^5.0.0",
+    "highlight.js": "^9.17.1",
     "html-entities": "^1.2.1",
     "html-minifier": "^3.5.15",
+    "htmlparser2": "^4.1.0",
     "image-data-uri": "^2.0.0",
     "image-type": "^3.0.0",
     "joplin-turndown": "^4.0.19",
-    "joplin-turndown-plugin-gfm": "^1.0.11",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "json-stringify-safe": "^5.0.1",
     "jssha": "^2.3.0",
+    "katex": "^0.11.1",
     "levenshtein": "^1.0.5",
-    "markdown-it": "^8.4.2",
+    "markdown-it": "^10.0.0",
+    "markdown-it-abbr": "^1.0.4",
+    "markdown-it-anchor": "^5.2.5",
+    "markdown-it-deflist": "^2.0.3",
+    "markdown-it-emoji": "^1.4.0",
+    "markdown-it-expand-tabs": "^1.0.13",
+    "markdown-it-footnote": "^3.0.2",
+    "markdown-it-ins": "^3.0.0",
+    "markdown-it-mark": "^3.0.0",
+    "markdown-it-multimd-table": "^4.0.1",
+    "markdown-it-sub": "^1.0.0",
+    "markdown-it-sup": "^1.0.0",
+    "markdown-it-toc-done-right": "^4.1.0",
     "md5": "^2.2.1",
+    "md5-file": "^4.0.0",
+    "memory-cache": "^0.2.0",
     "mime": "^2.0.3",
     "moment": "^2.24.0",
     "multiparty": "^4.2.1",
     "node-emoji": "^1.8.1",
     "node-fetch": "^1.7.1",
     "node-persist": "^2.1.0",
+    "patch-package": "^6.2.0",
     "promise": "^7.1.1",
     "proper-lockfile": "^2.0.1",
     "query-string": "4.3.4",
     "read-chunk": "^2.1.0",
     "redux": "^3.7.2",
     "request": "^2.88.0",
-    "sax": "^1.2.2",
+    "sax": "^1.2.4",
     "server-destroy": "^1.0.1",
     "sharp": "^0.23.2",
     "sprintf-js": "^1.1.1",
-    "sqlite3": "^4.0.7",
+    "sqlite3": "^4.1.1",
     "string-padding": "^1.0.2",
     "string-to-stream": "^1.1.0",
     "strip-ansi": "^4.0.0",
@@ -76,16 +104,16 @@
     "terminal-kit": "^1.30.0",
     "tkwidgets": "^0.5.26",
     "url-parse": "^1.4.7",
+    "uslug": "^1.0.4",
     "uuid": "^3.0.1",
     "valid-url": "^1.0.9",
     "word-wrap": "^1.2.3",
     "xml2js": "^0.4.19",
-    "yargs-parser": "^7.0.0"
+    "yargs-parser": "^7.0.0",
+    "gulp": "^4.0.2"
   },
   "devDependencies": {
-    "jasmine": "^3.5.0"
-  },
-  "scripts": {
-    "test": "jasmine"
+    "jasmine": "^3.5.0",
+    "temp": "^0.9.1"
   }
 }

CliClient/run.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-set -e
-CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-bash "$CLIENT_DIR/build.sh" && node "$CLIENT_DIR/build/main.js" --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev "$@"
-# bash "$CLIENT_DIR/build.sh" && node "$CLIENT_DIR/build/main.js" --profile ~/.config/joplin --stack-trace-enabled --log-level debug --env dev "$@"

CliClient/run_test.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-START_DIR="$(pwd)"
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-BUILD_DIR="$ROOT_DIR/tests-build"
-FILTER="$1"
-
-rsync -a --exclude "node_modules/" "$ROOT_DIR/tests/" "$BUILD_DIR/"
-rsync -a "$ROOT_DIR/../ReactNativeClient/lib/" "$BUILD_DIR/lib/"
-rsync -a "$ROOT_DIR/../ReactNativeClient/locales/" "$BUILD_DIR/locales/"
-mkdir -p "$BUILD_DIR/data"
-
-function finish {
-	cd "$START_DIR"
-}
-
-trap finish EXIT
-
-cd "$ROOT_DIR"
-
-if [[ $FILTER != "" ]]; then
-	npx jasmine --config=tests/support/jasmine.json --filter="$FILTER"
-else
-	npx jasmine --config=tests/support/jasmine.json
-fi

CliClient/tests/HtmlToHtml.js

@@ -0,0 +1,79 @@
+/* eslint-disable no-unused-vars */
+
+require('app-module-path').addPath(__dirname);
+
+const os = require('os');
+const { time } = require('lib/time-utils.js');
+const { filename } = require('lib/path-utils.js');
+const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
+const Folder = require('lib/models/Folder.js');
+const Note = require('lib/models/Note.js');
+const BaseModel = require('lib/BaseModel.js');
+const { shim } = require('lib/shim');
+const HtmlToHtml = require('lib/joplin-renderer/HtmlToHtml');
+const { enexXmlToMd } = require('lib/import-enex-md-gen.js');
+
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 60 * 60 * 1000; // Can run for a while since everything is in the same test unit
+
+process.on('unhandledRejection', (reason, p) => {
+	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+describe('HtmlToHtml', function() {
+
+	beforeEach(async (done) => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		done();
+	});
+
+	it('should convert from Html to Html', asyncTest(async () => {
+		const basePath = `${__dirname}/html_to_html`;
+		const files = await shim.fsDriver().readDirStats(basePath);
+		const htmlToHtml = new HtmlToHtml();
+
+		for (let i = 0; i < files.length; i++) {
+			const htmlSourceFilename = files[i].path;
+			if (htmlSourceFilename.indexOf('.src.html') < 0) continue;
+
+			const htmlSourceFilePath = `${basePath}/${htmlSourceFilename}`;
+			const htmlDestPath = `${basePath}/${filename(filename(htmlSourceFilePath))}.dest.html`;
+
+			// if (htmlSourceFilename !== 'table_with_header.html') continue;
+
+			const htmlToHtmlOptions = {
+				bodyOnly: true,
+			};
+
+			const sourceHtml = await shim.fsDriver().readFile(htmlSourceFilePath);
+			let expectedHtml = await shim.fsDriver().readFile(htmlDestPath);
+
+			const result = await htmlToHtml.render(sourceHtml, null, htmlToHtmlOptions);
+			let actualHtml = result.html;
+
+			if (os.EOL === '\r\n') {
+				expectedHtml = expectedHtml.replace(/\r\n/g, '\n');
+				actualHtml = actualHtml.replace(/\r\n/g, '\n');
+			}
+
+			if (actualHtml !== expectedHtml) {
+				console.info('');
+				console.info(`Error converting file: ${htmlSourceFilename}`);
+				console.info('--------------------------------- Got:');
+				console.info(actualHtml);
+				console.info('--------------------------------- Raw:');
+				console.info(actualHtml.split('\n'));
+				console.info('--------------------------------- Expected:');
+				console.info(expectedHtml.split('\n'));
+				console.info('--------------------------------------------');
+				console.info('');
+
+				expect(false).toBe(true);
+				// return;
+			} else {
+				expect(true).toBe(true);
+			}
+		}
+	}));
+
+});

CliClient/tests/MdToHtml.js

@@ -0,0 +1,83 @@
+/* eslint-disable no-unused-vars */
+
+require('app-module-path').addPath(__dirname);
+
+const os = require('os');
+const { time } = require('lib/time-utils.js');
+const { filename } = require('lib/path-utils.js');
+const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
+const Folder = require('lib/models/Folder.js');
+const Note = require('lib/models/Note.js');
+const BaseModel = require('lib/BaseModel.js');
+const { shim } = require('lib/shim');
+const MdToHtml = require('lib/joplin-renderer/MdToHtml');
+const { enexXmlToMd } = require('lib/import-enex-md-gen.js');
+
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 60 * 60 * 1000; // Can run for a while since everything is in the same test unit
+
+process.on('unhandledRejection', (reason, p) => {
+	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+describe('MdToHtml', function() {
+
+	beforeEach(async (done) => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		done();
+	});
+
+	it('should convert from Markdown to Html', asyncTest(async () => {
+		const basePath = `${__dirname}/md_to_html`;
+		const files = await shim.fsDriver().readDirStats(basePath);
+		const mdToHtml = new MdToHtml({
+			ResourceModel: {
+				isResourceUrl: () => false,
+			},
+		});
+
+		for (let i = 0; i < files.length; i++) {
+			const mdFilename = files[i].path;
+			if (mdFilename.indexOf('.md') < 0) continue;
+
+			const mdFilePath = `${basePath}/${mdFilename}`;
+			const htmlPath = `${basePath}/${filename(mdFilePath)}.html`;
+
+			// if (mdFilename !== 'sanitize_9.md') continue;
+
+			const mdToHtmlOptions = {
+				bodyOnly: true,
+			};
+
+			const markdown = await shim.fsDriver().readFile(mdFilePath);
+			let expectedHtml = await shim.fsDriver().readFile(htmlPath);
+
+			const result = await mdToHtml.render(markdown, null, mdToHtmlOptions);
+			let actualHtml = result.html;
+
+			if (os.EOL === '\r\n') {
+				expectedHtml = expectedHtml.replace(/\r\n/g, '\n');
+				actualHtml = actualHtml.replace(/\r\n/g, '\n');
+			}
+
+			if (actualHtml !== expectedHtml) {
+				console.info('');
+				console.info(`Error converting file: ${mdFilename}`);
+				console.info('--------------------------------- Got:');
+				console.info(actualHtml);
+				console.info('--------------------------------- Raw:');
+				console.info(actualHtml.split('\n'));
+				console.info('--------------------------------- Expected:');
+				console.info(expectedHtml.split('\n'));
+				console.info('--------------------------------------------');
+				console.info('');
+
+				expect(false).toBe(true);
+				// return;
+			} else {
+				expect(true).toBe(true);
+			}
+		}
+	}));
+
+});

CliClient/tests/MdToMd.js

@@ -0,0 +1,41 @@
+const mdImporterService = require('lib/services/InteropService_Importer_Md');
+const Note = require('lib/models/Note.js');
+const { setupDatabaseAndSynchronizer, switchClient } = require('test-utils.js');
+
+const importer = new mdImporterService();
+
+
+describe('InteropService_Importer_Md: importLocalImages', function() {
+	beforeEach(async (done) => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		done();
+	});
+	it('should import linked files and modify tags appropriately', async function() {
+		const tagNonExistentFile = '![does not exist](does_not_exist.png)';
+		const note = await importer.importFile(`${__dirname}/md_to_md/sample.md`, 'notebook');
+		let items = await Note.linkedItems(note.body);
+		expect(items.length).toBe(2);
+		const inexistentLinkUnchanged = note.body.includes(tagNonExistentFile);
+		expect(inexistentLinkUnchanged).toBe(true);
+	});
+	it('should only create 1 resource for duplicate links, all tags should be updated', async function() {
+		const note = await importer.importFile(`${__dirname}/md_to_md/sample-duplicate-links.md`, 'notebook');
+		let items = await Note.linkedItems(note.body);
+		expect(items.length).toBe(1);
+		const reg = new RegExp(items[0].id, 'g');
+		const matched = note.body.match(reg);
+		expect(matched.length).toBe(2);
+	});
+	it('should import linked files and modify tags appropriately when link is also in alt text', async function() {
+		const note = await importer.importFile(`${__dirname}/md_to_md/sample-link-in-alt-text.md`, 'notebook');
+		let items = await Note.linkedItems(note.body);
+		expect(items.length).toBe(1);
+	});
+	it('should passthrough unchanged if no links present', async function() {
+		const note = await importer.importFile(`${__dirname}/md_to_md/sample-no-links.md`, 'notebook');
+		let items = await Note.linkedItems(note.body);
+		expect(items.length).toBe(0);
+		expect(note.body).toContain('Unidentified vessel travelling at sub warp speed, bearing 235.7. Fluctuations in energy readings from it, Captain. All transporters off.');
+	});
+});

CliClient/tests/encryption.js

@@ -41,7 +41,7 @@ describe('Encryption', function() {
 		};
 
 		const encodedHeader = service.encodeHeader_(header);
-		const decodedHeader = service.decodeHeader_(encodedHeader);
+		const decodedHeader = service.decodeHeaderBytes_(encodedHeader);
 		delete decodedHeader.length;
 
 		expect(objectsEqual(header, decodedHeader)).toBe(true);
@@ -54,14 +54,14 @@ describe('Encryption', function() {
 
 		let hasThrown = false;
 		try {
-			await service.decryptMasterKey(masterKey, 'wrongpassword');
+			await service.decryptMasterKey_(masterKey, 'wrongpassword');
 		} catch (error) {
 			hasThrown = true;
 		}
 
 		expect(hasThrown).toBe(true);
 
-		const decryptedMasterKey = await service.decryptMasterKey(masterKey, '123456');
+		const decryptedMasterKey = await service.decryptMasterKey_(masterKey, '123456');
 		expect(decryptedMasterKey.length).toBe(512);
 	}));
 
@@ -69,7 +69,7 @@ describe('Encryption', function() {
 		let masterKey = await service.generateMasterKey('123456');
 		masterKey = await MasterKey.save(masterKey);
 
-		await service.loadMasterKey(masterKey, '123456', true);
+		await service.loadMasterKey_(masterKey, '123456', true);
 
 		const cipherText = await service.encryptString('some secret');
 		const plainText = await service.decryptString(cipherText);
@@ -87,11 +87,37 @@ describe('Encryption', function() {
 		expect(plainText2 === veryLongSecret).toBe(true);
 	}));
 
+	it('should decrypt various encryption methods', asyncTest(async () => {
+		let masterKey = await service.generateMasterKey('123456');
+		masterKey = await MasterKey.save(masterKey);
+		await service.loadMasterKey_(masterKey, '123456', true);
+
+		{
+			const cipherText = await service.encryptString('some secret', {
+				encryptionMethod: EncryptionService.METHOD_SJCL_2,
+			});
+			const plainText = await service.decryptString(cipherText);
+			expect(plainText).toBe('some secret');
+			const header = await service.decodeHeaderString(cipherText);
+			expect(header.encryptionMethod).toBe(EncryptionService.METHOD_SJCL_2);
+		}
+
+		{
+			const cipherText = await service.encryptString('some secret', {
+				encryptionMethod: EncryptionService.METHOD_SJCL_3,
+			});
+			const plainText = await service.decryptString(cipherText);
+			expect(plainText).toBe('some secret');
+			const header = await service.decodeHeaderString(cipherText);
+			expect(header.encryptionMethod).toBe(EncryptionService.METHOD_SJCL_3);
+		}
+	}));
+
 	it('should fail to decrypt if master key not present', asyncTest(async () => {
 		let masterKey = await service.generateMasterKey('123456');
 		masterKey = await MasterKey.save(masterKey);
 
-		await service.loadMasterKey(masterKey, '123456', true);
+		await service.loadMasterKey_(masterKey, '123456', true);
 
 		const cipherText = await service.encryptString('some secret');
 
@@ -107,7 +133,7 @@ describe('Encryption', function() {
 		let masterKey = await service.generateMasterKey('123456');
 		masterKey = await MasterKey.save(masterKey);
 
-		await service.loadMasterKey(masterKey, '123456', true);
+		await service.loadMasterKey_(masterKey, '123456', true);
 
 		let cipherText = await service.encryptString('some secret');
 		cipherText += 'ABCDEFGHIJ';
@@ -120,7 +146,7 @@ describe('Encryption', function() {
 	it('should encrypt and decrypt notes and folders', asyncTest(async () => {
 		let masterKey = await service.generateMasterKey('123456');
 		masterKey = await MasterKey.save(masterKey);
-		await service.loadMasterKey(masterKey, '123456', true);
+		await service.loadMasterKey_(masterKey, '123456', true);
 
 		let folder = await Folder.save({ title: 'folder' });
 		let note = await Note.save({ title: 'encrypted note', body: 'something', parent_id: folder.id });
@@ -151,7 +177,7 @@ describe('Encryption', function() {
 	it('should encrypt and decrypt files', asyncTest(async () => {
 		let masterKey = await service.generateMasterKey('123456');
 		masterKey = await MasterKey.save(masterKey);
-		await service.loadMasterKey(masterKey, '123456', true);
+		await service.loadMasterKey_(masterKey, '123456', true);
 
 		const sourcePath = `${__dirname}/../tests/support/photo.jpg`;
 		const encryptedPath = `${__dirname}/data/photo.crypted`;
@@ -164,4 +190,52 @@ describe('Encryption', function() {
 		expect(fileContentEqual(sourcePath, decryptedPath)).toBe(true);
 	}));
 
+	it('should encrypt invalid UTF-8 data', asyncTest(async () => {
+		let masterKey = await service.generateMasterKey('123456');
+		masterKey = await MasterKey.save(masterKey);
+
+		await service.loadMasterKey_(masterKey, '123456', true);
+
+		// First check that we can replicate the error with the old encryption method
+		service.defaultEncryptionMethod_ = EncryptionService.METHOD_SJCL;
+		const hasThrown = await checkThrowAsync(async () => await service.encryptString('🐶🐶🐶'.substr(0,5)));
+		expect(hasThrown).toBe(true);
+
+		// Now check that the new one fixes the problem
+		service.defaultEncryptionMethod_ = EncryptionService.METHOD_SJCL_1A;
+		const cipherText = await service.encryptString('🐶🐶🐶'.substr(0,5));
+		const plainText = await service.decryptString(cipherText);
+		expect(plainText).toBe('🐶🐶🐶'.substr(0,5));
+	}));
+
+	// it('should upgrade master key encryption mode', asyncTest(async () => {
+	// 	let masterKey = await service.generateMasterKey('123456', {
+	// 		encryptionMethod: EncryptionService.METHOD_SJCL_2,
+	// 	});
+	// 	masterKey = await MasterKey.save(masterKey);
+	// 	Setting.setObjectKey('encryption.passwordCache', masterKey.id, '123456');
+	// 	Setting.setValue('encryption.activeMasterKeyId', masterKey.id);
+
+	// 	await sleep(0.01);
+
+	// 	await service.loadMasterKeysFromSettings();
+
+	// 	masterKeyNew = await MasterKey.load(masterKey.id);
+
+	// 	// Check that the master key has been upgraded
+
+	// 	expect(masterKeyNew.created_time).toBe(masterKey.created_time);
+	// 	expect(masterKeyNew.updated_time === masterKey.updated_time).toBe(false);
+	// 	expect(masterKeyNew.content === masterKey.content).toBe(false);
+	// 	expect(masterKeyNew.encryption_method === masterKey.encryption_method).toBe(false);
+	// 	expect(masterKeyNew.checksum === masterKey.checksum).toBe(false);
+	// 	expect(masterKeyNew.encryption_method).toBe(service.defaultMasterKeyEncryptionMethod_);
+
+	// 	// Check that encryption still works
+
+	// 	const cipherText = await service.encryptString('some secret');
+	// 	const plainText = await service.decryptString(cipherText);
+	// 	expect(plainText).toBe('some secret');
+	// }));
+
 });