From 3d87e897537b3d298b0b0583307847925674f1bc Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 13 Oct 2021 10:55:39 +0100 Subject: [PATCH 1/5] Desktop, Cli: Fixes #5543: Fixed running out of memory when importing large ENEX files --- packages/lib/import-enex.ts | 52 ++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/lib/import-enex.ts b/packages/lib/import-enex.ts index 597d4b168..2b573895a 100644 --- a/packages/lib/import-enex.ts +++ b/packages/lib/import-enex.ts @@ -301,28 +301,38 @@ interface NoteResourceRecognition { } const preProcessFile = async (filePath: string): Promise => { - const content: string = await shim.fsDriver().readFile(filePath, 'utf8'); + // Disabled pre-processing for now because it runs out of memory: + // https://github.com/laurent22/joplin/issues/5543 + // + // It could be fixed by not loading the whole file in memory, but there are + // other issues because people import 1GB+ files so pre-processing + // everything means creating a new copy of that file, and that has its own + // problems. - // The note content in an ENEX file is wrapped in a CDATA block so it means - // that any "]]>" inside the note must be somehow escaped, or else the CDATA - // block would be closed at the wrong point. - // - // The problem is that Evernote appears to encode "]]>" as "]]]]>" - // instead of the more sensible "]]>", or perhaps they have nothing in - // place to properly escape data imported from their web clipper. In any - // case it results in invalid XML that Evernote cannot even import back. - // - // Handling that invalid XML with SAX would also be very tricky, so instead - // we add a pre-processing step that converts this tags to just ">". It - // should be safe to do so because such content can only be within the body - // of a note - and ">" or ">" is equivalent. - // - // Ref: https://discourse.joplinapp.org/t/20470/4 - const newContent = content.replace(/\]\]>/g, '>'); - if (content === newContent) return filePath; - const newFilePath = `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.enex`; - await shim.fsDriver().writeFile(newFilePath, newContent, 'utf8'); - return newFilePath; + return filePath; + + // const content: string = await shim.fsDriver().readFile(filePath, 'utf8'); + + // // The note content in an ENEX file is wrapped in a CDATA block so it means + // // that any "]]>" inside the note must be somehow escaped, or else the CDATA + // // block would be closed at the wrong point. + // // + // // The problem is that Evernote appears to encode "]]>" as "]]]]>" + // // instead of the more sensible "]]>", or perhaps they have nothing in + // // place to properly escape data imported from their web clipper. In any + // // case it results in invalid XML that Evernote cannot even import back. + // // + // // Handling that invalid XML with SAX would also be very tricky, so instead + // // we add a pre-processing step that converts this tags to just ">". It + // // should be safe to do so because such content can only be within the body + // // of a note - and ">" or ">" is equivalent. + // // + // // Ref: https://discourse.joplinapp.org/t/20470/4 + // const newContent = content.replace(/\]\]>/g, '>'); + // if (content === newContent) return filePath; + // const newFilePath = `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.enex`; + // await shim.fsDriver().writeFile(newFilePath, newContent, 'utf8'); + // return newFilePath; }; export default async function importEnex(parentFolderId: string, filePath: string, importOptions: ImportOptions = null) { From 523382ec8333d26af364d3ac4b77276b90115a45 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 13 Oct 2021 10:58:10 +0100 Subject: [PATCH 2/5] Desktop release v2.4.10 --- packages/app-desktop/package-lock.json | 4 ++-- packages/app-desktop/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/app-desktop/package-lock.json b/packages/app-desktop/package-lock.json index 842bec0ee..1a6c1a0a3 100644 --- a/packages/app-desktop/package-lock.json +++ b/packages/app-desktop/package-lock.json @@ -1,12 +1,12 @@ { "name": "@joplin/app-desktop", - "version": "2.4.9", + "version": "2.4.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@joplin/app-desktop", - "version": "2.4.9", + "version": "2.4.10", "license": "MIT", "dependencies": { "@fortawesome/fontawesome-free": "^5.13.0", diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index 74a3f055e..1d16a47b6 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -1,6 +1,6 @@ { "name": "@joplin/app-desktop", - "version": "2.4.9", + "version": "2.4.10", "description": "Joplin for Desktop", "main": "main.js", "private": true, From eca6e2ff3d4a0ef01895456484cdecf449a06cb2 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 13 Oct 2021 12:51:15 +0100 Subject: [PATCH 3/5] Desktop release v2.4.11 --- packages/app-desktop/package-lock.json | 4 ++-- packages/app-desktop/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/app-desktop/package-lock.json b/packages/app-desktop/package-lock.json index 1a6c1a0a3..dbc93e524 100644 --- a/packages/app-desktop/package-lock.json +++ b/packages/app-desktop/package-lock.json @@ -1,12 +1,12 @@ { "name": "@joplin/app-desktop", - "version": "2.4.10", + "version": "2.4.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@joplin/app-desktop", - "version": "2.4.10", + "version": "2.4.11", "license": "MIT", "dependencies": { "@fortawesome/fontawesome-free": "^5.13.0", diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index 1d16a47b6..695c2096b 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -1,6 +1,6 @@ { "name": "@joplin/app-desktop", - "version": "2.4.10", + "version": "2.4.11", "description": "Joplin for Desktop", "main": "main.js", "private": true, From 46786cc1867e88925bdb31235b489d43c381a5ea Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 13 Oct 2021 17:42:49 +0100 Subject: [PATCH 4/5] Tools: Update Electron Builder time server (previous was not working anymore) --- packages/app-desktop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index 695c2096b..06bec5c9a 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -37,7 +37,7 @@ "asar": true, "asarUnpack": "./node_modules/node-notifier/vendor/**", "win": { - "rfc3161TimeStampServer": "http://timestamp.comodoca.com/rfc3161", + "rfc3161TimeStampServer": "http://sha256timestamp.ws.symantec.com/sha256/timestamp", "icon": "../../Assets/ImageSources/Joplin.ico", "target": [ { From b45585dc215bc6dbe62e64ddd47de1626639c9a2 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 13 Oct 2021 17:43:03 +0100 Subject: [PATCH 5/5] Desktop release v2.4.12 --- packages/app-desktop/package-lock.json | 4 ++-- packages/app-desktop/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/app-desktop/package-lock.json b/packages/app-desktop/package-lock.json index dbc93e524..bbea97e37 100644 --- a/packages/app-desktop/package-lock.json +++ b/packages/app-desktop/package-lock.json @@ -1,12 +1,12 @@ { "name": "@joplin/app-desktop", - "version": "2.4.11", + "version": "2.4.12", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@joplin/app-desktop", - "version": "2.4.11", + "version": "2.4.12", "license": "MIT", "dependencies": { "@fortawesome/fontawesome-free": "^5.13.0", diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index 06bec5c9a..c9b3a5a77 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -1,6 +1,6 @@ { "name": "@joplin/app-desktop", - "version": "2.4.11", + "version": "2.4.12", "description": "Joplin for Desktop", "main": "main.js", "private": true,