self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-11-24 08:12:24 +02:00
Code Issues Releases Activity

Desktop, Cli: Fixes #5543: Fixed running out of memory when importing large ENEX files

Browse Source
This commit is contained in:
Laurent Cozic 2021-10-13 10:55:39 +01:00
parent acc5959284
commit 3d87e89753
1 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
packages/lib/import-enex.ts
View File

@ -301,28 +301,38 @@ interface NoteResourceRecognition {
} }
const preProcessFile = async (filePath: string): Promise<string> => { const preProcessFile = async (filePath: string): Promise<string> => {
const content: string = await shim.fsDriver().readFile(filePath, 'utf8'); // Disabled pre-processing for now because it runs out of memory:
// https://github.com/laurent22/joplin/issues/5543
//
// It could be fixed by not loading the whole file in memory, but there are
// other issues because people import 1GB+ files so pre-processing
// everything means creating a new copy of that file, and that has its own
// problems.
// The note content in an ENEX file is wrapped in a CDATA block so it means return filePath;
// that any "]]>" inside the note must be somehow escaped, or else the CDATA
// block would be closed at the wrong point. // const content: string = await shim.fsDriver().readFile(filePath, 'utf8');
//
// The problem is that Evernote appears to encode "]]>" as "]]<![CDATA[>]]>" // // The note content in an ENEX file is wrapped in a CDATA block so it means
// instead of the more sensible "]]&gt;", or perhaps they have nothing in // // that any "]]>" inside the note must be somehow escaped, or else the CDATA
// place to properly escape data imported from their web clipper. In any // // block would be closed at the wrong point.
// case it results in invalid XML that Evernote cannot even import back. // //
// // // The problem is that Evernote appears to encode "]]>" as "]]<![CDATA[>]]>"
// Handling that invalid XML with SAX would also be very tricky, so instead // // instead of the more sensible "]]&gt;", or perhaps they have nothing in
// we add a pre-processing step that converts this tags to just "&gt;". It // // place to properly escape data imported from their web clipper. In any
// should be safe to do so because such content can only be within the body // // case it results in invalid XML that Evernote cannot even import back.
// of a note - and ">" or "&gt;" is equivalent. // //
// // // Handling that invalid XML with SAX would also be very tricky, so instead
// Ref: https://discourse.joplinapp.org/t/20470/4 // // we add a pre-processing step that converts this tags to just "&gt;". It
const newContent = content.replace(/<!\[CDATA\[>\]\]>/g, '&gt;'); // // should be safe to do so because such content can only be within the body
if (content === newContent) return filePath; // // of a note - and ">" or "&gt;" is equivalent.
const newFilePath = `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.enex`; // //
await shim.fsDriver().writeFile(newFilePath, newContent, 'utf8'); // // Ref: https://discourse.joplinapp.org/t/20470/4
return newFilePath; // const newContent = content.replace(/<!\[CDATA\[>\]\]>/g, '&gt;');
// if (content === newContent) return filePath;
// const newFilePath = `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.enex`;
// await shim.fsDriver().writeFile(newFilePath, newContent, 'utf8');
// return newFilePath;
}; };
export default async function importEnex(parentFolderId: string, filePath: string, importOptions: ImportOptions = null) { export default async function importEnex(parentFolderId: string, filePath: string, importOptions: ImportOptions = null) {