From 09c1d58358d0ba07b5e8c33cacd983bfad434415 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 26 Jun 2017 19:03:01 +0000 Subject: [PATCH] Fixed issue with enex import and added progress info --- CliClient/app/import-enex.js | 65 ++++++++++++++++++++++++++---------- CliClient/app/main.js | 13 +++++++- CliClient/build.sh | 3 +- CliClient/run.sh | 2 +- 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index d99b13b27..5350e8801 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -132,10 +132,17 @@ async function saveNoteToStorage(note, fuzzyMatching = false) { function importEnex(parentFolderId, filePath, importOptions = null) { if (!importOptions) importOptions = {}; if (!('fuzzyMatching' in importOptions)) importOptions.fuzzyMatching = false; - - let stream = fs.createReadStream(filePath); + if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {}; + if (!('onError' in importOptions)) importOptions.onError = function(error) {}; return new Promise((resolve, reject) => { + let progressState = { + loaded: 0, + imported: 0, + }; + + let stream = fs.createReadStream(filePath); + let options = {}; let strict = true; let saxStream = require('sax').createStream(strict, options); @@ -147,9 +154,10 @@ function importEnex(parentFolderId, filePath, importOptions = null) { let noteResourceAttributes = null; let noteResourceRecognition = null; let notes = []; + let processingNotes = false; stream.on('error', (error) => { - reject(new Error(error.toString())); + importOptions.onError(error); }); function currentNodeName() { @@ -162,7 +170,11 @@ function importEnex(parentFolderId, filePath, importOptions = null) { return nodes[nodes.length - 1].attributes; } - function processNotes() { + async function processNotes() { + if (processingNotes) return; + processingNotes = true; + stream.pause(); + let chain = []; while (notes.length) { let note = notes.shift(); @@ -176,16 +188,22 @@ function importEnex(parentFolderId, filePath, importOptions = null) { note.body = body; return saveNoteToStorage(note, importOptions.fuzzyMatching); + }).then(() => { + progressState.imported++; + importOptions.onProgress(progressState); }); }); } - return promiseChain(chain); + return promiseChain(chain).then(() => { + stream.resume(); + processingNotes = false; + }); } - saxStream.on('error', function(e) { - reject(new Error(e.toString())); - }) + saxStream.on('error', (error) => { + importOptions.onError(error); + }); saxStream.on('text', function(text) { let n = currentNodeName(); @@ -253,13 +271,14 @@ function importEnex(parentFolderId, filePath, importOptions = null) { if (n == 'note') { note = removeUndefinedProperties(note); + progressState.loaded++; + importOptions.onProgress(progressState); + notes.push(note); + if (notes.length >= 10) { - stream.pause(); - processNotes().then(() => { - stream.resume(); - }).catch((error) => { - console.error('Error processing note', error); + processNotes().catch((error) => { + importOptions.onError(error); }); } note = null; @@ -293,10 +312,14 @@ function importEnex(parentFolderId, filePath, importOptions = null) { } else if (n == 'resource') { let decodedData = null; if (noteResource.dataEncoding == 'base64') { - decodedData = Buffer.from(noteResource.data, 'base64'); + try { + decodedData = Buffer.from(noteResource.data, 'base64'); + } catch (error) { + importOptions.onError(error); + } } else { - reject('Cannot decode resource with encoding: ' + noteResource.dataEncoding); - return; + importOptions.onError(new Error('Cannot decode resource with encoding: ' + noteResource.dataEncoding)); + decodedData = noteResource.data; // Just put the encoded data directly in the file so it can, potentially, be manually decoded later } let r = { @@ -313,7 +336,15 @@ function importEnex(parentFolderId, filePath, importOptions = null) { }); saxStream.on('end', function() { - processNotes().then(() => { resolve(); }); + // Wait till there is no more notes to process. + let iid = setInterval(() => { + if (notes.length) { + processNotes(); + } else { + clearInterval(iid); + resolve(); + } + }, 500); }); stream.pipe(saxStream); diff --git a/CliClient/app/main.js b/CliClient/app/main.js index 19121d2d5..6ae44896b 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -344,14 +344,25 @@ commands.push({ return; } + let redrawnCalled = false; let options = { fuzzyMatching: args.options['fuzzy-matching'] === true, + onProgress: (progressState) => { + let line = sprintf(_('Found: %d. Imported: %d.', progressState.loaded, progressState.imported)); + redrawnCalled = true; + vorpal.ui.redraw(line); + }, + onError: (error) => { + let s = error.trace ? error.trace : error.toString(); + this.log(s); + }, } folder = !folder ? await Folder.save({ title: folderTitle }) : folder; this.log(_('Importing notes...')); await importEnex(folder.id, filePath, options); - this.log(_('The notes have been imported into "%s"', folderTitle)); + if (redrawnCalled) vorpal.ui.redraw.done(); + this.log(_('Done.')); end(); }, diff --git a/CliClient/build.sh b/CliClient/build.sh index 7177c1353..890473891 100644 --- a/CliClient/build.sh +++ b/CliClient/build.sh @@ -3,4 +3,5 @@ CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" rm -f "$CLIENT_DIR/app/lib" ln -s "$CLIENT_DIR/../lib" "$CLIENT_DIR/app" -npm run build \ No newline at end of file +npm run build +cp "$CLIENT_DIR/package.json" "$CLIENT_DIR/build" \ No newline at end of file diff --git a/CliClient/run.sh b/CliClient/run.sh index 474ab71a9..cb0652648 100755 --- a/CliClient/run.sh +++ b/CliClient/run.sh @@ -2,4 +2,4 @@ set -e CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" bash $CLIENT_DIR/build.sh -NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestJoplin \ No newline at end of file +NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes \ No newline at end of file