Support importing JEX and raw data

CliClient/app/app.js

@@ -387,10 +387,11 @@ class Application extends BaseApplication {
 				await this.execCommand(argv);
 			} catch (error) {
 				if (this.showStackTraces_) {
-					console.info(error);
+					console.error(error);
 				} else {
 					console.info(error.message);
 				}
+				process.exit(1);
 			}
 		} else { // Otherwise open the GUI
 			this.initRedux();

CliClient/run_test.sh

@@ -9,7 +9,7 @@ rsync -a "$ROOT_DIR/build/locales/" "$BUILD_DIR/locales/"
 mkdir -p "$BUILD_DIR/data"
 
 if [[ $TEST_FILE == "" ]]; then
-	(cd "$ROOT_DIR" && npm test tests-build/synchronizer.js tests-build/encryption.js tests-build/ArrayUtils.js tests-build/models_Setting.js)
+	(cd "$ROOT_DIR" && npm test tests-build/synchronizer.js tests-build/encryption.js tests-build/ArrayUtils.js tests-build/models_Setting.js tests-build/services_InteropService.js)
 else
 	(cd "$ROOT_DIR" && npm test tests-build/$TEST_FILE.js)
 fi

CliClient/tests/services_InteropService.js

@@ -0,0 +1,212 @@
+require('app-module-path').addPath(__dirname);
+
+const { time } = require('lib/time-utils.js');
+const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
+const InteropService = require('lib/services/InteropService.js');
+const Folder = require('lib/models/Folder.js');
+const Note = require('lib/models/Note.js');
+const Tag = require('lib/models/Tag.js');
+const NoteTag = require('lib/models/NoteTag.js');
+const Resource = require('lib/models/Resource.js');
+const fs = require('fs-extra');
+const ArrayUtils = require('lib/ArrayUtils');
+const ObjectUtils = require('lib/ObjectUtils');
+const { shim } = require('lib/shim.js');
+
+process.on('unhandledRejection', (reason, p) => {
+	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+function exportDir() {
+	return __dirname + '/export';
+}
+
+function fieldsEqual(model1, model2, fieldNames) {
+	for (let i = 0; i < fieldNames.length; i++) {
+		const f = fieldNames[i];
+		expect(model1[f]).toBe(model2[f], 'For key ' + f);
+	}
+}
+
+describe('services_InteropService', function() {
+
+	beforeEach(async (done) => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+
+		const dir = exportDir();
+		await fs.remove(dir);
+		await fs.mkdirp(dir);
+		done();
+	});
+
+	it('should export and import folders', asyncTest(async () => {
+		const service = new InteropService();
+		let folder1 = await Folder.save({ title: "folder1" });
+		folder1 = await Folder.load(folder1.id);
+		const filePath = exportDir() + '/test.jex';
+
+		await service.export({ path: filePath });
+
+		await Folder.delete(folder1.id);
+
+		await service.import({ path: filePath });
+
+		// Check that a new folder, with a new ID, has been created
+
+		expect(await Folder.count()).toBe(1);
+		let folder2 = (await Folder.all())[0];
+		expect(folder2.id).not.toBe(folder1.id);
+		expect(folder2.title).toBe(folder1.title);
+
+		await service.import({ path: filePath });
+
+		// As there was already a folder with the same title, check that the new one has been renamed
+
+		await Folder.delete(folder2.id);
+		let folder3 = (await Folder.all())[0];
+		expect(await Folder.count()).toBe(1);
+		expect(folder3.title).not.toBe(folder2.title);
+
+		let fieldNames = Folder.fieldNames();
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'id');
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'title');
+
+		fieldsEqual(folder3, folder1, fieldNames);
+	}));
+
+	it('should export and import folders and notes', asyncTest(async () => {
+		const service = new InteropService();
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		note1 = await Note.load(note1.id);
+		const filePath = exportDir() + '/test.jex';
+
+		await service.export({ path: filePath });
+
+		await Folder.delete(folder1.id);
+		await Note.delete(note1.id);
+
+		await service.import({ path: filePath });
+
+		expect(await Note.count()).toBe(1);
+		let note2 = (await Note.all())[0];
+		let folder2 = (await Folder.all())[0];
+
+		expect(note1.parent_id).not.toBe(note2.parent_id);
+		expect(note1.id).not.toBe(note2.id);
+		expect(note2.parent_id).toBe(folder2.id);
+
+		let fieldNames = Note.fieldNames();
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'id');
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'parent_id');
+
+		fieldsEqual(note1, note2, fieldNames);
+
+		await service.import({ path: filePath });
+
+		note2 = (await Note.all())[0];
+		let note3 = (await Note.all())[1];
+
+		expect(note2.id).not.toBe(note3.id);
+		expect(note2.parent_id).not.toBe(note3.parent_id);
+
+		fieldsEqual(note2, note3, fieldNames);
+	}));
+
+	it('should export and import notes to specific folder', asyncTest(async () => {
+		const service = new InteropService();
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		note1 = await Note.load(note1.id);
+		const filePath = exportDir() + '/test.jex';
+
+		await service.export({ path: filePath });
+
+		await Note.delete(note1.id);
+
+		await service.import({ path: filePath, destinationFolderId: folder1.id });
+
+		expect(await Note.count()).toBe(1);
+		expect(await Folder.count()).toBe(1);
+
+		expect(await checkThrowAsync(async () => await service.import({ path: filePath, destinationFolderId: 'oops' }))).toBe(true);
+	}));
+
+	it('should export and import tags', asyncTest(async () => {
+		const service = new InteropService();
+		const filePath = exportDir() + '/test.jex';
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		let tag1 = await Tag.save({ title: 'mon tag' });
+		tag1 = await Tag.load(tag1.id);
+		await Tag.addNote(tag1.id, note1.id);
+
+		await service.export({ path: filePath });
+
+		await Folder.delete(folder1.id);
+		await Note.delete(note1.id);
+		await Tag.delete(tag1.id);
+
+		await service.import({ path: filePath });
+
+		expect(await Tag.count()).toBe(1);
+		let tag2 = (await Tag.all())[0];
+		let note2 = (await Note.all())[0];
+		expect(tag1.id).not.toBe(tag2.id);
+
+		let fieldNames = Note.fieldNames();
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'id');
+		fieldsEqual(tag1, tag2, fieldNames);
+
+		let noteIds = await Tag.noteIds(tag2.id);
+		expect(noteIds.length).toBe(1);
+		expect(noteIds[0]).toBe(note2.id);
+
+		await service.import({ path: filePath });
+
+		// If importing again, no new tag should be created as one with
+		// the same name already existed. The newly imported note should
+		// however go under that already existing tag.
+		expect(await Tag.count()).toBe(1);
+		noteIds = await Tag.noteIds(tag2.id);
+		expect(noteIds.length).toBe(2);
+	}));
+
+	it('should export and import resources', asyncTest(async () => {
+		const service = new InteropService();
+		const filePath = exportDir() + '/test.jex';
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
+		note1 = await Note.load(note1.id);
+		let resourceIds = Note.linkedResourceIds(note1.body);
+		let resource1 = await Resource.load(resourceIds[0]);
+
+		await service.export({ path: filePath });
+		
+		await Note.delete(note1.id);
+
+		await service.import({ path: filePath });
+
+		expect(await Resource.count()).toBe(2);
+
+		let note2 = (await Note.all())[0];
+		expect(note2.body).not.toBe(note1.body);
+		resourceIds = Note.linkedResourceIds(note2.body);
+		expect(resourceIds.length).toBe(1);
+		let resource2 = await Resource.load(resourceIds[0]);
+		expect(resource2.id).not.toBe(resource1.id);
+
+		let fieldNames = Note.fieldNames();
+		fieldNames = ArrayUtils.removeElement(fieldNames, 'id');
+		fieldsEqual(resource1, resource2, fieldNames);
+
+		const resourcePath1 = Resource.fullPath(resource1);
+		const resourcePath2 = Resource.fullPath(resource2);
+
+		expect(resourcePath1).not.toBe(resourcePath2);
+		expect(fileContentEqual(resourcePath1, resourcePath2)).toBe(true);
+	}));
+
+});

ReactNativeClient/lib/ArrayUtils.js

@@ -15,7 +15,6 @@ ArrayUtils.removeElement = function(array, element) {
 
 // https://stackoverflow.com/a/10264318/561309
 ArrayUtils.binarySearch = function(items, value) {
-
 	var startIndex  = 0,
 		stopIndex   = items.length - 1,
 		middle      = Math.floor((stopIndex + startIndex)/2);
@@ -37,4 +36,13 @@ ArrayUtils.binarySearch = function(items, value) {
 	return (items[middle] != value) ? -1 : middle;
 }
 
+ArrayUtils.findByKey = function(array, key, value) {
+	for (let i = 0; i < array.length; i++) {
+		const o = array[i];
+		if (typeof o !== 'object') continue;
+		if (o[key] === value) return o;
+	}
+	return null;
+}
+
 module.exports = ArrayUtils;

ReactNativeClient/lib/ObjectUtils.js

@@ -28,4 +28,20 @@ ObjectUtils.sortByValue = function(object) {
 	return output;
 }
 
+ObjectUtils.fieldsEqual = function(o1, o2) {
+	if ((!o1 || !o2) && (o1 !== o2)) return false;
+	
+	for (let k in o1) {
+		if (!o1.hasOwnProperty(k)) continue;
+		if (o1[k] !== o2[k]) return false;
+	}
+
+	const c1 = Object.getOwnPropertyNames(o1);
+	const c2 = Object.getOwnPropertyNames(o2);
+
+	if (c1.length !== c2.length) return false;
+
+	return true;
+}
+
 module.exports = ObjectUtils;

ReactNativeClient/lib/models/Tag.js

@@ -102,6 +102,10 @@ class Tag extends BaseItem {
 		return this.modelSelectAll('SELECT * FROM tags WHERE id IN ("' + tagIds.join('","') + '")');
 	}
 
+	static async loadByTitle(title) {
+		return this.loadByField('title', title, { caseInsensitive: true });
+	}
+
 	static async setNoteTagsByTitles(noteId, tagTitles) {
 		const previousTags = await this.tagsByNoteId(noteId);
 		const addedTitles = [];
@@ -109,7 +113,7 @@ class Tag extends BaseItem {
 		for (let i = 0; i < tagTitles.length; i++) {
 			const title = tagTitles[i].trim().toLowerCase();
 			if (!title) continue;
-			let tag = await this.loadByField('title', title, { caseInsensitive: true });
+			let tag = await this.loadByTitle(title);
 			if (!tag) tag = await Tag.save({ title: title }, { userSideValidation: true });
 			await this.addNote(tag.id, noteId);
 			addedTitles.push(title);

ReactNativeClient/lib/services/InteropService.js

@@ -10,6 +10,14 @@ const fs = require('fs-extra');
 const md5 = require('md5');
 const { sprintf } = require('sprintf-js');
 const { shim } = require('lib/shim');
+const { fileExtension } = require('lib/path-utils');
+const { uuid } = require('lib/uuid.js');
+
+async function temporaryDirectory(createIt) {
+	const tempDir = require('os').tmpdir() + '/' + md5(Math.random() + Date.now());
+	if (createIt) await fs.mkdirp(tempDir);
+	return tempDir;
+}
 
 class RawExporter {
 
@@ -41,7 +49,7 @@ class JexExporter {
 	async init(destPath) {
 		if (await shim.fsDriver().isDirectory(destPath)) throw new Error('Path is a directory: ' + destPath);
 
-		this.tempDir_ = require('os').tmpdir() + '/' + md5(Math.random() + Date.now());
+		this.tempDir_ = await temporaryDirectory(false);
 		this.destPath_ = destPath;
 		this.rawExporter_ = new RawExporter();
 		await this.rawExporter_.init(this.tempDir_);
@@ -71,6 +79,152 @@ class JexExporter {
 
 }
 
+class RawImporter {
+
+	async init(sourceDir) {
+		this.sourceDir_ = sourceDir;
+	}
+
+	async exec(result, options) {
+		const noteIdMap = {};
+		const folderIdMap = {};
+		const resourceIdMap = {};
+		const tagIdMap = {};
+		const createdResources = {};
+		const noteTagsToCreate = [];
+		const destinationFolderId = options.destinationFolderId;
+
+		const replaceResourceNoteIds = (noteBody) => {
+			let output = noteBody;
+			const resourceIds = Note.linkedResourceIds(noteBody);
+
+			for (let i = 0; i < resourceIds.length; i++) {
+				const id = resourceIds[i];
+				if (!resourceIdMap[id]) resourceIdMap[id] = uuid.create();
+				output = output.replace(new RegExp(id, 'gi'), resourceIdMap[id]);
+			}
+
+			return output;
+		}
+
+		const stats = await shim.fsDriver().readDirStats(this.sourceDir_);
+		for (let i = 0; i < stats.length; i++) {
+			const stat = stats[i];
+			if (stat.isDirectory()) continue;
+			if (fileExtension(stat.path).toLowerCase() !== 'md') continue;
+
+			const content = await shim.fsDriver().readFile(this.sourceDir_ + '/' + stat.path);
+			let item = await BaseItem.unserialize(content);
+			const itemType = item.type_;
+			const ItemClass = BaseItem.itemClass(item);
+
+			delete item.type_;
+
+			if (itemType === BaseModel.TYPE_NOTE) {
+				if (!folderIdMap[item.parent_id]) folderIdMap[item.parent_id] = destinationFolderId ? destinationFolderId : uuid.create();
+				const noteId = uuid.create();
+				noteIdMap[item.id] = noteId;
+				item.id = noteId;
+				item.parent_id = folderIdMap[item.parent_id];
+				item.body = replaceResourceNoteIds(item.body);
+			} else if (itemType === BaseModel.TYPE_FOLDER) {
+				if (destinationFolderId) continue;
+
+				if (!folderIdMap[item.id]) folderIdMap[item.id] = uuid.create();
+				item.id = folderIdMap[item.id];
+				item.title = await Folder.findUniqueFolderTitle(item.title);
+			} else if (itemType === BaseModel.TYPE_RESOURCE) {
+				if (!resourceIdMap[item.id]) resourceIdMap[item.id] = uuid.create();
+				item.id = resourceIdMap[item.id];
+				createdResources[item.id] = item;
+			} else if (itemType === BaseModel.TYPE_TAG) {
+				const tag = await Tag.loadByTitle(item.title); 
+				if (tag) {
+					tagIdMap[item.id] = tag.id;
+					continue;
+				}
+
+				const tagId = uuid.create();
+				tagIdMap[item.id] = tagId;
+				item.id = tagId;
+			} else if (itemType === BaseModel.TYPE_NOTE_TAG) {
+				noteTagsToCreate.push(item);
+				continue;
+			}
+
+			await ItemClass.save(item, { isNew: true, autoTimestamp: false });
+		}
+
+		for (let i = 0; i < noteTagsToCreate.length; i++) {
+			const noteTag = noteTagsToCreate[i];
+			const newNoteId = noteIdMap[noteTag.note_id];
+			const newTagId = tagIdMap[noteTag.tag_id];
+
+			if (!newNoteId) {
+				result.warnings.push(sprintf('Non-existent note %s referenced in tag %s', noteTag.note_id, noteTag.tag_id));
+				continue;
+			}
+
+			if (!newTagId) {
+				result.warnings.push(sprintf('Non-existent tag %s for note %s', noteTag.tag_id, noteTag.note_id));
+				continue;
+			}
+
+			noteTag.id = uuid.create();
+			noteTag.note_id = newNoteId;
+			noteTag.tag_id = newTagId;
+
+			await NoteTag.save(noteTag, { isNew: true });
+		}
+
+		const resourceStats = await shim.fsDriver().readDirStats(this.sourceDir_ + '/resources');
+
+		for (let i = 0; i < resourceStats.length; i++) {
+			const resourceFilePath = this.sourceDir_ + '/resources/' + resourceStats[i].path;
+			const oldId = Resource.pathToId(resourceFilePath);
+			const newId = resourceIdMap[oldId];
+			if (!newId) {
+				result.warnings.push(sprintf('Resource file is not referenced in any note and so was not imported: %s', oldId));
+				continue;
+			}
+
+			const resource = createdResources[newId];
+			const destPath = Resource.fullPath(resource);
+			await shim.fsDriver().copy(resourceFilePath, destPath);
+		}
+
+		return result;
+	}
+
+}
+
+class JexImporter {
+
+	async init(sourcePath, options) {
+		this.sourcePath_ = sourcePath;
+	}
+
+	async exec(result, options) {
+		const tempDir = await temporaryDirectory(true);
+
+		await require('tar').extract({
+			strict: true,
+			portable: true,
+			file: this.sourcePath_,
+			cwd: tempDir,
+		});
+
+		const importer = newImporter('raw');
+		await importer.init(tempDir);
+		result = await importer.exec(result, options);
+
+		await fs.remove(tempDir);
+
+		return result;		
+	}
+
+}
+
 function newExporter(format) {
 	if (format === 'raw') {
 		return new RawExporter();
@@ -94,14 +248,39 @@ function newImporter(format) {
 class InteropService {
 
 	async import(options) {
+		options = Object.assign({}, {
+			format: 'auto',
+			destinationFolderId: null,
+		}, options);
+
+		if (options.format === 'auto') {
+			const ext = fileExtension(options.path);
+			if (ext.toLowerCase() === 'jex') {
+				options.format = 'jex';
+			} else {
+				throw new Error('Cannot automatically detect source format from path: ' + options.path);
+			}
+		}
+
+		if (options.destinationFolderId) {
+			const folder = await Folder.load(options.destinationFolderId);
+			if (!folder) throw new Error('Notebook not found: ' + options.destinationFolderId);
+		}
+
+		let result = { warnings: [] }
+
 		const importer = newImporter(options.format);
 		await importer.init(options.path);
+		result = await importer.exec(result, options);
+
+		return result;
 	}
 
 	async export(options) {
 		const exportPath = options.path ? options.path : null;
 		const sourceFolderIds = options.sourceFolderIds ? options.sourceFolderIds : [];
 		const sourceNoteIds = options.sourceNoteIds ? options.sourceNoteIds : [];
+		const exportFormat = options.format ? options.format : 'jex';
 		const result = { warnings: [] }
 		const itemsToExport = [];
 
@@ -155,7 +334,7 @@ class InteropService {
 			await queueExportItem(BaseModel.TYPE_TAG, exportedTagIds[i]);
 		}
 
-		const exporter = newExporter(options.format);
+		const exporter = newExporter(exportFormat);
 		await exporter.init(exportPath);
 
 		for (let i = 0; i < itemsToExport.length; i++) {

ReactNativeClient/lib/shim-init-node.js

@@ -80,8 +80,12 @@ function shimInit() {
 
 		await Resource.save(resource, { isNew: true });
 
+		const newBody = [];
+		if (note.body) newBody.push(note.body);
+		newBody.push(Resource.markdownTag(resource));
+
 		const newNote = Object.assign({}, note, {
-			body: note.body + "\n\n" + Resource.markdownTag(resource),
+			body: newBody.join('\n\n'),
 		});
 		return await Note.save(newNote);
 	}