Support importing JEX and raw data

ReactNativeClient/lib/ArrayUtils.js

@@ -15,7 +15,6 @@ ArrayUtils.removeElement = function(array, element) {
 
 // https://stackoverflow.com/a/10264318/561309
 ArrayUtils.binarySearch = function(items, value) {
-
 	var startIndex  = 0,
 		stopIndex   = items.length - 1,
 		middle      = Math.floor((stopIndex + startIndex)/2);
@@ -37,4 +36,13 @@ ArrayUtils.binarySearch = function(items, value) {
 	return (items[middle] != value) ? -1 : middle;
 }
 
+ArrayUtils.findByKey = function(array, key, value) {
+	for (let i = 0; i < array.length; i++) {
+		const o = array[i];
+		if (typeof o !== 'object') continue;
+		if (o[key] === value) return o;
+	}
+	return null;
+}
+
 module.exports = ArrayUtils;

ReactNativeClient/lib/ObjectUtils.js

@@ -28,4 +28,20 @@ ObjectUtils.sortByValue = function(object) {
 	return output;
 }
 
+ObjectUtils.fieldsEqual = function(o1, o2) {
+	if ((!o1 || !o2) && (o1 !== o2)) return false;
+	
+	for (let k in o1) {
+		if (!o1.hasOwnProperty(k)) continue;
+		if (o1[k] !== o2[k]) return false;
+	}
+
+	const c1 = Object.getOwnPropertyNames(o1);
+	const c2 = Object.getOwnPropertyNames(o2);
+
+	if (c1.length !== c2.length) return false;
+
+	return true;
+}
+
 module.exports = ObjectUtils;

ReactNativeClient/lib/models/Tag.js

@@ -102,6 +102,10 @@ class Tag extends BaseItem {
 		return this.modelSelectAll('SELECT * FROM tags WHERE id IN ("' + tagIds.join('","') + '")');
 	}
 
+	static async loadByTitle(title) {
+		return this.loadByField('title', title, { caseInsensitive: true });
+	}
+
 	static async setNoteTagsByTitles(noteId, tagTitles) {
 		const previousTags = await this.tagsByNoteId(noteId);
 		const addedTitles = [];
@@ -109,7 +113,7 @@ class Tag extends BaseItem {
 		for (let i = 0; i < tagTitles.length; i++) {
 			const title = tagTitles[i].trim().toLowerCase();
 			if (!title) continue;
-			let tag = await this.loadByField('title', title, { caseInsensitive: true });
+			let tag = await this.loadByTitle(title);
 			if (!tag) tag = await Tag.save({ title: title }, { userSideValidation: true });
 			await this.addNote(tag.id, noteId);
 			addedTitles.push(title);

ReactNativeClient/lib/services/InteropService.js

@@ -10,6 +10,14 @@ const fs = require('fs-extra');
 const md5 = require('md5');
 const { sprintf } = require('sprintf-js');
 const { shim } = require('lib/shim');
+const { fileExtension } = require('lib/path-utils');
+const { uuid } = require('lib/uuid.js');
+
+async function temporaryDirectory(createIt) {
+	const tempDir = require('os').tmpdir() + '/' + md5(Math.random() + Date.now());
+	if (createIt) await fs.mkdirp(tempDir);
+	return tempDir;
+}
 
 class RawExporter {
 
@@ -41,7 +49,7 @@ class JexExporter {
 	async init(destPath) {
 		if (await shim.fsDriver().isDirectory(destPath)) throw new Error('Path is a directory: ' + destPath);
 
-		this.tempDir_ = require('os').tmpdir() + '/' + md5(Math.random() + Date.now());
+		this.tempDir_ = await temporaryDirectory(false);
 		this.destPath_ = destPath;
 		this.rawExporter_ = new RawExporter();
 		await this.rawExporter_.init(this.tempDir_);
@@ -71,6 +79,152 @@ class JexExporter {
 
 }
 
+class RawImporter {
+
+	async init(sourceDir) {
+		this.sourceDir_ = sourceDir;
+	}
+
+	async exec(result, options) {
+		const noteIdMap = {};
+		const folderIdMap = {};
+		const resourceIdMap = {};
+		const tagIdMap = {};
+		const createdResources = {};
+		const noteTagsToCreate = [];
+		const destinationFolderId = options.destinationFolderId;
+
+		const replaceResourceNoteIds = (noteBody) => {
+			let output = noteBody;
+			const resourceIds = Note.linkedResourceIds(noteBody);
+
+			for (let i = 0; i < resourceIds.length; i++) {
+				const id = resourceIds[i];
+				if (!resourceIdMap[id]) resourceIdMap[id] = uuid.create();
+				output = output.replace(new RegExp(id, 'gi'), resourceIdMap[id]);
+			}
+
+			return output;
+		}
+
+		const stats = await shim.fsDriver().readDirStats(this.sourceDir_);
+		for (let i = 0; i < stats.length; i++) {
+			const stat = stats[i];
+			if (stat.isDirectory()) continue;
+			if (fileExtension(stat.path).toLowerCase() !== 'md') continue;
+
+			const content = await shim.fsDriver().readFile(this.sourceDir_ + '/' + stat.path);
+			let item = await BaseItem.unserialize(content);
+			const itemType = item.type_;
+			const ItemClass = BaseItem.itemClass(item);
+
+			delete item.type_;
+
+			if (itemType === BaseModel.TYPE_NOTE) {
+				if (!folderIdMap[item.parent_id]) folderIdMap[item.parent_id] = destinationFolderId ? destinationFolderId : uuid.create();
+				const noteId = uuid.create();
+				noteIdMap[item.id] = noteId;
+				item.id = noteId;
+				item.parent_id = folderIdMap[item.parent_id];
+				item.body = replaceResourceNoteIds(item.body);
+			} else if (itemType === BaseModel.TYPE_FOLDER) {
+				if (destinationFolderId) continue;
+
+				if (!folderIdMap[item.id]) folderIdMap[item.id] = uuid.create();
+				item.id = folderIdMap[item.id];
+				item.title = await Folder.findUniqueFolderTitle(item.title);
+			} else if (itemType === BaseModel.TYPE_RESOURCE) {
+				if (!resourceIdMap[item.id]) resourceIdMap[item.id] = uuid.create();
+				item.id = resourceIdMap[item.id];
+				createdResources[item.id] = item;
+			} else if (itemType === BaseModel.TYPE_TAG) {
+				const tag = await Tag.loadByTitle(item.title); 
+				if (tag) {
+					tagIdMap[item.id] = tag.id;
+					continue;
+				}
+
+				const tagId = uuid.create();
+				tagIdMap[item.id] = tagId;
+				item.id = tagId;
+			} else if (itemType === BaseModel.TYPE_NOTE_TAG) {
+				noteTagsToCreate.push(item);
+				continue;
+			}
+
+			await ItemClass.save(item, { isNew: true, autoTimestamp: false });
+		}
+
+		for (let i = 0; i < noteTagsToCreate.length; i++) {
+			const noteTag = noteTagsToCreate[i];
+			const newNoteId = noteIdMap[noteTag.note_id];
+			const newTagId = tagIdMap[noteTag.tag_id];
+
+			if (!newNoteId) {
+				result.warnings.push(sprintf('Non-existent note %s referenced in tag %s', noteTag.note_id, noteTag.tag_id));
+				continue;
+			}
+
+			if (!newTagId) {
+				result.warnings.push(sprintf('Non-existent tag %s for note %s', noteTag.tag_id, noteTag.note_id));
+				continue;
+			}
+
+			noteTag.id = uuid.create();
+			noteTag.note_id = newNoteId;
+			noteTag.tag_id = newTagId;
+
+			await NoteTag.save(noteTag, { isNew: true });
+		}
+
+		const resourceStats = await shim.fsDriver().readDirStats(this.sourceDir_ + '/resources');
+
+		for (let i = 0; i < resourceStats.length; i++) {
+			const resourceFilePath = this.sourceDir_ + '/resources/' + resourceStats[i].path;
+			const oldId = Resource.pathToId(resourceFilePath);
+			const newId = resourceIdMap[oldId];
+			if (!newId) {
+				result.warnings.push(sprintf('Resource file is not referenced in any note and so was not imported: %s', oldId));
+				continue;
+			}
+
+			const resource = createdResources[newId];
+			const destPath = Resource.fullPath(resource);
+			await shim.fsDriver().copy(resourceFilePath, destPath);
+		}
+
+		return result;
+	}
+
+}
+
+class JexImporter {
+
+	async init(sourcePath, options) {
+		this.sourcePath_ = sourcePath;
+	}
+
+	async exec(result, options) {
+		const tempDir = await temporaryDirectory(true);
+
+		await require('tar').extract({
+			strict: true,
+			portable: true,
+			file: this.sourcePath_,
+			cwd: tempDir,
+		});
+
+		const importer = newImporter('raw');
+		await importer.init(tempDir);
+		result = await importer.exec(result, options);
+
+		await fs.remove(tempDir);
+
+		return result;		
+	}
+
+}
+
 function newExporter(format) {
 	if (format === 'raw') {
 		return new RawExporter();
@@ -94,14 +248,39 @@ function newImporter(format) {
 class InteropService {
 
 	async import(options) {
+		options = Object.assign({}, {
+			format: 'auto',
+			destinationFolderId: null,
+		}, options);
+
+		if (options.format === 'auto') {
+			const ext = fileExtension(options.path);
+			if (ext.toLowerCase() === 'jex') {
+				options.format = 'jex';
+			} else {
+				throw new Error('Cannot automatically detect source format from path: ' + options.path);
+			}
+		}
+
+		if (options.destinationFolderId) {
+			const folder = await Folder.load(options.destinationFolderId);
+			if (!folder) throw new Error('Notebook not found: ' + options.destinationFolderId);
+		}
+
+		let result = { warnings: [] }
+
 		const importer = newImporter(options.format);
 		await importer.init(options.path);
+		result = await importer.exec(result, options);
+
+		return result;
 	}
 
 	async export(options) {
 		const exportPath = options.path ? options.path : null;
 		const sourceFolderIds = options.sourceFolderIds ? options.sourceFolderIds : [];
 		const sourceNoteIds = options.sourceNoteIds ? options.sourceNoteIds : [];
+		const exportFormat = options.format ? options.format : 'jex';
 		const result = { warnings: [] }
 		const itemsToExport = [];
 
@@ -155,7 +334,7 @@ class InteropService {
 			await queueExportItem(BaseModel.TYPE_TAG, exportedTagIds[i]);
 		}
 
-		const exporter = newExporter(options.format);
+		const exporter = newExporter(exportFormat);
 		await exporter.init(exportPath);
 
 		for (let i = 0; i < itemsToExport.length; i++) {

ReactNativeClient/lib/shim-init-node.js

@@ -80,8 +80,12 @@ function shimInit() {
 
 		await Resource.save(resource, { isNew: true });
 
+		const newBody = [];
+		if (note.body) newBody.push(note.body);
+		newBody.push(Resource.markdownTag(resource));
+
 		const newNote = Object.assign({}, note, {
-			body: note.body + "\n\n" + Resource.markdownTag(resource),
+			body: newBody.join('\n\n'),
 		});
 		return await Note.save(newNote);
 	}