joplin/ReactNativeClient/lib/services/InteropService.js

const BaseItem = require('lib/models/BaseItem.js');
const BaseModel = require('lib/BaseModel.js');
const Resource = require('lib/models/Resource.js');
const Folder = require('lib/models/Folder.js');
const NoteTag = require('lib/models/NoteTag.js');
const Note = require('lib/models/Note.js');
const Tag = require('lib/models/Tag.js');
const { basename, filename } = require('lib/path-utils.js');
const fs = require('fs-extra');
const md5 = require('md5');
const { sprintf } = require('sprintf-js');
const { shim } = require('lib/shim');
const { _ } = require('lib/locale');
const { fileExtension } = require('lib/path-utils');
const { uuid } = require('lib/uuid.js');
const { importEnex } = require('lib/import-enex');

async function temporaryDirectory(createIt) {
	const tempDir = require('os').tmpdir() + '/' + md5(Math.random() + Date.now());
	if (createIt) await fs.mkdirp(tempDir);
	return tempDir;
}

class RawExporter {

	async init(destDir) {
		this.destDir_ = destDir;
		this.resourceDir_ = destDir ? destDir + '/resources' : null;

		await fs.mkdirp(this.destDir_);
		await fs.mkdirp(this.resourceDir_);
	}

	async processItem(ItemClass, item) {
		const serialized = await ItemClass.serialize(item);
		const filePath = this.destDir_ + '/' + ItemClass.systemPath(item);
		await fs.writeFile(filePath, serialized);
	}

	async processResource(resource, filePath) {
		const destResourcePath = this.resourceDir_ + '/' + basename(filePath);
		await fs.copyFile(filePath, destResourcePath, { overwrite: true });
	}

	async close() {}

}

class JexExporter {

	async init(destPath) {
		if (await shim.fsDriver().isDirectory(destPath)) throw new Error('Path is a directory: ' + destPath);

		this.tempDir_ = await temporaryDirectory(false);
		this.destPath_ = destPath;
		this.rawExporter_ = new RawExporter();
		await this.rawExporter_.init(this.tempDir_);
	}

	async processItem(ItemClass, item) {
		return this.rawExporter_.processItem(ItemClass, item);
	}

	async processResource(resource, filePath) {
		return this.rawExporter_.processResource(resource, filePath);
	}

	async close() {
		const stats = await shim.fsDriver().readDirStats(this.tempDir_, { recursive: true });
		const filePaths = stats.map((a) => a.path);

		await require('tar').create({
			strict: true,
			portable: true,
			file: this.destPath_,
			cwd: this.tempDir_,
		}, filePaths);

		await fs.remove(this.tempDir_);
	}

}

class RawImporter {

	async init(sourceDir, options) {
		this.sourceDir_ = sourceDir;
		this.options_ = options;
	}

	async exec(result) {
		const noteIdMap = {};
		const folderIdMap = {};
		const resourceIdMap = {};
		const tagIdMap = {};
		const createdResources = {};
		const noteTagsToCreate = [];
		const destinationFolderId = this.options_.destinationFolderId;

		const replaceResourceNoteIds = (noteBody) => {
			let output = noteBody;
			const resourceIds = Note.linkedResourceIds(noteBody);

			for (let i = 0; i < resourceIds.length; i++) {
				const id = resourceIds[i];
				if (!resourceIdMap[id]) resourceIdMap[id] = uuid.create();
				output = output.replace(new RegExp(id, 'gi'), resourceIdMap[id]);
			}

			return output;
		}

		const stats = await shim.fsDriver().readDirStats(this.sourceDir_);
		for (let i = 0; i < stats.length; i++) {
			const stat = stats[i];
			if (stat.isDirectory()) continue;
			if (fileExtension(stat.path).toLowerCase() !== 'md') continue;

			const content = await shim.fsDriver().readFile(this.sourceDir_ + '/' + stat.path);
			let item = await BaseItem.unserialize(content);
			const itemType = item.type_;
			const ItemClass = BaseItem.itemClass(item);

			delete item.type_;

			if (itemType === BaseModel.TYPE_NOTE) {
				if (!folderIdMap[item.parent_id]) folderIdMap[item.parent_id] = destinationFolderId ? destinationFolderId : uuid.create();
				const noteId = uuid.create();
				noteIdMap[item.id] = noteId;
				item.id = noteId;
				item.parent_id = folderIdMap[item.parent_id];
				item.body = replaceResourceNoteIds(item.body);
			} else if (itemType === BaseModel.TYPE_FOLDER) {
				if (destinationFolderId) continue;

				if (!folderIdMap[item.id]) folderIdMap[item.id] = uuid.create();
				item.id = folderIdMap[item.id];
				item.title = await Folder.findUniqueFolderTitle(item.title);
			} else if (itemType === BaseModel.TYPE_RESOURCE) {
				if (!resourceIdMap[item.id]) resourceIdMap[item.id] = uuid.create();
				item.id = resourceIdMap[item.id];
				createdResources[item.id] = item;
			} else if (itemType === BaseModel.TYPE_TAG) {
				const tag = await Tag.loadByTitle(item.title); 
				if (tag) {
					tagIdMap[item.id] = tag.id;
					continue;
				}

				const tagId = uuid.create();
				tagIdMap[item.id] = tagId;
				item.id = tagId;
			} else if (itemType === BaseModel.TYPE_NOTE_TAG) {
				noteTagsToCreate.push(item);
				continue;
			}

			await ItemClass.save(item, { isNew: true, autoTimestamp: false });
		}

		for (let i = 0; i < noteTagsToCreate.length; i++) {
			const noteTag = noteTagsToCreate[i];
			const newNoteId = noteIdMap[noteTag.note_id];
			const newTagId = tagIdMap[noteTag.tag_id];

			if (!newNoteId) {
				result.warnings.push(sprintf('Non-existent note %s referenced in tag %s', noteTag.note_id, noteTag.tag_id));
				continue;
			}

			if (!newTagId) {
				result.warnings.push(sprintf('Non-existent tag %s for note %s', noteTag.tag_id, noteTag.note_id));
				continue;
			}

			noteTag.id = uuid.create();
			noteTag.note_id = newNoteId;
			noteTag.tag_id = newTagId;

			await NoteTag.save(noteTag, { isNew: true });
		}

		const resourceStats = await shim.fsDriver().readDirStats(this.sourceDir_ + '/resources');

		for (let i = 0; i < resourceStats.length; i++) {
			const resourceFilePath = this.sourceDir_ + '/resources/' + resourceStats[i].path;
			const oldId = Resource.pathToId(resourceFilePath);
			const newId = resourceIdMap[oldId];
			if (!newId) {
				result.warnings.push(sprintf('Resource file is not referenced in any note and so was not imported: %s', oldId));
				continue;
			}

			const resource = createdResources[newId];
			const destPath = Resource.fullPath(resource);
			await shim.fsDriver().copy(resourceFilePath, destPath);
		}

		return result;
	}

}

class JexImporter {

	async init(sourcePath, options) {
		this.sourcePath_ = sourcePath;
		this.options_ = options;
	}

	async exec(result) {
		const tempDir = await temporaryDirectory(true);

		try {
			await require('tar').extract({
				strict: true,
				portable: true,
				file: this.sourcePath_,
				cwd: tempDir,
			});
		} catch (error) {
			let msg = ['Cannot untar file ' + this.sourcePath_, error.message];
			if (error.data) msg.push(JSON.stringify(error.data));
			let e = new Error(msg.join(': '));
			throw e;
		}

		const importer = newImporter('raw');
		await importer.init(tempDir, this.options_);
		result = await importer.exec(result);

		await fs.remove(tempDir);

		return result;		
	}

}

class MdImporter {

	async init(sourcePath, options) {
		this.sourcePath_ = sourcePath;
		this.options_ = options;
	}

	async exec(result) {
		if (!this.options_.destinationFolder) throw new Error('Destination folder must be specified');

		const parentFolderId = this.options_.destinationFolder.id;

		const filePaths = [];
		if (await shim.fsDriver().isDirectory(this.sourcePath_)) {
			const stats = await shim.fsDriver().readDirStats(this.sourcePath_);
			for (let i = 0; i < stats.length; i++) {
				const stat = stats[i];
				if (fileExtension(stat.path).toLowerCase() === 'md') {
					filePaths.push(this.sourcePath_ + '/' + stat.path);
				}
			}
		} else {
			filePaths.push(this.sourcePath_);
		}

		for (let i = 0; i < filePaths.length; i++) {
			const path = filePaths[i];
			const stat = await shim.fsDriver().stat(path);
			if (!stat) throw new Error('Cannot read ' + path);
			const title = filename(path);
			const body = await shim.fsDriver().readFile(path);
			const note = {
				parent_id: parentFolderId,
				title: title,
				body: body,
				updated_time: stat.mtime.getTime(),
				created_time: stat.birthtime.getTime(),
				user_updated_time: stat.mtime.getTime(),
				user_created_time: stat.birthtime.getTime(),
			};
			await Note.save(note, { autoTimestamp: false });
		}

		return result;
	}

}

class EnexImporter {

	async init(sourcePath, options) {
		this.sourcePath_ = sourcePath;
		this.options_ = options;
	}

	async exec(result) {
		let folder = this.options_.destinationFolder;

		if (!folder) {
			const folderTitle = await Folder.findUniqueFolderTitle(filename(this.sourcePath_));
			folder = await Folder.save({ title: folderTitle });
		}

		await importEnex(folder.id, this.sourcePath_, this.options_);

		return result;
	}

}

function newExporter(format) {
	if (format === 'raw') {
		return new RawExporter();
	} else if (format === 'jex') {
		return new JexExporter();
	} else {
		throw new Error('Unknown format: ' + format);
	}
}

function newImporter(format) {
	if (format === 'raw') {
		return new RawImporter();
	} else if (format === 'jex') {
		return new JexImporter();
	} else if (format === 'md') {
		return new MdImporter();
	} else if (format === 'enex') {
		return new EnexImporter();
	} else {
		throw new Error('Unknown format: ' + format);
	}
}

class InteropService {

	async import(options) {
		if (!await shim.fsDriver().exists(options.path)) throw new Error(_('Cannot find "%s".', options.path));

		options = Object.assign({}, {
			format: 'auto',
			destinationFolderId: null,
			destinationFolder: null,
		}, options);

		if (options.format === 'auto') {
			const ext = fileExtension(options.path).toLowerCase();
			if (ext === 'jex') {
				options.format = 'jex';
			} else if (ext === 'enex') {
				options.format = 'enex';
			} else {
				throw new Error('Cannot automatically detect source format from path: ' + options.path);
			}
		}

		if (options.destinationFolderId) {
			const folder = await Folder.load(options.destinationFolderId);
			if (!folder) throw new Error('Notebook not found: ' + options.destinationFolderId);
			options.destinationFolder = folder;
		}

		let result = { warnings: [] }

		const importer = newImporter(options.format);
		await importer.init(options.path, options);
		result = await importer.exec(result);

		return result;
	}

	async export(options) {
		const exportPath = options.path ? options.path : null;
		const sourceFolderIds = options.sourceFolderIds ? options.sourceFolderIds : [];
		const sourceNoteIds = options.sourceNoteIds ? options.sourceNoteIds : [];
		const exportFormat = options.format ? options.format : 'jex';
		const result = { warnings: [] }
		const itemsToExport = [];

		const queueExportItem = (itemType, itemOrId) => {
			itemsToExport.push({
				type: itemType,
				itemOrId: itemOrId
			});
		}

		let exportedNoteIds = [];
		let resourceIds = [];
		const folderIds = await Folder.allIds();

		for (let folderIndex = 0; folderIndex < folderIds.length; folderIndex++) {
			const folderId = folderIds[folderIndex];
			if (sourceFolderIds.length && sourceFolderIds.indexOf(folderId) < 0) continue;

			if (!sourceNoteIds.length) await queueExportItem(BaseModel.TYPE_FOLDER, folderId);

			const noteIds = await Folder.noteIds(folderId);

			for (let noteIndex = 0; noteIndex < noteIds.length; noteIndex++) {
				const noteId = noteIds[noteIndex];
				if (sourceNoteIds.length && sourceNoteIds.indexOf(noteId) < 0) continue;
				const note = await Note.load(noteId);
				await queueExportItem(BaseModel.TYPE_NOTE, note);
				exportedNoteIds.push(noteId);

				const rids = Note.linkedResourceIds(note.body);
				resourceIds = resourceIds.concat(rids);
			}
		}

		for (let i = 0; i < resourceIds.length; i++) {
			await queueExportItem(BaseModel.TYPE_RESOURCE, resourceIds[i]);
		}

		const noteTags = await NoteTag.all();

		let exportedTagIds = [];

		for (let i = 0; i < noteTags.length; i++) {
			const noteTag = noteTags[i];
			if (exportedNoteIds.indexOf(noteTag.note_id) < 0) continue;
			await queueExportItem(BaseModel.TYPE_NOTE_TAG, noteTag.id);
			exportedTagIds.push(noteTag.tag_id);
		}

		for (let i = 0; i < exportedTagIds.length; i++) {
			await queueExportItem(BaseModel.TYPE_TAG, exportedTagIds[i]);
		}

		const exporter = newExporter(exportFormat);
		await exporter.init(exportPath);

		for (let i = 0; i < itemsToExport.length; i++) {
			const itemType = itemsToExport[i].type;
			const ItemClass = BaseItem.getClassByItemType(itemType);
			const itemOrId = itemsToExport[i].itemOrId;
			const item = typeof itemOrId === 'object' ? itemOrId : await ItemClass.load(itemOrId);

			if (!item) {
				if (itemType === BaseModel.TYPE_RESOURCE) {
					result.warnings.push(sprintf('A resource that does not exist is referenced in a note. The resource was skipped. Resource ID: %s', itemOrId));
				} else {
					result.warnings.push(sprintf('Cannot find item with type "%s" and ID %s. Item was skipped.', ItemClass.tableName(), JSON.stringify(itemOrId)));
				}
				continue;
			}

			await exporter.processItem(ItemClass, item);

			if (itemType == BaseModel.TYPE_RESOURCE) {
				const resourcePath = Resource.fullPath(item);
				await exporter.processResource(item, resourcePath);
			}
		}

		await exporter.close();

		return result;
	}

}

module.exports = InteropService;