2020-10-09 19:35:46 +02:00
|
|
|
import { ImportExportResult } from './types';
|
2020-11-05 18:58:23 +02:00
|
|
|
import { _ } from '../../locale';
|
2020-10-09 19:35:46 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
import InteropService_Importer_Base from './InteropService_Importer_Base';
|
|
|
|
import Folder from '../../models/Folder';
|
|
|
|
import Note from '../../models/Note';
|
2021-08-23 01:35:45 +02:00
|
|
|
import { NoteEntity } from '../database/types';
|
|
|
|
import { basename, filename, rtrimSlashes, fileExtension, dirname } from '../../path-utils';
|
2021-01-22 19:41:11 +02:00
|
|
|
import shim from '../../shim';
|
|
|
|
import markdownUtils from '../../markdownUtils';
|
2021-08-23 01:35:45 +02:00
|
|
|
import htmlUtils from '../../htmlUtils';
|
2022-05-26 16:57:44 +02:00
|
|
|
import { unique } from '../../ArrayUtils';
|
2020-11-05 18:58:23 +02:00
|
|
|
const { pregQuote } = require('../../string-utils-common');
|
2021-08-23 01:35:45 +02:00
|
|
|
import { MarkupToHtml } from '@joplin/renderer';
|
2023-12-15 14:47:03 +02:00
|
|
|
import { isDataUrl } from '@joplin/utils/url';
|
2024-02-08 18:55:34 +02:00
|
|
|
import { stripBom } from '../../string-utils';
|
2018-02-26 21:25:54 +02:00
|
|
|
|
2020-10-09 19:35:46 +02:00
|
|
|
export default class InteropService_Importer_Md extends InteropService_Importer_Base {
|
2023-11-15 15:33:20 +02:00
|
|
|
protected importedNotes: Record<string, NoteEntity> = {};
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-03-06 16:22:01 +02:00
|
|
|
public async exec(result: ImportExportResult) {
|
2018-02-27 22:04:38 +02:00
|
|
|
let parentFolderId = null;
|
2018-02-26 21:25:54 +02:00
|
|
|
|
2018-09-09 21:32:23 +02:00
|
|
|
const sourcePath = rtrimSlashes(this.sourcePath_);
|
2018-06-26 01:07:53 +02:00
|
|
|
|
2018-02-26 21:25:54 +02:00
|
|
|
const filePaths = [];
|
2018-09-09 21:32:23 +02:00
|
|
|
if (await shim.fsDriver().isDirectory(sourcePath)) {
|
2018-02-27 22:04:38 +02:00
|
|
|
if (!this.options_.destinationFolder) {
|
2018-09-09 21:32:23 +02:00
|
|
|
const folderTitle = await Folder.findUniqueItemTitle(basename(sourcePath));
|
2018-02-27 22:04:38 +02:00
|
|
|
const folder = await Folder.save({ title: folderTitle });
|
|
|
|
parentFolderId = folder.id;
|
|
|
|
} else {
|
|
|
|
parentFolderId = this.options_.destinationFolder.id;
|
|
|
|
}
|
2018-09-09 21:32:23 +02:00
|
|
|
|
2020-11-25 16:40:25 +02:00
|
|
|
await this.importDirectory(sourcePath, parentFolderId);
|
2018-02-26 21:25:54 +02:00
|
|
|
} else {
|
2018-02-27 22:04:38 +02:00
|
|
|
if (!this.options_.destinationFolder) throw new Error(_('Please specify the notebook where the notes should be imported to.'));
|
2019-07-29 15:43:53 +02:00
|
|
|
parentFolderId = this.options_.destinationFolder.id;
|
2018-09-09 21:32:23 +02:00
|
|
|
filePaths.push(sourcePath);
|
2018-02-26 21:25:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
for (let i = 0; i < filePaths.length; i++) {
|
2018-09-09 21:32:23 +02:00
|
|
|
await this.importFile(filePaths[i], parentFolderId);
|
2018-02-26 21:25:54 +02:00
|
|
|
}
|
|
|
|
|
2023-11-15 15:33:20 +02:00
|
|
|
for (const importedLocalPath of Object.keys(this.importedNotes)) {
|
|
|
|
const note = this.importedNotes[importedLocalPath];
|
|
|
|
const updatedBody = await this.importLocalFiles(importedLocalPath, note.body, note.parent_id);
|
|
|
|
const updatedNote = {
|
|
|
|
...this.importedNotes[importedLocalPath],
|
|
|
|
body: updatedBody || note.body,
|
|
|
|
};
|
|
|
|
this.importedNotes[importedLocalPath] = await Note.save(updatedNote, { isNew: false, autoTimestamp: false });
|
|
|
|
}
|
|
|
|
|
2018-02-26 21:25:54 +02:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2023-03-06 16:22:01 +02:00
|
|
|
public async importDirectory(dirPath: string, parentFolderId: string) {
|
2018-09-09 21:32:23 +02:00
|
|
|
const supportedFileExtension = this.metadata().fileExtensions;
|
|
|
|
const stats = await shim.fsDriver().readDirStats(dirPath);
|
|
|
|
for (let i = 0; i < stats.length; i++) {
|
|
|
|
const stat = stats[i];
|
|
|
|
|
|
|
|
if (stat.isDirectory()) {
|
2022-03-28 18:13:13 +02:00
|
|
|
if (await this.isDirectoryEmpty(`${dirPath}/${stat.path}`)) {
|
|
|
|
continue;
|
|
|
|
}
|
2018-09-09 21:32:23 +02:00
|
|
|
const folderTitle = await Folder.findUniqueItemTitle(basename(stat.path));
|
|
|
|
const folder = await Folder.save({ title: folderTitle, parent_id: parentFolderId });
|
2020-11-25 16:40:25 +02:00
|
|
|
await this.importDirectory(`${dirPath}/${basename(stat.path)}`, folder.id);
|
2018-09-09 21:32:23 +02:00
|
|
|
} else if (supportedFileExtension.indexOf(fileExtension(stat.path).toLowerCase()) >= 0) {
|
2020-11-25 16:40:25 +02:00
|
|
|
await this.importFile(`${dirPath}/${stat.path}`, parentFolderId);
|
2018-09-09 21:32:23 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-28 18:13:13 +02:00
|
|
|
private async isDirectoryEmpty(dirPath: string) {
|
|
|
|
const supportedFileExtension = this.metadata().fileExtensions;
|
|
|
|
const innerStats = await shim.fsDriver().readDirStats(dirPath);
|
|
|
|
for (let i = 0; i < innerStats.length; i++) {
|
|
|
|
const innerStat = innerStats[i];
|
|
|
|
|
|
|
|
if (innerStat.isDirectory()) {
|
|
|
|
if (!(await this.isDirectoryEmpty(`${dirPath}/${innerStat.path}`))) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
} else if (supportedFileExtension.indexOf(fileExtension(innerStat.path).toLowerCase()) >= 0) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2021-08-23 01:35:45 +02:00
|
|
|
private trimAnchorLink(link: string) {
|
|
|
|
if (link.indexOf('#') <= 0) return link;
|
|
|
|
|
|
|
|
const splitted = link.split('#');
|
|
|
|
splitted.pop();
|
|
|
|
return splitted.join('#');
|
|
|
|
}
|
|
|
|
|
2023-06-30 10:55:56 +02:00
|
|
|
// Parse text for links, attempt to find local file, if found create Joplin resource
|
|
|
|
// and update link accordingly.
|
2023-03-06 16:22:01 +02:00
|
|
|
public async importLocalFiles(filePath: string, md: string, parentFolderId: string) {
|
Desktop: when importing MD files create resources for local linked files (#2262)
* md importer: first pass import attachment resources with markdown files
* md importer: import resources from md - no unneeded saves, check if files exist, regex name
* md importer: test import of local files as resources, separate method for importing linked files, comment regex matching md tags
* md importer: move stateful regex to method scope, remove spurius await
* md importer: lint
* md importer: respond to PR comments: remove test nesting, test sample, check if path is dir, use shim.fsDriver
* md importer: use file-path methods for getting attachment path
* md importer: use extractImageUrls helper, test for file with zero links
* md importer: try catch around importLocalImages, improve test
* md importer: importing attached images cover case where link also appears elsewhere in doc
* md importer: only create 1 resource if note contains duplicate links, test
* md importer: remove log
* md importer: remove use of lodash
2020-01-19 17:39:38 +02:00
|
|
|
let updated = md;
|
2021-08-23 01:35:45 +02:00
|
|
|
const markdownLinks = markdownUtils.extractFileUrls(md);
|
|
|
|
const htmlLinks = htmlUtils.extractFileUrls(md);
|
|
|
|
const fileLinks = unique(markdownLinks.concat(htmlLinks));
|
2023-11-15 15:33:20 +02:00
|
|
|
for (const encodedLink of fileLinks) {
|
2020-03-27 14:20:38 +02:00
|
|
|
const link = decodeURI(encodedLink);
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-12-15 14:47:03 +02:00
|
|
|
if (isDataUrl(link)) {
|
|
|
|
// Just leave it as it is. We could potentially import
|
|
|
|
// it as a resource but for now that's good enough.
|
|
|
|
} else {
|
|
|
|
// Handle anchor links appropriately
|
|
|
|
const trimmedLink = this.trimAnchorLink(link);
|
|
|
|
const attachmentPath = filename(`${dirname(filePath)}/${trimmedLink}`, true);
|
|
|
|
const pathWithExtension = `${attachmentPath}.${fileExtension(trimmedLink)}`;
|
|
|
|
const stat = await shim.fsDriver().stat(pathWithExtension);
|
|
|
|
const isDir = stat ? stat.isDirectory() : false;
|
|
|
|
if (stat && !isDir) {
|
|
|
|
const supportedFileExtension = this.metadata().fileExtensions;
|
|
|
|
const resolvedPath = shim.fsDriver().resolve(pathWithExtension);
|
|
|
|
let id = '';
|
|
|
|
// If the link looks like a note, then import it
|
|
|
|
if (supportedFileExtension.indexOf(fileExtension(trimmedLink).toLowerCase()) >= 0) {
|
|
|
|
// If the note hasn't been imported yet, do so now
|
|
|
|
if (!this.importedNotes[resolvedPath]) {
|
|
|
|
await this.importFile(resolvedPath, parentFolderId);
|
|
|
|
}
|
|
|
|
|
|
|
|
id = this.importedNotes[resolvedPath].id;
|
|
|
|
} else {
|
2023-12-15 15:28:09 +02:00
|
|
|
const resource = await shim.createResourceFromPath(pathWithExtension, null, { resizeLargeImages: 'never' });
|
2023-12-15 14:47:03 +02:00
|
|
|
id = resource.id;
|
|
|
|
}
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-12-15 14:47:03 +02:00
|
|
|
// The first is a normal link, the second is supports the <link> and [](<link with spaces>) syntax
|
|
|
|
// Only opening patterns are consider in order to cover all occurances
|
|
|
|
// We need to use the encoded link as well because some links (link's with spaces)
|
|
|
|
// will appear encoded in the source. Other links (unicode chars) will not
|
|
|
|
const linksToReplace = [this.trimAnchorLink(link), this.trimAnchorLink(encodedLink)];
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-12-15 14:47:03 +02:00
|
|
|
for (let j = 0; j < linksToReplace.length; j++) {
|
|
|
|
const linkToReplace = pregQuote(linksToReplace[j]);
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-12-15 14:47:03 +02:00
|
|
|
// Markdown links
|
|
|
|
updated = markdownUtils.replaceResourceUrl(updated, linkToReplace, id);
|
2021-08-23 01:35:45 +02:00
|
|
|
|
2023-12-15 14:47:03 +02:00
|
|
|
// HTML links
|
|
|
|
updated = htmlUtils.replaceResourceUrl(updated, linkToReplace, id);
|
|
|
|
}
|
2021-08-23 01:35:45 +02:00
|
|
|
}
|
Desktop: when importing MD files create resources for local linked files (#2262)
* md importer: first pass import attachment resources with markdown files
* md importer: import resources from md - no unneeded saves, check if files exist, regex name
* md importer: test import of local files as resources, separate method for importing linked files, comment regex matching md tags
* md importer: move stateful regex to method scope, remove spurius await
* md importer: lint
* md importer: respond to PR comments: remove test nesting, test sample, check if path is dir, use shim.fsDriver
* md importer: use file-path methods for getting attachment path
* md importer: use extractImageUrls helper, test for file with zero links
* md importer: try catch around importLocalImages, improve test
* md importer: importing attached images cover case where link also appears elsewhere in doc
* md importer: only create 1 resource if note contains duplicate links, test
* md importer: remove log
* md importer: remove use of lodash
2020-01-19 17:39:38 +02:00
|
|
|
}
|
2023-11-15 15:33:20 +02:00
|
|
|
}
|
Desktop: when importing MD files create resources for local linked files (#2262)
* md importer: first pass import attachment resources with markdown files
* md importer: import resources from md - no unneeded saves, check if files exist, regex name
* md importer: test import of local files as resources, separate method for importing linked files, comment regex matching md tags
* md importer: move stateful regex to method scope, remove spurius await
* md importer: lint
* md importer: respond to PR comments: remove test nesting, test sample, check if path is dir, use shim.fsDriver
* md importer: use file-path methods for getting attachment path
* md importer: use extractImageUrls helper, test for file with zero links
* md importer: try catch around importLocalImages, improve test
* md importer: importing attached images cover case where link also appears elsewhere in doc
* md importer: only create 1 resource if note contains duplicate links, test
* md importer: remove log
* md importer: remove use of lodash
2020-01-19 17:39:38 +02:00
|
|
|
return updated;
|
|
|
|
}
|
|
|
|
|
2023-03-06 16:22:01 +02:00
|
|
|
public async importFile(filePath: string, parentFolderId: string) {
|
2021-08-23 01:35:45 +02:00
|
|
|
const resolvedPath = shim.fsDriver().resolve(filePath);
|
|
|
|
if (this.importedNotes[resolvedPath]) return this.importedNotes[resolvedPath];
|
|
|
|
|
|
|
|
const stat = await shim.fsDriver().stat(resolvedPath);
|
|
|
|
if (!stat) throw new Error(`Cannot read ${resolvedPath}`);
|
|
|
|
const ext = fileExtension(resolvedPath);
|
|
|
|
const title = filename(resolvedPath);
|
2024-02-08 18:55:34 +02:00
|
|
|
const body = stripBom(await shim.fsDriver().readFile(resolvedPath));
|
|
|
|
|
2018-09-09 21:32:23 +02:00
|
|
|
const note = {
|
|
|
|
parent_id: parentFolderId,
|
|
|
|
title: title,
|
2021-08-23 01:35:45 +02:00
|
|
|
body: body,
|
2018-09-09 21:32:23 +02:00
|
|
|
updated_time: stat.mtime.getTime(),
|
|
|
|
created_time: stat.birthtime.getTime(),
|
|
|
|
user_updated_time: stat.mtime.getTime(),
|
|
|
|
user_created_time: stat.birthtime.getTime(),
|
2021-08-23 01:35:45 +02:00
|
|
|
markup_language: ext === 'html' ? MarkupToHtml.MARKUP_LANGUAGE_HTML : MarkupToHtml.MARKUP_LANGUAGE_MARKDOWN,
|
2018-09-09 21:32:23 +02:00
|
|
|
};
|
2021-08-23 01:35:45 +02:00
|
|
|
this.importedNotes[resolvedPath] = await Note.save(note, { autoTimestamp: false });
|
|
|
|
|
|
|
|
return this.importedNotes[resolvedPath];
|
2018-09-09 21:32:23 +02:00
|
|
|
}
|
2018-02-26 21:25:54 +02:00
|
|
|
}
|