1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-15 09:04:04 +02:00
joplin/CliClient/app/import-enex-md-gen.js

621 lines
18 KiB
JavaScript
Raw Normal View History

2017-07-17 20:59:40 +02:00
import stringPadding from 'string-padding';
const BLOCK_OPEN = "[[BLOCK_OPEN]]";
const BLOCK_CLOSE = "[[BLOCK_CLOSE]]";
const NEWLINE = "[[NEWLINE]]";
const NEWLINE_MERGED = "[[MERGED]]";
const SPACE = "[[SPACE]]";
2017-06-25 01:19:11 +02:00
function processMdArrayNewLines(md) {
while (md.length && md[0] == BLOCK_OPEN) {
md.shift();
}
while (md.length && md[md.length - 1] == BLOCK_CLOSE) {
md.pop();
}
let temp = [];
let last = '';
for (let i = 0; i < md.length; i++) { let v = md[i];
if (isNewLineBlock(last) && isNewLineBlock(v) && last == v) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == BLOCK_CLOSE && v == BLOCK_OPEN) {
temp.pop();
temp.push(NEWLINE_MERGED);
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_CLOSE)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
// NEW!!!
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_OPEN)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
if (md.length > 2) {
if (md[md.length - 2] == NEWLINE_MERGED && md[md.length - 1] == NEWLINE) {
md.pop();
}
}
let output = '';
let previous = '';
let start = true;
for (let i = 0; i < md.length; i++) { let v = md[i];
let add = '';
if (v == BLOCK_CLOSE || v == BLOCK_OPEN || v == NEWLINE || v == NEWLINE_MERGED) {
add = "\n";
} else if (v == SPACE) {
if (previous == SPACE || previous == "\n" || start) {
continue; // skip
} else {
add = " ";
}
} else {
add = v;
}
start = false;
output += add;
previous = add;
}
if (!output.trim().length) return '';
return output;
}
function isWhiteSpace(c) {
return c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\t' || c == ' ';
}
// Like QString::simpified(), except that it preserves non-breaking spaces (which
// Evernote uses for identation, etc.)
function simplifyString(s) {
let output = '';
let previousWhite = false;
for (let i = 0; i < s.length; i++) {
let c = s[i];
let isWhite = isWhiteSpace(c);
if (previousWhite && isWhite) {
// skip
} else {
output += c;
}
previousWhite = isWhite;
}
while (output.length && isWhiteSpace(output[0])) output = output.substr(1);
while (output.length && isWhiteSpace(output[output.length - 1])) output = output.substr(0, output.length - 1);
return output;
}
function collapseWhiteSpaceAndAppend(lines, state, text) {
if (state.inCode) {
text = "\t" + text;
lines.push(text);
} else {
// Remove all \n and \r from the left and right of the text
while (text.length && (text[0] == "\n" || text[0] == "\r")) text = text.substr(1);
while (text.length && (text[text.length - 1] == "\n" || text[text.length - 1] == "\r")) text = text.substr(0, text.length - 1);
// Collapse all white spaces to just one. If there are spaces to the left and right of the string
// also collapse them to just one space.
let spaceLeft = text.length && text[0] == ' ';
let spaceRight = text.length && text[text.length - 1] == ' ';
text = simplifyString(text);
if (!spaceLeft && !spaceRight && text == "") return lines;
if (spaceLeft) lines.push(SPACE);
lines.push(text);
if (spaceRight) lines.push(SPACE);
}
return lines;
}
const imageMimeTypes = ["image/cgm", "image/fits", "image/g3fax", "image/gif", "image/ief", "image/jp2", "image/jpeg", "image/jpm", "image/jpx", "image/naplps", "image/png", "image/prs.btif", "image/prs.pti", "image/t38", "image/tiff", "image/tiff-fx", "image/vnd.adobe.photoshop", "image/vnd.cns.inf2", "image/vnd.djvu", "image/vnd.dwg", "image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst", "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc", "image/vnd.globalgraphics.pgb", "image/vnd.microsoft.icon", "image/vnd.mix", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.sealed.png", "image/vnd.sealedmedia.softseal.gif", "image/vnd.sealedmedia.softseal.jpg", "image/vnd.svf", "image/vnd.wap.wbmp", "image/vnd.xiff"];
function isImageMimeType(m) {
return imageMimeTypes.indexOf(m) >= 0;
}
function addResourceTag(lines, resource, alt = "") {
let tagAlt = alt == "" ? resource.alt : alt;
if (!tagAlt) tagAlt = '';
if (isImageMimeType(resource.mime)) {
lines.push("![");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
} else {
lines.push("[");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
}
return lines;
}
function isBlockTag(n) {
2017-07-13 21:29:10 +02:00
return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center";
2017-06-25 01:19:11 +02:00
}
function isStrongTag(n) {
return n == "strong" || n == "b";
}
function isEmTag(n) {
return n == "em" || n == "i" || n == "u";
}
function isAnchor(n) {
return n == "a";
}
function isIgnoredEndTag(n) {
2017-07-13 21:29:10 +02:00
return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s' || n == 'tbody';
2017-06-25 01:19:11 +02:00
}
function isListTag(n) {
return n == "ol" || n == "ul";
}
// Elements that don't require any special treatment beside adding a newline character
function isNewLineOnlyEndTag(n) {
2017-07-13 21:29:10 +02:00
return n=="div" || n=="p" || n=="li" || n=="h1" || n=="h2" || n=="h3" || n=="h4" || n=="h5" || n=="dl" || n=="dd" || n=="center";
2017-06-25 01:19:11 +02:00
}
function isCodeTag(n) {
return n == "pre" || n == "code";
}
function isNewLineBlock(s) {
return s == BLOCK_OPEN || s == BLOCK_CLOSE;
}
function xmlNodeText(xmlNode) {
if (!xmlNode || !xmlNode.length) return '';
return xmlNode[0];
}
function enexXmlToMdArray(stream, resources) {
resources = resources.slice();
return new Promise((resolve, reject) => {
let state = {
inCode: false,
lists: [],
anchorAttributes: [],
};
let options = {};
let strict = true;
var saxStream = require('sax').createStream(strict, options)
2017-07-13 21:29:10 +02:00
let section = {
type: 'text',
lines: [],
parent: null,
};
2017-06-25 01:19:11 +02:00
saxStream.on('error', function(e) {
reject(e);
})
saxStream.on('text', function(text) {
2017-07-13 21:29:10 +02:00
section.lines = collapseWhiteSpaceAndAppend(section.lines, state, text);
2017-06-25 01:19:11 +02:00
})
saxStream.on('opentag', function(node) {
let n = node.name.toLowerCase();
if (n == 'en-note') {
// Start of note
} else if (isBlockTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN);
} else if (n == 'table') {
let newSection = {
type: 'table',
lines: [],
parent: section,
};
section.lines.push(newSection);
section = newSection;
} else if (n == 'tbody') {
// Ignore it
} else if (n == 'tr') {
if (section.type != 'table') throw new Error('Found a <tr> tag outside of a table');
let newSection = {
type: 'tr',
lines: [],
parent: section,
2017-07-17 20:59:40 +02:00
isHeader: false,
2017-07-13 21:29:10 +02:00
}
section.lines.push(newSection);
section = newSection;
} else if (n == 'td' || n == 'th') {
if (section.type != 'tr') throw new Error('Found a <td> tag outside of a <tr>');
2017-07-17 20:59:40 +02:00
if (n == 'th') section.isHeader = true;
2017-07-13 21:29:10 +02:00
let newSection = {
type: 'td',
lines: [],
parent: section,
};
section.lines.push(newSection);
section = newSection;
2017-06-25 01:19:11 +02:00
} else if (isListTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN);
2017-06-25 01:19:11 +02:00
state.lists.push({ tag: n, counter: 1 });
} else if (n == 'li') {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN);
2017-06-25 01:19:11 +02:00
if (!state.lists.length) {
reject("Found <li> tag without being inside a list"); // TODO: could be a warning, but nothing to handle warnings at the moment
return;
}
let container = state.lists[state.lists.length - 1];
if (container.tag == "ul") {
2017-07-13 21:29:10 +02:00
section.lines.push("- ");
2017-06-25 01:19:11 +02:00
} else {
2017-07-13 21:29:10 +02:00
section.lines.push(container.counter + '. ');
2017-06-25 01:19:11 +02:00
container.counter++;
}
} else if (isStrongTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push("**");
2017-06-25 01:19:11 +02:00
} else if (n == 's') {
// Not supported
} else if (isAnchor(n)) {
state.anchorAttributes.push(node.attributes);
2017-07-13 21:29:10 +02:00
section.lines.push('[');
2017-06-25 01:19:11 +02:00
} else if (isEmTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push("*");
2017-06-25 01:19:11 +02:00
} else if (n == "en-todo") {
let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' ';
2017-07-13 21:29:10 +02:00
section.lines.push('- [' + x + '] ');
2017-06-25 01:19:11 +02:00
} else if (n == "hr") {
2017-07-15 18:14:15 +02:00
// Needs to be surrounded by new lines so that it's properly rendered as a line when converting to HTML
section.lines.push(NEWLINE);
section.lines.push('----------------------------------------');
section.lines.push(NEWLINE);
section.lines.push(NEWLINE);
2017-06-25 01:19:11 +02:00
} else if (n == "h1") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("# ");
2017-06-25 01:19:11 +02:00
} else if (n == "h2") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("## ");
2017-06-25 01:19:11 +02:00
} else if (n == "h3") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("### ");
2017-06-25 01:19:11 +02:00
} else if (n == "h4") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("#### ");
2017-06-25 01:19:11 +02:00
} else if (n == "h5") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("##### ");
2017-06-25 01:19:11 +02:00
} else if (n == "h6") {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN); section.lines.push("###### ");
2017-06-25 01:19:11 +02:00
} else if (isCodeTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_OPEN);
2017-06-25 01:19:11 +02:00
state.inCode = true;
} else if (n == "br") {
2017-07-13 21:29:10 +02:00
section.lines.push(NEWLINE);
2017-06-25 01:19:11 +02:00
} else if (n == "en-media") {
const hash = node.attributes.hash;
let resource = null;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (r.id == hash) {
resource = r;
resources.splice(i, 1);
break;
}
}
if (!resource) {
// This is a bit of a hack. Notes sometime have resources attached to it, but those <resource> tags don't contain
// an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note
// will contain a corresponding <en-media/> tag, which has the ID in the "hash" attribute. All this information
// has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an
// example of note that shows this problem:
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export2.dtd">
// <en-export export-date="20161221T203133Z" application="Evernote/Windows" version="6.x">
// <note>
// <title>Commande</title>
// <content>
// <![CDATA[
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
// <en-note>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media>
// </en-note>
// ]]>
// </content>
// <created>20160921T203424Z</created>
// <updated>20160921T203438Z</updated>
// <note-attributes>
// <reminder-order>20160902T140445Z</reminder-order>
// <reminder-done-time>20160924T101120Z</reminder-done-time>
// </note-attributes>
// <resource>
// <data encoding="base64">........</data>
// <mime>image/png</mime>
// <width>150</width>
// <height>150</height>
// </resource>
// </note>
// </en-export>
let found = false;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (!r.id) {
r.id = hash;
resources[i] = r;
found = true;
break;
}
}
if (!found) {
console.warn('Hash with no associated resource: ' + hash);
}
} else {
// If the resource does not appear among the note's resources, it
// means it's an attachement. It will be appended along with the
// other remaining resources at the bottom of the markdown text.
if (!!resource.id) {
2017-07-13 21:29:10 +02:00
section.lines = addResourceTag(section.lines, resource, node.attributes.alt);
2017-06-25 01:19:11 +02:00
}
}
} else if (n == "span" || n == "font") {
// Ignore
} else {
console.warn("Unsupported start tag: " + n);
}
})
saxStream.on('closetag', function(n) {
if (n == 'en-note') {
// End of note
} else if (isNewLineOnlyEndTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_CLOSE);
} else if (n == 'td' || n == 'th') {
section = section.parent;
} else if (n == 'tr') {
section = section.parent;
} else if (n == 'table') {
section = section.parent;
2017-06-25 01:19:11 +02:00
} else if (isIgnoredEndTag(n)) {
// Skip
} else if (isListTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_CLOSE);
2017-06-25 01:19:11 +02:00
state.lists.pop();
} else if (isStrongTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push("**");
2017-06-25 01:19:11 +02:00
} else if (isEmTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push("*");
2017-06-25 01:19:11 +02:00
} else if (isCodeTag(n)) {
state.inCode = false;
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_CLOSE);
2017-06-25 01:19:11 +02:00
} else if (isAnchor(n)) {
let attributes = state.anchorAttributes.pop();
let url = attributes && attributes.href ? attributes.href : '';
2017-07-13 21:16:01 +02:00
2017-07-13 21:29:10 +02:00
if (section.lines.length < 1) throw new Error('Invalid anchor tag closing'); // Sanity check, but normally not possible
2017-07-13 21:16:01 +02:00
// When closing the anchor tag, check if there's is any text content. If not
// put the URL as is (don't wrap it in [](url)). The markdown parser, using
// GitHub flavour, will turn this URL into a link. This is to generate slightly
// cleaner markdown.
2017-07-13 21:29:10 +02:00
let previous = section.lines[section.lines.length - 1];
2017-07-13 21:16:01 +02:00
if (previous == '[') {
2017-07-13 21:29:10 +02:00
section.lines.pop();
section.lines.push(url);
2017-07-13 21:16:01 +02:00
} else if (!previous || previous == url) {
2017-07-13 21:29:10 +02:00
section.lines.pop();
section.lines.pop();
section.lines.push(url);
2017-07-13 21:16:01 +02:00
} else {
2017-07-13 21:29:10 +02:00
section.lines.push('](' + url + ')');
2017-07-13 21:16:01 +02:00
}
2017-06-25 01:19:11 +02:00
} else if (isListTag(n)) {
2017-07-13 21:29:10 +02:00
section.lines.push(BLOCK_CLOSE);
2017-06-25 01:19:11 +02:00
state.lists.pop();
} else if (n == "en-media") {
// Skip
} else if (isIgnoredEndTag(n)) {
// Skip
} else {
console.warn("Unsupported end tag: " + n);
}
})
saxStream.on('attribute', function(attr) {
})
saxStream.on('end', function() {
resolve({
2017-07-13 21:29:10 +02:00
content: section,
2017-06-25 01:19:11 +02:00
resources: resources,
});
})
stream.pipe(saxStream);
});
}
2017-07-17 20:59:40 +02:00
function setTableCellContent(table) {
if (!table.type == 'table') throw new Error('Only for tables');
for (let trIndex = 0; trIndex < table.lines.length; trIndex++) {
const tr = table.lines[trIndex];
for (let tdIndex = 0; tdIndex < tr.lines.length; tdIndex++) {
2017-07-19 00:38:07 +02:00
let td = tr.lines[tdIndex];
2017-07-17 20:59:40 +02:00
td.content = processMdArrayNewLines(td.lines);
td.content = td.content.replace(/\n\n\n\n\n/g, ' ');
td.content = td.content.replace(/\n\n\n\n/g, ' ');
td.content = td.content.replace(/\n\n\n/g, ' ');
td.content = td.content.replace(/\n\n/g, ' ');
td.content = td.content.replace(/\n/g, ' ');
2017-07-13 21:29:10 +02:00
}
}
2017-07-17 20:59:40 +02:00
return table;
2017-07-13 21:29:10 +02:00
}
2017-07-17 20:59:40 +02:00
function cellWidth(cellText) {
const lines = cellText.split("\n");
let maxWidth = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.length > maxWidth) maxWidth = line.length;
}
return maxWidth;
}
2017-07-13 21:29:10 +02:00
2017-07-17 20:59:40 +02:00
function colWidths(table) {
let output = [];
for (let trIndex = 0; trIndex < table.lines.length; trIndex++) {
const tr = table.lines[trIndex];
for (let tdIndex = 0; tdIndex < tr.lines.length; tdIndex++) {
const td = tr.lines[tdIndex];
const w = cellWidth(td.content);
if (output.length <= tdIndex) output.push(0);
if (w > output[tdIndex]) output[tdIndex] = w;
}
}
return output;
}
2017-07-13 21:29:10 +02:00
2017-07-17 20:59:40 +02:00
function drawTable(table, colWidths) {
// | First Header | Second Header |
// | ------------- | ------------- |
// | Content Cell | Content Cell |
// | Content Cell | Content Cell |
// There must be at least 3 dashes separating each header cell.
// https://gist.github.com/IanWang/28965e13cdafdef4e11dc91f578d160d#tables
const minColWidth = 3;
2017-07-17 20:59:40 +02:00
let lines = [];
let headerDone = false;
for (let trIndex = 0; trIndex < table.lines.length; trIndex++) {
const tr = table.lines[trIndex];
const isHeader = tr.isHeader;
let line = [];
let headerLine = [];
let emptyHeader = null;
for (let tdIndex = 0; tdIndex < colWidths.length; tdIndex++) {
const width = Math.max(minColWidth, colWidths[tdIndex]);
2017-07-19 00:38:07 +02:00
const cell = tr.lines[tdIndex] ? tr.lines[tdIndex].content : '';
2017-07-17 20:59:40 +02:00
line.push(stringPadding(cell, width, ' ', stringPadding.RIGHT));
if (!headerDone) {
if (!isHeader) {
if (!emptyHeader) emptyHeader = [];
let h = stringPadding(' ', width, ' ', stringPadding.RIGHT);
if (!width) h = '';
emptyHeader.push(h);
2017-07-13 21:29:10 +02:00
}
2017-07-17 20:59:40 +02:00
headerLine.push('-'.repeat(width));
2017-07-13 21:29:10 +02:00
}
2017-07-17 20:59:40 +02:00
}
if (emptyHeader) {
lines.push('| ' + emptyHeader.join(' | ') + ' |');
lines.push('| ' + headerLine.join(' | ') + ' |');
headerDone = true;
}
lines.push('| ' + line.join(' | ') + ' |');
if (!headerDone) {
lines.push('| ' + headerLine.join(' | ') + ' |');
headerDone = true;
2017-07-13 21:29:10 +02:00
}
}
2017-07-17 20:59:40 +02:00
return lines.join('<<<<:D>>>>' + NEWLINE + '<<<<:D>>>>').split('<<<<:D>>>>');
2017-07-13 21:29:10 +02:00
}
2017-06-25 01:19:11 +02:00
async function enexXmlToMd(stream, resources) {
let result = await enexXmlToMdArray(stream, resources);
2017-07-13 21:16:01 +02:00
2017-07-17 20:59:40 +02:00
let mdLines = [];
for (let i = 0; i < result.content.lines.length; i++) {
let line = result.content.lines[i];
if (typeof line === 'object') { // A table
let table = setTableCellContent(line);
2017-07-19 00:38:07 +02:00
//console.log(require('util').inspect(table, false, null))
2017-07-17 20:59:40 +02:00
const cw = colWidths(table);
const tableLines = drawTable(table, cw);
mdLines.push(BLOCK_OPEN);
mdLines = mdLines.concat(tableLines);
mdLines.push(BLOCK_CLOSE);
} else { // an actual line
mdLines.push(line);
}
}
2017-07-13 21:29:10 +02:00
2017-06-25 01:19:11 +02:00
let firstAttachment = true;
for (let i = 0; i < result.resources.length; i++) {
let r = result.resources[i];
if (firstAttachment) mdLines.push(NEWLINE);
mdLines.push(NEWLINE);
mdLines = addResourceTag(mdLines, r, r.filename);
firstAttachment = false;
}
return processMdArrayNewLines(mdLines);
}
export { enexXmlToMd, processMdArrayNewLines, NEWLINE, addResourceTag };