diff --git a/CliClient/app/import-enex-md-gen.js b/CliClient/app/import-enex-md-gen.js new file mode 100644 index 000000000..c24d310fb --- /dev/null +++ b/CliClient/app/import-enex-md-gen.js @@ -0,0 +1,443 @@ +const BLOCK_OPEN = "
"; +const BLOCK_CLOSE = "
"; +const NEWLINE = "
"; +const NEWLINE_MERGED = ""; +const SPACE = ""; + +function processMdArrayNewLines(md) { + while (md.length && md[0] == BLOCK_OPEN) { + md.shift(); + } + + while (md.length && md[md.length - 1] == BLOCK_CLOSE) { + md.pop(); + } + + let temp = []; + let last = ''; + for (let i = 0; i < md.length; i++) { let v = md[i]; + if (isNewLineBlock(last) && isNewLineBlock(v) && last == v) { + // Skip it + } else { + temp.push(v); + } + last = v; + } + md = temp; + + + + temp = []; + last = ""; + for (let i = 0; i < md.length; i++) { let v = md[i]; + if (last == BLOCK_CLOSE && v == BLOCK_OPEN) { + temp.pop(); + temp.push(NEWLINE_MERGED); + } else { + temp.push(v); + } + last = v; + } + md = temp; + + + + temp = []; + last = ""; + for (let i = 0; i < md.length; i++) { let v = md[i]; + if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_CLOSE)) { + // Skip it + } else { + temp.push(v); + } + last = v; + } + md = temp; + + + + // NEW!!! + temp = []; + last = ""; + for (let i = 0; i < md.length; i++) { let v = md[i]; + if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_OPEN)) { + // Skip it + } else { + temp.push(v); + } + last = v; + } + md = temp; + + + + + if (md.length > 2) { + if (md[md.length - 2] == NEWLINE_MERGED && md[md.length - 1] == NEWLINE) { + md.pop(); + } + } + + let output = ''; + let previous = ''; + let start = true; + for (let i = 0; i < md.length; i++) { let v = md[i]; + let add = ''; + if (v == BLOCK_CLOSE || v == BLOCK_OPEN || v == NEWLINE || v == NEWLINE_MERGED) { + add = "\n"; + } else if (v == SPACE) { + if (previous == SPACE || previous == "\n" || start) { + continue; // skip + } else { + add = " "; + } + } else { + add = v; + } + start = false; + output += add; + previous = add; + } + + if (!output.trim().length) return ''; + + return output; +} + +function isWhiteSpace(c) { + return c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\t' || c == ' '; +} + +// Like QString::simpified(), except that it preserves non-breaking spaces (which +// Evernote uses for identation, etc.) +function simplifyString(s) { + let output = ''; + let previousWhite = false; + for (let i = 0; i < s.length; i++) { + let c = s[i]; + let isWhite = isWhiteSpace(c); + if (previousWhite && isWhite) { + // skip + } else { + output += c; + } + previousWhite = isWhite; + } + + while (output.length && isWhiteSpace(output[0])) output = output.substr(1); + while (output.length && isWhiteSpace(output[output.length - 1])) output = output.substr(0, output.length - 1); + + return output; +} + +function collapseWhiteSpaceAndAppend(lines, state, text) { + if (state.inCode) { + text = "\t" + text; + lines.push(text); + } else { + // Remove all \n and \r from the left and right of the text + while (text.length && (text[0] == "\n" || text[0] == "\r")) text = text.substr(1); + while (text.length && (text[text.length - 1] == "\n" || text[text.length - 1] == "\r")) text = text.substr(0, text.length - 1); + + // Collapse all white spaces to just one. If there are spaces to the left and right of the string + // also collapse them to just one space. + let spaceLeft = text.length && text[0] == ' '; + let spaceRight = text.length && text[text.length - 1] == ' '; + text = simplifyString(text); + + if (!spaceLeft && !spaceRight && text == "") return lines; + + if (spaceLeft) lines.push(SPACE); + lines.push(text); + if (spaceRight) lines.push(SPACE); + } + + return lines; +} + +const imageMimeTypes = ["image/cgm", "image/fits", "image/g3fax", "image/gif", "image/ief", "image/jp2", "image/jpeg", "image/jpm", "image/jpx", "image/naplps", "image/png", "image/prs.btif", "image/prs.pti", "image/t38", "image/tiff", "image/tiff-fx", "image/vnd.adobe.photoshop", "image/vnd.cns.inf2", "image/vnd.djvu", "image/vnd.dwg", "image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst", "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc", "image/vnd.globalgraphics.pgb", "image/vnd.microsoft.icon", "image/vnd.mix", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.sealed.png", "image/vnd.sealedmedia.softseal.gif", "image/vnd.sealedmedia.softseal.jpg", "image/vnd.svf", "image/vnd.wap.wbmp", "image/vnd.xiff"]; + +function isImageMimeType(m) { + return imageMimeTypes.indexOf(m) >= 0; +} + +function addResourceTag(lines, resource, alt = "") { + let tagAlt = alt == "" ? resource.alt : alt; + if (!tagAlt) tagAlt = ''; + if (isImageMimeType(resource.mime)) { + lines.push("!["); + lines.push(tagAlt); + lines.push("](:/" + resource.id + ")"); + } else { + lines.push("["); + lines.push(tagAlt); + lines.push("](:/" + resource.id + ")"); + } + + return lines; +} + + +function isBlockTag(n) { + return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody"; +} + +function isStrongTag(n) { + return n == "strong" || n == "b"; +} + +function isEmTag(n) { + return n == "em" || n == "i" || n == "u"; +} + +function isAnchor(n) { + return n == "a"; +} + +function isIgnoredEndTag(n) { + return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s'; +} + +function isListTag(n) { + return n == "ol" || n == "ul"; +} + +// Elements that don't require any special treatment beside adding a newline character +function isNewLineOnlyEndTag(n) { + return n=="div" || n=="p" || n=="li" || n=="h1" || n=="h2" || n=="h3" || n=="h4" || n=="h5" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody"; +} + +function isCodeTag(n) { + return n == "pre" || n == "code"; +} + +function isNewLineBlock(s) { + return s == BLOCK_OPEN || s == BLOCK_CLOSE; +} + +function xmlNodeText(xmlNode) { + if (!xmlNode || !xmlNode.length) return ''; + return xmlNode[0]; +} + +function enexXmlToMdArray(stream, resources) { + resources = resources.slice(); + + return new Promise((resolve, reject) => { + let output = []; + + let state = { + inCode: false, + lists: [], + anchorAttributes: [], + }; + + let options = {}; + let strict = true; + var saxStream = require('sax').createStream(strict, options) + + saxStream.on('error', function(e) { + reject(e); + }) + + saxStream.on('text', function(text) { + output = collapseWhiteSpaceAndAppend(output, state, text); + }) + + saxStream.on('opentag', function(node) { + let n = node.name.toLowerCase(); + if (n == 'en-note') { + // Start of note + } else if (isBlockTag(n)) { + output.push(BLOCK_OPEN); + } else if (isListTag(n)) { + output.push(BLOCK_OPEN); + state.lists.push({ tag: n, counter: 1 }); + } else if (n == 'li') { + output.push(BLOCK_OPEN); + if (!state.lists.length) { + reject("Found
  • tag without being inside a list"); // TODO: could be a warning, but nothing to handle warnings at the moment + return; + } + + let container = state.lists[state.lists.length - 1]; + if (container.tag == "ul") { + output.push("- "); + } else { + output.push(container.counter + '. '); + container.counter++; + } + } else if (isStrongTag(n)) { + output.push("**"); + } else if (n == 's') { + // Not supported + } else if (isAnchor(n)) { + state.anchorAttributes.push(node.attributes); + output.push('['); + } else if (isEmTag(n)) { + output.push("*"); + } else if (n == "en-todo") { + let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' '; + output.push('- [' + x + '] '); + } else if (n == "hr") { + output.push('------------------------------------------------------------------------------'); + } else if (n == "h1") { + output.push(BLOCK_OPEN); output.push("# "); + } else if (n == "h2") { + output.push(BLOCK_OPEN); output.push("## "); + } else if (n == "h3") { + output.push(BLOCK_OPEN); output.push("### "); + } else if (n == "h4") { + output.push(BLOCK_OPEN); output.push("#### "); + } else if (n == "h5") { + output.push(BLOCK_OPEN); output.push("##### "); + } else if (n == "h6") { + output.push(BLOCK_OPEN); output.push("###### "); + } else if (isCodeTag(n)) { + output.push(BLOCK_OPEN); + state.inCode = true; + } else if (n == "br") { + output.push(NEWLINE); + } else if (n == "en-media") { + const hash = node.attributes.hash; + + let resource = null; + for (let i = 0; i < resources.length; i++) { + let r = resources[i]; + if (r.id == hash) { + resource = r; + resources.splice(i, 1); + break; + } + } + + if (!resource) { + // This is a bit of a hack. Notes sometime have resources attached to it, but those tags don't contain + // an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note + // will contain a corresponding tag, which has the ID in the "hash" attribute. All this information + // has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an + // example of note that shows this problem: + + // + // + // + // + // Commande + // + // + // + // + // + // + // ]]> + // + // 20160921T203424Z + // 20160921T203438Z + // + // 20160902T140445Z + // 20160924T101120Z + // + // + // ........ + // image/png + // 150 + // 150 + // + // + // + + let found = false; + for (let i = 0; i < resources.length; i++) { + let r = resources[i]; + if (!r.id) { + r.id = hash; + resources[i] = r; + found = true; + break; + } + } + + if (!found) { + console.warn('Hash with no associated resource: ' + hash); + } + } else { + // If the resource does not appear among the note's resources, it + // means it's an attachement. It will be appended along with the + // other remaining resources at the bottom of the markdown text. + if (!!resource.id) { + output = addResourceTag(output, resource, node.attributes.alt); + } + } + } else if (n == "span" || n == "font") { + // Ignore + } else { + console.warn("Unsupported start tag: " + n); + } + }) + + saxStream.on('closetag', function(n) { + if (n == 'en-note') { + // End of note + } else if (isNewLineOnlyEndTag(n)) { + output.push(BLOCK_CLOSE); + } else if (isIgnoredEndTag(n)) { + // Skip + } else if (isListTag(n)) { + output.push(BLOCK_CLOSE); + state.lists.pop(); + } else if (isStrongTag(n)) { + output.push("**"); + } else if (isEmTag(n)) { + output.push("*"); + } else if (isCodeTag(n)) { + state.inCode = false; + output.push(BLOCK_CLOSE); + } else if (isAnchor(n)) { + let attributes = state.anchorAttributes.pop(); + let url = attributes && attributes.href ? attributes.href : ''; + output.push('](' + url + ')'); + } else if (isListTag(n)) { + output.push(BLOCK_CLOSE); + state.lists.pop(); + } else if (n == "en-media") { + // Skip + } else if (isIgnoredEndTag(n)) { + // Skip + } else { + console.warn("Unsupported end tag: " + n); + } + + }) + + saxStream.on('attribute', function(attr) { + + }) + + saxStream.on('end', function() { + resolve({ + lines: output, + resources: resources, + }); + }) + + stream.pipe(saxStream); + }); +} + +async function enexXmlToMd(stream, resources) { + let result = await enexXmlToMdArray(stream, resources); + let mdLines = result.lines; + let firstAttachment = true; + for (let i = 0; i < result.resources.length; i++) { + let r = result.resources[i]; + if (firstAttachment) mdLines.push(NEWLINE); + mdLines.push(NEWLINE); + mdLines = addResourceTag(mdLines, r, r.filename); + firstAttachment = false; + } + + return processMdArrayNewLines(mdLines); +} + +export { enexXmlToMd, processMdArrayNewLines, NEWLINE, addResourceTag }; \ No newline at end of file diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index 14072479c..7c129d89f 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -8,439 +8,13 @@ import { BaseModel } from 'lib/base-model.js'; import { Note } from 'lib/models/note.js'; import { Resource } from 'lib/models/resource.js'; import { Folder } from 'lib/models/folder.js'; +import { enexXmlToMd } from './import-enex-md-gen.js'; import jsSHA from "jssha"; const Promise = require('promise'); const fs = require('fs-extra'); const stringToStream = require('string-to-stream') -const BLOCK_OPEN = "
    "; -const BLOCK_CLOSE = "
    "; -const NEWLINE = "
    "; -const NEWLINE_MERGED = ""; -const SPACE = ""; - -function processMdArrayNewLines(md) { - while (md.length && md[0] == BLOCK_OPEN) { - md.shift(); - } - - while (md.length && md[md.length - 1] == BLOCK_CLOSE) { - md.pop(); - } - - let temp = []; - let last = ''; - for (let i = 0; i < md.length; i++) { let v = md[i]; - if (isNewLineBlock(last) && isNewLineBlock(v) && last == v) { - // Skip it - } else { - temp.push(v); - } - last = v; - } - md = temp; - - - - temp = []; - last = ""; - for (let i = 0; i < md.length; i++) { let v = md[i]; - if (last == BLOCK_CLOSE && v == BLOCK_OPEN) { - temp.pop(); - temp.push(NEWLINE_MERGED); - } else { - temp.push(v); - } - last = v; - } - md = temp; - - - - temp = []; - last = ""; - for (let i = 0; i < md.length; i++) { let v = md[i]; - if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_CLOSE)) { - // Skip it - } else { - temp.push(v); - } - last = v; - } - md = temp; - - - - // NEW!!! - temp = []; - last = ""; - for (let i = 0; i < md.length; i++) { let v = md[i]; - if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_OPEN)) { - // Skip it - } else { - temp.push(v); - } - last = v; - } - md = temp; - - - - - if (md.length > 2) { - if (md[md.length - 2] == NEWLINE_MERGED && md[md.length - 1] == NEWLINE) { - md.pop(); - } - } - - let output = ''; - let previous = ''; - let start = true; - for (let i = 0; i < md.length; i++) { let v = md[i]; - let add = ''; - if (v == BLOCK_CLOSE || v == BLOCK_OPEN || v == NEWLINE || v == NEWLINE_MERGED) { - add = "\n"; - } else if (v == SPACE) { - if (previous == SPACE || previous == "\n" || start) { - continue; // skip - } else { - add = " "; - } - } else { - add = v; - } - start = false; - output += add; - previous = add; - } - - if (!output.trim().length) return ''; - - return output; -} - -function isWhiteSpace(c) { - return c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\t' || c == ' '; -} - -// Like QString::simpified(), except that it preserves non-breaking spaces (which -// Evernote uses for identation, etc.) -function simplifyString(s) { - let output = ''; - let previousWhite = false; - for (let i = 0; i < s.length; i++) { - let c = s[i]; - let isWhite = isWhiteSpace(c); - if (previousWhite && isWhite) { - // skip - } else { - output += c; - } - previousWhite = isWhite; - } - - while (output.length && isWhiteSpace(output[0])) output = output.substr(1); - while (output.length && isWhiteSpace(output[output.length - 1])) output = output.substr(0, output.length - 1); - - return output; -} - -function collapseWhiteSpaceAndAppend(lines, state, text) { - if (state.inCode) { - text = "\t" + text; - lines.push(text); - } else { - // Remove all \n and \r from the left and right of the text - while (text.length && (text[0] == "\n" || text[0] == "\r")) text = text.substr(1); - while (text.length && (text[text.length - 1] == "\n" || text[text.length - 1] == "\r")) text = text.substr(0, text.length - 1); - - // Collapse all white spaces to just one. If there are spaces to the left and right of the string - // also collapse them to just one space. - let spaceLeft = text.length && text[0] == ' '; - let spaceRight = text.length && text[text.length - 1] == ' '; - text = simplifyString(text); - - if (!spaceLeft && !spaceRight && text == "") return lines; - - if (spaceLeft) lines.push(SPACE); - lines.push(text); - if (spaceRight) lines.push(SPACE); - } - - return lines; -} - -const imageMimeTypes = ["image/cgm", "image/fits", "image/g3fax", "image/gif", "image/ief", "image/jp2", "image/jpeg", "image/jpm", "image/jpx", "image/naplps", "image/png", "image/prs.btif", "image/prs.pti", "image/t38", "image/tiff", "image/tiff-fx", "image/vnd.adobe.photoshop", "image/vnd.cns.inf2", "image/vnd.djvu", "image/vnd.dwg", "image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst", "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc", "image/vnd.globalgraphics.pgb", "image/vnd.microsoft.icon", "image/vnd.mix", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.sealed.png", "image/vnd.sealedmedia.softseal.gif", "image/vnd.sealedmedia.softseal.jpg", "image/vnd.svf", "image/vnd.wap.wbmp", "image/vnd.xiff"]; - -function isImageMimeType(m) { - return imageMimeTypes.indexOf(m) >= 0; -} - -function addResourceTag(lines, resource, alt = "") { - let tagAlt = alt == "" ? resource.alt : alt; - if (!tagAlt) tagAlt = ''; - if (isImageMimeType(resource.mime)) { - lines.push("!["); - lines.push(tagAlt); - lines.push("](:/" + resource.id + ")"); - } else { - lines.push("["); - lines.push(tagAlt); - lines.push("](:/" + resource.id + ")"); - } - - return lines; -} - - -function enexXmlToMd(stream, resources) { - resources = resources.slice(); - - return new Promise((resolve, reject) => { - let output = []; - - let state = { - inCode: false, - lists: [], - anchorAttributes: [], - }; - - let options = {}; - let strict = true; - var saxStream = require('sax').createStream(strict, options) - - saxStream.on('error', function(e) { - reject(e); - }) - - saxStream.on('text', function(text) { - output = collapseWhiteSpaceAndAppend(output, state, text); - }) - - saxStream.on('opentag', function(node) { - let n = node.name.toLowerCase(); - if (n == 'en-note') { - // Start of note - } else if (isBlockTag(n)) { - output.push(BLOCK_OPEN); - } else if (isListTag(n)) { - output.push(BLOCK_OPEN); - state.lists.push({ tag: n, counter: 1 }); - } else if (n == 'li') { - output.push(BLOCK_OPEN); - if (!state.lists.length) { - reject("Found
  • tag without being inside a list"); // TODO: could be a warning, but nothing to handle warnings at the moment - return; - } - - let container = state.lists[state.lists.length - 1]; - if (container.tag == "ul") { - output.push("- "); - } else { - output.push(container.counter + '. '); - container.counter++; - } - } else if (isStrongTag(n)) { - output.push("**"); - } else if (n == 's') { - // Not supported - } else if (isAnchor(n)) { - state.anchorAttributes.push(node.attributes); - output.push('['); - } else if (isEmTag(n)) { - output.push("*"); - } else if (n == "en-todo") { - let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' '; - output.push('- [' + x + '] '); - } else if (n == "hr") { - output.push('------------------------------------------------------------------------------'); - } else if (n == "h1") { - output.push(BLOCK_OPEN); output.push("# "); - } else if (n == "h2") { - output.push(BLOCK_OPEN); output.push("## "); - } else if (n == "h3") { - output.push(BLOCK_OPEN); output.push("### "); - } else if (n == "h4") { - output.push(BLOCK_OPEN); output.push("#### "); - } else if (n == "h5") { - output.push(BLOCK_OPEN); output.push("##### "); - } else if (n == "h6") { - output.push(BLOCK_OPEN); output.push("###### "); - } else if (isCodeTag(n)) { - output.push(BLOCK_OPEN); - state.inCode = true; - } else if (n == "br") { - output.push(NEWLINE); - } else if (n == "en-media") { - const hash = node.attributes.hash; - - let resource = null; - for (let i = 0; i < resources.length; i++) { - let r = resources[i]; - if (r.id == hash) { - resource = r; - resources.splice(i, 1); - break; - } - } - - if (!resource) { - // This is a bit of a hack. Notes sometime have resources attached to it, but those tags don't contain - // an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note - // will contain a corresponding tag, which has the ID in the "hash" attribute. All this information - // has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an - // example of note that shows this problem: - - // - // - // - // - // Commande - // - // - // - // - // - // - // ]]> - // - // 20160921T203424Z - // 20160921T203438Z - // - // 20160902T140445Z - // 20160924T101120Z - // - // - // ........ - // image/png - // 150 - // 150 - // - // - // - - let found = false; - for (let i = 0; i < resources.length; i++) { - let r = resources[i]; - if (!r.id) { - r.id = hash; - resources[i] = r; - found = true; - break; - } - } - - if (!found) { - console.warn('Hash with no associated resource: ' + hash); - } - } else { - // If the resource does not appear among the note's resources, it - // means it's an attachement. It will be appended along with the - // other remaining resources at the bottom of the markdown text. - if (!!resource.id) { - output = addResourceTag(output, resource, node.attributes.alt); - } - } - } else if (n == "span" || n == "font") { - // Ignore - } else { - console.warn("Unsupported start tag: " + n); - } - }) - - saxStream.on('closetag', function(n) { - if (n == 'en-note') { - // End of note - } else if (isNewLineOnlyEndTag(n)) { - output.push(BLOCK_CLOSE); - } else if (isIgnoredEndTag(n)) { - // Skip - } else if (isListTag(n)) { - output.push(BLOCK_CLOSE); - state.lists.pop(); - } else if (isStrongTag(n)) { - output.push("**"); - } else if (isEmTag(n)) { - output.push("*"); - } else if (isCodeTag(n)) { - state.inCode = false; - output.push(BLOCK_CLOSE); - } else if (isAnchor(n)) { - let attributes = state.anchorAttributes.pop(); - let url = attributes && attributes.href ? attributes.href : ''; - output.push('](' + url + ')'); - } else if (isListTag(n)) { - output.push(BLOCK_CLOSE); - state.lists.pop(); - } else if (n == "en-media") { - // Skip - } else if (isIgnoredEndTag(n)) { - // Skip - } else { - console.warn("Unsupported end tag: " + n); - } - - }) - - saxStream.on('attribute', function(attr) { - - }) - - saxStream.on('end', function() { - resolve({ - lines: output, - resources: resources, - }); - }) - - stream.pipe(saxStream); - }); -} - -function isBlockTag(n) { - return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody"; -} - -function isStrongTag(n) { - return n == "strong" || n == "b"; -} - -function isEmTag(n) { - return n == "em" || n == "i" || n == "u"; -} - -function isAnchor(n) { - return n == "a"; -} - -function isIgnoredEndTag(n) { - return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s'; -} - -function isListTag(n) { - return n == "ol" || n == "ul"; -} - -// Elements that don't require any special treatment beside adding a newline character -function isNewLineOnlyEndTag(n) { - return n=="div" || n=="p" || n=="li" || n=="h1" || n=="h2" || n=="h3" || n=="h4" || n=="h5" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody"; -} - -function isCodeTag(n) { - return n == "pre" || n == "code"; -} - -function isNewLineBlock(s) { - return s == BLOCK_OPEN || s == BLOCK_CLOSE; -} - -function xmlNodeText(xmlNode) { - if (!xmlNode || !xmlNode.length) return ''; - return xmlNode[0]; -} - let existingTimestamps = []; function uniqueCreatedTimestamp(timestamp) { @@ -460,68 +34,22 @@ function uniqueCreatedTimestamp(timestamp) { return timestamp; } -function dateToTimestamp(s) { +function dateToTimestamp(s, zeroIfInvalid = false) { let m = moment(s, 'YYYYMMDDTHHmmssZ'); if (!m.isValid()) { + if (zeroIfInvalid) return 0; throw new Error('Invalid date: ' + s); } return m.toDate().getTime(); } -function evernoteXmlToMdArray(xml) { - return parseXml(xml).then((xml) => { - console.info(xml); - }); -} - function extractRecognitionObjId(recognitionXml) { const r = recognitionXml.match(/objID="(.*?)"/); return r && r.length >= 2 ? r[1] : null; } function filePutContents(filePath, content) { - return new Promise((resolve, reject) => { - fs.writeFile(filePath, content, function(error) { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); -} - -function setModifiedTime(filePath, time) { - return new Promise((resolve, reject) => { - fs.utimes(filePath, time, time, (error) => { - if (error) { - reject(error); - return; - } - resolve(); - }) - }); -} - -function createDirectory(path) { - return new Promise((resolve, reject) => { - fs.exists(path, (exists) => { - if (exists) { - resolve(); - return; - } - - const mkdirp = require('mkdirp'); - - mkdirp(path, (error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); - }); + return fs.writeFile(filePath, content); } function removeUndefinedProperties(note) { @@ -580,25 +108,21 @@ async function saveNoteToStorage(note) { diff.type_ = existingNote.type_; return Note.save(diff, { autoTimestamp: false }); } else { - - // id: noteResource.id, - // data: decodedData, - // mime: noteResource.mime, - // title: noteResource.filename, - // filename: noteResource.filename, - - // CREATE TABLE resources ( - // id TEXT PRIMARY KEY, - // title TEXT, - // mime TEXT, - // filename TEXT, - // created_time INT, - // updated_time INT - for (let i = 0; i < note.resources.length; i++) { let resource = note.resources[i]; let toSave = Object.assign({}, resource); delete toSave.data; + + // The same resource sometimes appear twice in the same enex (exact same ID and file). + // In that case, just skip it - it means two different notes might be linked to the + // same resource. + let existingResource = await Resource.load(toSave.id); + if (existingResource) { + // console.warn('Trying to save: ' + JSON.stringify(toSave)); + // console.warn('But duplicate: ' + JSON.stringify(existingResource)); + continue; + } + await Resource.save(toSave, { isNew: true }); await filePutContents(Resource.fullPath(toSave), resource.data); } @@ -646,27 +170,14 @@ function importEnex(parentFolderId, filePath) { let note = notes.shift(); const contentStream = stringToStream(note.bodyXml); chain.push(() => { - return enexXmlToMd(contentStream, note.resources).then((result) => { + return enexXmlToMd(contentStream, note.resources).then((body) => { delete note.bodyXml; - let mdLines = result.lines; - let firstAttachment = true; - for (let i = 0; i < result.resources.length; i++) { - let r = result.resources[i]; - if (firstAttachment) mdLines.push(NEWLINE); - mdLines.push(NEWLINE); - mdLines = addResourceTag(mdLines, r, r.filename); - firstAttachment = false; - } - - note.parent_id = parentFolderId; - note.body = processMdArrayNewLines(result.lines); note.id = uuid.create(); + note.parent_id = parentFolderId; + note.body = body; return saveNoteToStorage(note); - - // SAVE NOTE HERE - // saveNoteToDisk(parentFolder, note); }); }); } @@ -700,6 +211,8 @@ function importEnex(parentFolderId, filePath) { note.updated_time = dateToTimestamp(text); } else if (n == 'tag') { note.tags.push(text); + } else { + console.warn('Unsupported note tag: ' + n); } } }) @@ -763,6 +276,21 @@ function importEnex(parentFolderId, filePath) { note.longitude = noteAttributes.longitude; note.altitude = noteAttributes.altitude; note.author = noteAttributes.author; + note.is_todo = !!noteAttributes['reminder-order']; + note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true); + note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true); + note.order = dateToTimestamp(noteAttributes['reminder-order'], true); + note.source = !!noteAttributes.source ? 'evernote.' + noteAttributes.source : 'evernote'; + note.source_application = 'joplin.cli'; + + if (noteAttributes['reminder-time']) { + console.info('======================================================'); + console.info(noteAttributes); + console.info('------------------------------------------------------'); + console.info(note); + console.info('======================================================'); + } + noteAttributes = null; } else if (n == 'resource') { let decodedData = null; @@ -781,8 +309,6 @@ function importEnex(parentFolderId, filePath) { filename: noteResource.filename, }; - // r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE - note.resources.push(r); noteResource = null; } diff --git a/CliClient/app/main.js b/CliClient/app/main.js index 6e546ec67..b4971412c 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -62,7 +62,7 @@ async function main() { console.info('DELETING ALL DATA'); await db.exec('DELETE FROM notes'); await db.exec('DELETE FROM changes'); - await db.exec('DELETE FROM folders'); + await db.exec('DELETE FROM folders WHERE is_default != 1'); await db.exec('DELETE FROM resources'); await db.exec('DELETE FROM deleted_items'); await db.exec('DELETE FROM tags'); @@ -72,7 +72,7 @@ async function main() { //let folder = await Folder.loadByField('title', 'test'); - await importEnex(folder.id, '/mnt/c/Users/Laurent/Desktop/Laurent.enex'); //'/mnt/c/Users/Laurent/Desktop/Laurent.enex'); + await importEnex(folder.id, '/mnt/c/Users/Laurent/Desktop/afaire.enex'); return; diff --git a/CliClient/run_test.sh b/CliClient/run_test.sh index d5a1bd98f..4297b9df6 100755 --- a/CliClient/run_test.sh +++ b/CliClient/run_test.sh @@ -6,4 +6,5 @@ mkdir -p "$CLIENT_DIR/tests-build/data" ln -s "$CLIENT_DIR/build/lib" "$CLIENT_DIR/tests-build" #npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js -npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js \ No newline at end of file +#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js +npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/models/folder.js \ No newline at end of file diff --git a/CliClient/tests/models/folder.js b/CliClient/tests/models/folder.js new file mode 100644 index 000000000..ae2ae4422 --- /dev/null +++ b/CliClient/tests/models/folder.js @@ -0,0 +1,52 @@ +import { time } from 'lib/time-utils.js'; +import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient } from 'test-utils.js'; +import { createFoldersAndNotes } from 'test-data.js'; +import { Folder } from 'lib/models/folder.js'; +import { Note } from 'lib/models/note.js'; +import { Setting } from 'lib/models/setting.js'; +import { BaseItem } from 'lib/models/base-item.js'; +import { BaseModel } from 'lib/base-model.js'; + +process.on('unhandledRejection', (reason, p) => { + console.error('Unhandled promise rejection at: Promise', p, 'reason:', reason); +}); + +async function thereIsOnlyOneDefaultFolder() { + let count = 0; + let folders = await Folder.all(); + for (let i = 0; i < folders.length; i++) { + if (!!folders[i].is_default) count++; + } + return count === 1; +} + +describe('Folder', function() { + + beforeEach( async (done) => { + await setupDatabase(1); + switchClient(1); + done(); + }); + + it('should have one default folder only', async (done) => { + let f1 = await Folder.save({ title: 'folder1', is_default: 1 }); + let f2 = await Folder.save({ title: 'folder2' }); + let f3 = await Folder.save({ title: 'folder3' }); + + await Folder.save({ id: f2.id, is_default: 1 }); + f2 = await Folder.load(f2.id); + + expect(f2.is_default).toBe(1); + + let r = await thereIsOnlyOneDefaultFolder(); + expect(r).toBe(true); + + await Folder.save({ id: f2.id, is_default: 0 }); + f2 = await Folder.load(f2.id); + + expect(f2.is_default).toBe(1); + + done(); + }); + +}); \ No newline at end of file diff --git a/lib/base-model.js b/lib/base-model.js index 2e1a5ac7b..0abc6b041 100644 --- a/lib/base-model.js +++ b/lib/base-model.js @@ -106,6 +106,7 @@ class BaseModel { if (!('trackDeleted' in options)) options.trackDeleted = null; if (!('isNew' in options)) options.isNew = 'auto'; if (!('autoTimestamp' in options)) options.autoTimestamp = true; + if (!('transactionNextQueries' in options)) options.transactionNextQueries = []; return options; } @@ -173,7 +174,7 @@ class BaseModel { o = temp; let query = {}; - let itemId = o.id; + let modelId = o.id; if (options.autoTimestamp && this.hasField('updated_time')) { o.updated_time = time.unixMs(); @@ -181,8 +182,8 @@ class BaseModel { if (options.isNew) { if (this.useUuid() && !o.id) { - itemId = uuid.create(); - o.id = itemId; + modelId = uuid.create(); + o.id = modelId; } if (!o.created_time && this.hasField('created_time')) { @@ -197,7 +198,7 @@ class BaseModel { query = Database.updateQuery(this.tableName(), temp, where); } - query.id = itemId; + query.id = modelId; // Log.info('Saving', JSON.stringify(o)); @@ -212,42 +213,17 @@ class BaseModel { let queries = []; let saveQuery = this.saveQuery(o, options); - let itemId = saveQuery.id; + let modelId = saveQuery.id; queries.push(saveQuery); - // TODO: DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED - // if (options.trackChanges && this.trackChanges()) { - // // Cannot import this class the normal way due to cyclical dependencies between Change and BaseModel - // // which are not handled by React Native. - // const { Change } = require('src/models/change.js'); - - // if (isNew) { - // let change = Change.newChange(); - // change.type = Change.TYPE_CREATE; - // change.item_id = itemId; - // change.item_type = this.itemType(); - - // queries.push(Change.saveQuery(change)); - // } else { - // for (let n in o) { - // if (!o.hasOwnProperty(n)) continue; - // if (n == 'id') continue; - - // let change = Change.newChange(); - // change.type = Change.TYPE_UPDATE; - // change.item_id = itemId; - // change.item_type = this.itemType(); - // change.item_field = n; - - // queries.push(Change.saveQuery(change)); - // } - // } - // } + for (let i = 0; i < options.transactionNextQueries.length; i++) { + queries.push(options.transactionNextQueries[i]); + } return this.db().transactionExecBatch(queries).then(() => { o = Object.assign({}, o); - o.id = itemId; + o.id = modelId; o = this.addModelMd(o); return this.filter(o); }).catch((error) => { diff --git a/lib/database.js b/lib/database.js index 8bc417c49..b133d4ffc 100644 --- a/lib/database.js +++ b/lib/database.js @@ -26,12 +26,12 @@ CREATE TABLE notes ( latitude NUMERIC NOT NULL DEFAULT 0, longitude NUMERIC NOT NULL DEFAULT 0, altitude NUMERIC NOT NULL DEFAULT 0, - source TEXT NOT NULL DEFAULT "", author TEXT NOT NULL DEFAULT "", source_url TEXT NOT NULL DEFAULT "", is_todo INT NOT NULL DEFAULT 0, todo_due INT NOT NULL DEFAULT 0, todo_completed INT NOT NULL DEFAULT 0, + source TEXT NOT NULL DEFAULT "", source_application TEXT NOT NULL DEFAULT "", application_data TEXT NOT NULL DEFAULT "", \`order\` INT NOT NULL DEFAULT 0 @@ -163,7 +163,8 @@ class Database { this.logger().info('Database was open successfully'); return this.initialize(); }).catch((error) => { - this.logger().error('Cannot open database: ', error); + this.logger().error('Cannot open database:'); + this.logger().error(error); }); } @@ -192,7 +193,8 @@ class Database { if (queries.length <= 0) return Promise.resolve(); if (queries.length == 1) { - return this.exec(queries[0].sql, queries[0].params); + let q = this.wrapQuery(queries[0]); + return this.exec(q.sql, q.params); } // There can be only one transaction running at a time so queue @@ -282,6 +284,7 @@ class Database { } logQuery(sql, params = null) { + console.info(sql, params); if (!this.debugMode()) return; if (params !== null) { @@ -420,25 +423,6 @@ class Database { }); } }); - - - // }).then(() => { - // let p = this.exec('DELETE FROM notes').then(() => { - // return this.exec('DELETE FROM folders'); - // }).then(() => { - // return this.exec('DELETE FROM changes'); - // }).then(() => { - // return this.exec('DELETE FROM settings WHERE `key` = "sync.lastRevId"'); - // }); - - // return p.then(() => { - // return this.exec('UPDATE settings SET `value` = "' + uuid.create() + '" WHERE `key` = "clientId"'); - // }).then(() => { - // return this.exec('DELETE FROM settings WHERE `key` != "clientId"'); - // }); - - // return p; - }).catch((error) => { if (error && error.code != 0 && error.code != 'SQLITE_ERROR') { this.logger().error(error); @@ -454,7 +438,7 @@ class Database { let queries = this.wrapQueries(this.sqlStringToLines(structureSql)); queries.push(this.wrapQuery('INSERT INTO settings (`key`, `value`, `type`) VALUES ("clientId", "' + uuid.create() + '", "' + Database.enumId('settings', 'string') + '")')); - queries.push(this.wrapQuery('INSERT INTO folders (`id`, `title`, `is_default`, `created_time`) VALUES ("' + uuid.create() + '", "' + _('Default list') + '", 1, ' + Math.round((new Date()).getTime() / 1000) + ')')); + queries.push(this.wrapQuery('INSERT INTO folders (`id`, `title`, `is_default`, `created_time`) VALUES ("' + uuid.create() + '", "' + _('Notebook') + '", 1, ' + (new Date()).getTime() + ')')); return this.transactionExecBatch(queries).then(() => { this.logger().info('Database schema created successfully'); diff --git a/lib/mime-utils.js b/lib/mime-utils.js index d5de7b60d..5a9bccdcc 100644 --- a/lib/mime-utils.js +++ b/lib/mime-utils.js @@ -5,7 +5,15 @@ const mime = { toFileExtension(mimeType) { mimeType = mimeType.toLowerCase(); for (let i = 0; i < mimeTypes.length; i++) { - if (mimeType == mimeTypes[i].t) return mimeTypes[i].e[0]; + const t = mimeTypes[i]; + if (mimeType == t.t) { + // Return the first file extension that is 3 characters long + // If none exist return the first one in the list. + for (let j = 0; j < t.e.length; j++) { + if (t.e[j].length == 3) return t.e[j]; + } + return t.e[0]; + } } return null; }, diff --git a/lib/models/folder.js b/lib/models/folder.js index 1257a01bc..ff73f840a 100644 --- a/lib/models/folder.js +++ b/lib/models/folder.js @@ -97,10 +97,28 @@ class Folder extends BaseItem { return folders.concat(notes); } + static async defaultFolder() { + return this.modelSelectOne('SELECT * FROM folders WHERE is_default = 1'); + } + static save(o, options = null) { return Folder.loadByField('title', o.title).then((existingFolder) => { if (existingFolder && existingFolder.id != o.id) throw new Error(_('A folder with title "%s" already exists', o.title)); + if ('is_default' in o) { + if (!o.is_default) { + o = Object.assign({}, o); + delete o.is_default; + Log.warn('is_default property cannot be set to 0 directly. Instead, set the folder that should become the default to 1.'); + } else { + if (!options) options = {}; + if (!options.transactionNextQueries) options.transactionNextQueries = []; + options.transactionNextQueries.push( + { sql: 'UPDATE folders SET is_default = 0 WHERE id != ?', params: [o.id] }, + ); + } + } + return super.save(o, options).then((folder) => { this.dispatch({ type: 'FOLDERS_UPDATE_ONE', diff --git a/lib/models/resource.js b/lib/models/resource.js index c8339d087..6b63712c6 100644 --- a/lib/models/resource.js +++ b/lib/models/resource.js @@ -1,6 +1,7 @@ import { BaseModel } from 'lib/base-model.js'; import { Setting } from 'lib/models/setting.js'; import { mime } from 'lib/mime-utils.js'; +import { filename } from 'lib/path-utils.js'; class Resource extends BaseModel { @@ -18,6 +19,10 @@ class Resource extends BaseModel { return Setting.value('resourceDir') + '/' + resource.id + extension; } + static pathToId(path) { + return filename(path); + } + } export { Resource }; \ No newline at end of file diff --git a/lib/path-utils.js b/lib/path-utils.js index d184d7e41..858245658 100644 --- a/lib/path-utils.js +++ b/lib/path-utils.js @@ -11,6 +11,16 @@ function basename(path) { return s[s.length - 1]; } +function filename(path) { + if (!path) throw new Error('Path is empty'); + let output = dirname(path); + if (output.indexOf('.') < 0) return output; + + output = output.split('.'); + output.pop(); + return output.join('.'); +} + function isHidden(path) { let b = basename(path); if (!b.length) throw new Error('Path empty or not a valid path: ' + path);