self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-23 18:53:36 +02:00
Code Issues Releases Activity

Handle default folder

Browse Source
This commit is contained in:
Laurent Cozic 2017-06-25 00:19:11 +01:00
parent 4ad0601bfa
commit 38be57c1f6
11 changed files with 593 additions and 570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

443
CliClient/app/import-enex-md-gen.js Normal file
View File

@ -0,0 +1,443 @@
const BLOCK_OPEN = "<div>";
const BLOCK_CLOSE = "</div>";
const NEWLINE = "<br/>";
const NEWLINE_MERGED = "<merged/>";
const SPACE = "<space/>";
function processMdArrayNewLines(md) {
while (md.length && md[0] == BLOCK_OPEN) {
md.shift();
}
while (md.length && md[md.length - 1] == BLOCK_CLOSE) {
md.pop();
}
let temp = [];
let last = '';
for (let i = 0; i < md.length; i++) { let v = md[i];
if (isNewLineBlock(last) && isNewLineBlock(v) && last == v) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == BLOCK_CLOSE && v == BLOCK_OPEN) {
temp.pop();
temp.push(NEWLINE_MERGED);
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_CLOSE)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
// NEW!!!
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_OPEN)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
if (md.length > 2) {
if (md[md.length - 2] == NEWLINE_MERGED && md[md.length - 1] == NEWLINE) {
md.pop();
}
}
let output = '';
let previous = '';
let start = true;
for (let i = 0; i < md.length; i++) { let v = md[i];
let add = '';
if (v == BLOCK_CLOSE || v == BLOCK_OPEN || v == NEWLINE || v == NEWLINE_MERGED) {
add = "\n";
} else if (v == SPACE) {
if (previous == SPACE || previous == "\n" || start) {
continue; // skip
} else {
add = " ";
}
} else {
add = v;
}
start = false;
output += add;
previous = add;
}
if (!output.trim().length) return '';
return output;
}
function isWhiteSpace(c) {
return c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\t' || c == ' ';
}
// Like QString::simpified(), except that it preserves non-breaking spaces (which
// Evernote uses for identation, etc.)
function simplifyString(s) {
let output = '';
let previousWhite = false;
for (let i = 0; i < s.length; i++) {
let c = s[i];
let isWhite = isWhiteSpace(c);
if (previousWhite && isWhite) {
// skip
} else {
output += c;
}
previousWhite = isWhite;
}
while (output.length && isWhiteSpace(output[0])) output = output.substr(1);
while (output.length && isWhiteSpace(output[output.length - 1])) output = output.substr(0, output.length - 1);
return output;
}
function collapseWhiteSpaceAndAppend(lines, state, text) {
if (state.inCode) {
text = "\t" + text;
lines.push(text);
} else {
// Remove all \n and \r from the left and right of the text
while (text.length && (text[0] == "\n" || text[0] == "\r")) text = text.substr(1);
while (text.length && (text[text.length - 1] == "\n" || text[text.length - 1] == "\r")) text = text.substr(0, text.length - 1);
// Collapse all white spaces to just one. If there are spaces to the left and right of the string
// also collapse them to just one space.
let spaceLeft = text.length && text[0] == ' ';
let spaceRight = text.length && text[text.length - 1] == ' ';
text = simplifyString(text);
if (!spaceLeft && !spaceRight && text == "") return lines;
if (spaceLeft) lines.push(SPACE);
lines.push(text);
if (spaceRight) lines.push(SPACE);
}
return lines;
}
const imageMimeTypes = ["image/cgm", "image/fits", "image/g3fax", "image/gif", "image/ief", "image/jp2", "image/jpeg", "image/jpm", "image/jpx", "image/naplps", "image/png", "image/prs.btif", "image/prs.pti", "image/t38", "image/tiff", "image/tiff-fx", "image/vnd.adobe.photoshop", "image/vnd.cns.inf2", "image/vnd.djvu", "image/vnd.dwg", "image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst", "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc", "image/vnd.globalgraphics.pgb", "image/vnd.microsoft.icon", "image/vnd.mix", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.sealed.png", "image/vnd.sealedmedia.softseal.gif", "image/vnd.sealedmedia.softseal.jpg", "image/vnd.svf", "image/vnd.wap.wbmp", "image/vnd.xiff"];
function isImageMimeType(m) {
return imageMimeTypes.indexOf(m) >= 0;
}
function addResourceTag(lines, resource, alt = "") {
let tagAlt = alt == "" ? resource.alt : alt;
if (!tagAlt) tagAlt = '';
if (isImageMimeType(resource.mime)) {
lines.push("![");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
} else {
lines.push("[");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
}
return lines;
}
function isBlockTag(n) {
return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
}
function isStrongTag(n) {
return n == "strong" || n == "b";
}
function isEmTag(n) {
return n == "em" || n == "i" || n == "u";
}
function isAnchor(n) {
return n == "a";
}
function isIgnoredEndTag(n) {
return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s';
}
function isListTag(n) {
return n == "ol" || n == "ul";
}
// Elements that don't require any special treatment beside adding a newline character
function isNewLineOnlyEndTag(n) {
return n=="div" || n=="p" || n=="li" || n=="h1" || n=="h2" || n=="h3" || n=="h4" || n=="h5" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
}
function isCodeTag(n) {
return n == "pre" || n == "code";
}
function isNewLineBlock(s) {
return s == BLOCK_OPEN || s == BLOCK_CLOSE;
}
function xmlNodeText(xmlNode) {
if (!xmlNode || !xmlNode.length) return '';
return xmlNode[0];
}
function enexXmlToMdArray(stream, resources) {
resources = resources.slice();
return new Promise((resolve, reject) => {
let output = [];
let state = {
inCode: false,
lists: [],
anchorAttributes: [],
};
let options = {};
let strict = true;
var saxStream = require('sax').createStream(strict, options)
saxStream.on('error', function(e) {
reject(e);
})
saxStream.on('text', function(text) {
output = collapseWhiteSpaceAndAppend(output, state, text);
})
saxStream.on('opentag', function(node) {
let n = node.name.toLowerCase();
if (n == 'en-note') {
// Start of note
} else if (isBlockTag(n)) {
output.push(BLOCK_OPEN);
} else if (isListTag(n)) {
output.push(BLOCK_OPEN);
state.lists.push({ tag: n, counter: 1 });
} else if (n == 'li') {
output.push(BLOCK_OPEN);
if (!state.lists.length) {
reject("Found <li> tag without being inside a list"); // TODO: could be a warning, but nothing to handle warnings at the moment
return;
}
let container = state.lists[state.lists.length - 1];
if (container.tag == "ul") {
output.push("- ");
} else {
output.push(container.counter + '. ');
container.counter++;
}
} else if (isStrongTag(n)) {
output.push("**");
} else if (n == 's') {
// Not supported
} else if (isAnchor(n)) {
state.anchorAttributes.push(node.attributes);
output.push('[');
} else if (isEmTag(n)) {
output.push("*");
} else if (n == "en-todo") {
let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' ';
output.push('- [' + x + '] ');
} else if (n == "hr") {
output.push('------------------------------------------------------------------------------');
} else if (n == "h1") {
output.push(BLOCK_OPEN); output.push("# ");
} else if (n == "h2") {
output.push(BLOCK_OPEN); output.push("## ");
} else if (n == "h3") {
output.push(BLOCK_OPEN); output.push("### ");
} else if (n == "h4") {
output.push(BLOCK_OPEN); output.push("#### ");
} else if (n == "h5") {
output.push(BLOCK_OPEN); output.push("##### ");
} else if (n == "h6") {
output.push(BLOCK_OPEN); output.push("###### ");
} else if (isCodeTag(n)) {
output.push(BLOCK_OPEN);
state.inCode = true;
} else if (n == "br") {
output.push(NEWLINE);
} else if (n == "en-media") {
const hash = node.attributes.hash;
let resource = null;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (r.id == hash) {
resource = r;
resources.splice(i, 1);
break;
}
}
if (!resource) {
// This is a bit of a hack. Notes sometime have resources attached to it, but those <resource> tags don't contain
// an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note
// will contain a corresponding <en-media/> tag, which has the ID in the "hash" attribute. All this information
// has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an
// example of note that shows this problem:
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export2.dtd">
// <en-export export-date="20161221T203133Z" application="Evernote/Windows" version="6.x">
// <note>
// <title>Commande</title>
// <content>
// <![CDATA[
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
// <en-note>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media>
// </en-note>
// ]]>
// </content>
// <created>20160921T203424Z</created>
// <updated>20160921T203438Z</updated>
// <note-attributes>
// <reminder-order>20160902T140445Z</reminder-order>
// <reminder-done-time>20160924T101120Z</reminder-done-time>
// </note-attributes>
// <resource>
// <data encoding="base64">........</data>
// <mime>image/png</mime>
// <width>150</width>
// <height>150</height>
// </resource>
// </note>
// </en-export>
let found = false;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (!r.id) {
r.id = hash;
resources[i] = r;
found = true;
break;
}
}
if (!found) {
console.warn('Hash with no associated resource: ' + hash);
}
} else {
// If the resource does not appear among the note's resources, it
// means it's an attachement. It will be appended along with the
// other remaining resources at the bottom of the markdown text.
if (!!resource.id) {
output = addResourceTag(output, resource, node.attributes.alt);
}
}
} else if (n == "span" || n == "font") {
// Ignore
} else {
console.warn("Unsupported start tag: " + n);
}
})
saxStream.on('closetag', function(n) {
if (n == 'en-note') {
// End of note
} else if (isNewLineOnlyEndTag(n)) {
output.push(BLOCK_CLOSE);
} else if (isIgnoredEndTag(n)) {
// Skip
} else if (isListTag(n)) {
output.push(BLOCK_CLOSE);
state.lists.pop();
} else if (isStrongTag(n)) {
output.push("**");
} else if (isEmTag(n)) {
output.push("*");
} else if (isCodeTag(n)) {
state.inCode = false;
output.push(BLOCK_CLOSE);
} else if (isAnchor(n)) {
let attributes = state.anchorAttributes.pop();
let url = attributes && attributes.href ? attributes.href : '';
output.push('](' + url + ')');
} else if (isListTag(n)) {
output.push(BLOCK_CLOSE);
state.lists.pop();
} else if (n == "en-media") {
// Skip
} else if (isIgnoredEndTag(n)) {
// Skip
} else {
console.warn("Unsupported end tag: " + n);
}
})
saxStream.on('attribute', function(attr) {
})
saxStream.on('end', function() {
resolve({
lines: output,
resources: resources,
});
})
stream.pipe(saxStream);
});
}
async function enexXmlToMd(stream, resources) {
let result = await enexXmlToMdArray(stream, resources);
let mdLines = result.lines;
let firstAttachment = true;
for (let i = 0; i < result.resources.length; i++) {
let r = result.resources[i];
if (firstAttachment) mdLines.push(NEWLINE);
mdLines.push(NEWLINE);
mdLines = addResourceTag(mdLines, r, r.filename);
firstAttachment = false;
}
return processMdArrayNewLines(mdLines);
}
export { enexXmlToMd, processMdArrayNewLines, NEWLINE, addResourceTag };

544
CliClient/app/import-enex.js
View File

@ -8,439 +8,13 @@ import { BaseModel } from 'lib/base-model.js';
import { Note } from 'lib/models/note.js'; import { Note } from 'lib/models/note.js';
import { Resource } from 'lib/models/resource.js'; import { Resource } from 'lib/models/resource.js';
import { Folder } from 'lib/models/folder.js'; import { Folder } from 'lib/models/folder.js';
import { enexXmlToMd } from './import-enex-md-gen.js';
import jsSHA from "jssha"; import jsSHA from "jssha";
const Promise = require('promise'); const Promise = require('promise');
const fs = require('fs-extra'); const fs = require('fs-extra');
const stringToStream = require('string-to-stream') const stringToStream = require('string-to-stream')
const BLOCK_OPEN = "<div>";
const BLOCK_CLOSE = "</div>";
const NEWLINE = "<br/>";
const NEWLINE_MERGED = "<merged/>";
const SPACE = "<space/>";
function processMdArrayNewLines(md) {
while (md.length && md[0] == BLOCK_OPEN) {
md.shift();
}
while (md.length && md[md.length - 1] == BLOCK_CLOSE) {
md.pop();
}
let temp = [];
let last = '';
for (let i = 0; i < md.length; i++) { let v = md[i];
if (isNewLineBlock(last) && isNewLineBlock(v) && last == v) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == BLOCK_CLOSE && v == BLOCK_OPEN) {
temp.pop();
temp.push(NEWLINE_MERGED);
} else {
temp.push(v);
}
last = v;
}
md = temp;
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_CLOSE)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
// NEW!!!
temp = [];
last = "";
for (let i = 0; i < md.length; i++) { let v = md[i];
if (last == NEWLINE && (v == NEWLINE_MERGED || v == BLOCK_OPEN)) {
// Skip it
} else {
temp.push(v);
}
last = v;
}
md = temp;
if (md.length > 2) {
if (md[md.length - 2] == NEWLINE_MERGED && md[md.length - 1] == NEWLINE) {
md.pop();
}
}
let output = '';
let previous = '';
let start = true;
for (let i = 0; i < md.length; i++) { let v = md[i];
let add = '';
if (v == BLOCK_CLOSE || v == BLOCK_OPEN || v == NEWLINE || v == NEWLINE_MERGED) {
add = "\n";
} else if (v == SPACE) {
if (previous == SPACE || previous == "\n" || start) {
continue; // skip
} else {
add = " ";
}
} else {
add = v;
}
start = false;
output += add;
previous = add;
}
if (!output.trim().length) return '';
return output;
}
function isWhiteSpace(c) {
return c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\t' || c == ' ';
}
// Like QString::simpified(), except that it preserves non-breaking spaces (which
// Evernote uses for identation, etc.)
function simplifyString(s) {
let output = '';
let previousWhite = false;
for (let i = 0; i < s.length; i++) {
let c = s[i];
let isWhite = isWhiteSpace(c);
if (previousWhite && isWhite) {
// skip
} else {
output += c;
}
previousWhite = isWhite;
}
while (output.length && isWhiteSpace(output[0])) output = output.substr(1);
while (output.length && isWhiteSpace(output[output.length - 1])) output = output.substr(0, output.length - 1);
return output;
}
function collapseWhiteSpaceAndAppend(lines, state, text) {
if (state.inCode) {
text = "\t" + text;
lines.push(text);
} else {
// Remove all \n and \r from the left and right of the text
while (text.length && (text[0] == "\n" || text[0] == "\r")) text = text.substr(1);
while (text.length && (text[text.length - 1] == "\n" || text[text.length - 1] == "\r")) text = text.substr(0, text.length - 1);
// Collapse all white spaces to just one. If there are spaces to the left and right of the string
// also collapse them to just one space.
let spaceLeft = text.length && text[0] == ' ';
let spaceRight = text.length && text[text.length - 1] == ' ';
text = simplifyString(text);
if (!spaceLeft && !spaceRight && text == "") return lines;
if (spaceLeft) lines.push(SPACE);
lines.push(text);
if (spaceRight) lines.push(SPACE);
}
return lines;
}
const imageMimeTypes = ["image/cgm", "image/fits", "image/g3fax", "image/gif", "image/ief", "image/jp2", "image/jpeg", "image/jpm", "image/jpx", "image/naplps", "image/png", "image/prs.btif", "image/prs.pti", "image/t38", "image/tiff", "image/tiff-fx", "image/vnd.adobe.photoshop", "image/vnd.cns.inf2", "image/vnd.djvu", "image/vnd.dwg", "image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst", "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc", "image/vnd.globalgraphics.pgb", "image/vnd.microsoft.icon", "image/vnd.mix", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.sealed.png", "image/vnd.sealedmedia.softseal.gif", "image/vnd.sealedmedia.softseal.jpg", "image/vnd.svf", "image/vnd.wap.wbmp", "image/vnd.xiff"];
function isImageMimeType(m) {
return imageMimeTypes.indexOf(m) >= 0;
}
function addResourceTag(lines, resource, alt = "") {
let tagAlt = alt == "" ? resource.alt : alt;
if (!tagAlt) tagAlt = '';
if (isImageMimeType(resource.mime)) {
lines.push("![");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
} else {
lines.push("[");
lines.push(tagAlt);
lines.push("](:/" + resource.id + ")");
}
return lines;
}
function enexXmlToMd(stream, resources) {
resources = resources.slice();
return new Promise((resolve, reject) => {
let output = [];
let state = {
inCode: false,
lists: [],
anchorAttributes: [],
};
let options = {};
let strict = true;
var saxStream = require('sax').createStream(strict, options)
saxStream.on('error', function(e) {
reject(e);
})
saxStream.on('text', function(text) {
output = collapseWhiteSpaceAndAppend(output, state, text);
})
saxStream.on('opentag', function(node) {
let n = node.name.toLowerCase();
if (n == 'en-note') {
// Start of note
} else if (isBlockTag(n)) {
output.push(BLOCK_OPEN);
} else if (isListTag(n)) {
output.push(BLOCK_OPEN);
state.lists.push({ tag: n, counter: 1 });
} else if (n == 'li') {
output.push(BLOCK_OPEN);
if (!state.lists.length) {
reject("Found <li> tag without being inside a list"); // TODO: could be a warning, but nothing to handle warnings at the moment
return;
}
let container = state.lists[state.lists.length - 1];
if (container.tag == "ul") {
output.push("- ");
} else {
output.push(container.counter + '. ');
container.counter++;
}
} else if (isStrongTag(n)) {
output.push("**");
} else if (n == 's') {
// Not supported
} else if (isAnchor(n)) {
state.anchorAttributes.push(node.attributes);
output.push('[');
} else if (isEmTag(n)) {
output.push("*");
} else if (n == "en-todo") {
let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' ';
output.push('- [' + x + '] ');
} else if (n == "hr") {
output.push('------------------------------------------------------------------------------');
} else if (n == "h1") {
output.push(BLOCK_OPEN); output.push("# ");
} else if (n == "h2") {
output.push(BLOCK_OPEN); output.push("## ");
} else if (n == "h3") {
output.push(BLOCK_OPEN); output.push("### ");
} else if (n == "h4") {
output.push(BLOCK_OPEN); output.push("#### ");
} else if (n == "h5") {
output.push(BLOCK_OPEN); output.push("##### ");
} else if (n == "h6") {
output.push(BLOCK_OPEN); output.push("###### ");
} else if (isCodeTag(n)) {
output.push(BLOCK_OPEN);
state.inCode = true;
} else if (n == "br") {
output.push(NEWLINE);
} else if (n == "en-media") {
const hash = node.attributes.hash;
let resource = null;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (r.id == hash) {
resource = r;
resources.splice(i, 1);
break;
}
}
if (!resource) {
// This is a bit of a hack. Notes sometime have resources attached to it, but those <resource> tags don't contain
// an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note
// will contain a corresponding <en-media/> tag, which has the ID in the "hash" attribute. All this information
// has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an
// example of note that shows this problem:
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export2.dtd">
// <en-export export-date="20161221T203133Z" application="Evernote/Windows" version="6.x">
// <note>
// <title>Commande</title>
// <content>
// <![CDATA[
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
// <en-note>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media>
// </en-note>
// ]]>
// </content>
// <created>20160921T203424Z</created>
// <updated>20160921T203438Z</updated>
// <note-attributes>
// <reminder-order>20160902T140445Z</reminder-order>
// <reminder-done-time>20160924T101120Z</reminder-done-time>
// </note-attributes>
// <resource>
// <data encoding="base64">........</data>
// <mime>image/png</mime>
// <width>150</width>
// <height>150</height>
// </resource>
// </note>
// </en-export>
let found = false;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (!r.id) {
r.id = hash;
resources[i] = r;
found = true;
break;
}
}
if (!found) {
console.warn('Hash with no associated resource: ' + hash);
}
} else {
// If the resource does not appear among the note's resources, it
// means it's an attachement. It will be appended along with the
// other remaining resources at the bottom of the markdown text.
if (!!resource.id) {
output = addResourceTag(output, resource, node.attributes.alt);
}
}
} else if (n == "span" || n == "font") {
// Ignore
} else {
console.warn("Unsupported start tag: " + n);
}
})
saxStream.on('closetag', function(n) {
if (n == 'en-note') {
// End of note
} else if (isNewLineOnlyEndTag(n)) {
output.push(BLOCK_CLOSE);
} else if (isIgnoredEndTag(n)) {
// Skip
} else if (isListTag(n)) {
output.push(BLOCK_CLOSE);
state.lists.pop();
} else if (isStrongTag(n)) {
output.push("**");
} else if (isEmTag(n)) {
output.push("*");
} else if (isCodeTag(n)) {
state.inCode = false;
output.push(BLOCK_CLOSE);
} else if (isAnchor(n)) {
let attributes = state.anchorAttributes.pop();
let url = attributes && attributes.href ? attributes.href : '';
output.push('](' + url + ')');
} else if (isListTag(n)) {
output.push(BLOCK_CLOSE);
state.lists.pop();
} else if (n == "en-media") {
// Skip
} else if (isIgnoredEndTag(n)) {
// Skip
} else {
console.warn("Unsupported end tag: " + n);
}
})
saxStream.on('attribute', function(attr) {
})
saxStream.on('end', function() {
resolve({
lines: output,
resources: resources,
});
})
stream.pipe(saxStream);
});
}
function isBlockTag(n) {
return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
}
function isStrongTag(n) {
return n == "strong" || n == "b";
}
function isEmTag(n) {
return n == "em" || n == "i" || n == "u";
}
function isAnchor(n) {
return n == "a";
}
function isIgnoredEndTag(n) {
return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s';
}
function isListTag(n) {
return n == "ol" || n == "ul";
}
// Elements that don't require any special treatment beside adding a newline character
function isNewLineOnlyEndTag(n) {
return n=="div" || n=="p" || n=="li" || n=="h1" || n=="h2" || n=="h3" || n=="h4" || n=="h5" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
}
function isCodeTag(n) {
return n == "pre" || n == "code";
}
function isNewLineBlock(s) {
return s == BLOCK_OPEN || s == BLOCK_CLOSE;
}
function xmlNodeText(xmlNode) {
if (!xmlNode || !xmlNode.length) return '';
return xmlNode[0];
}
let existingTimestamps = []; let existingTimestamps = [];
function uniqueCreatedTimestamp(timestamp) { function uniqueCreatedTimestamp(timestamp) {
@ -460,68 +34,22 @@ function uniqueCreatedTimestamp(timestamp) {
return timestamp; return timestamp;
} }
function dateToTimestamp(s) { function dateToTimestamp(s, zeroIfInvalid = false) {
let m = moment(s, 'YYYYMMDDTHHmmssZ'); let m = moment(s, 'YYYYMMDDTHHmmssZ');
if (!m.isValid()) { if (!m.isValid()) {
if (zeroIfInvalid) return 0;
throw new Error('Invalid date: ' + s); throw new Error('Invalid date: ' + s);
} }
return m.toDate().getTime(); return m.toDate().getTime();
} }
function evernoteXmlToMdArray(xml) {
return parseXml(xml).then((xml) => {
console.info(xml);
});
}
function extractRecognitionObjId(recognitionXml) { function extractRecognitionObjId(recognitionXml) {
const r = recognitionXml.match(/objID="(.*?)"/); const r = recognitionXml.match(/objID="(.*?)"/);
return r && r.length >= 2 ? r[1] : null; return r && r.length >= 2 ? r[1] : null;
} }
function filePutContents(filePath, content) { function filePutContents(filePath, content) {
return new Promise((resolve, reject) => { return fs.writeFile(filePath, content);
fs.writeFile(filePath, content, function(error) {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
function setModifiedTime(filePath, time) {
return new Promise((resolve, reject) => {
fs.utimes(filePath, time, time, (error) => {
if (error) {
reject(error);
return;
}
resolve();
})
});
}
function createDirectory(path) {
return new Promise((resolve, reject) => {
fs.exists(path, (exists) => {
if (exists) {
resolve();
return;
}
const mkdirp = require('mkdirp');
mkdirp(path, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
});
} }
function removeUndefinedProperties(note) { function removeUndefinedProperties(note) {
@ -580,25 +108,21 @@ async function saveNoteToStorage(note) {
diff.type_ = existingNote.type_; diff.type_ = existingNote.type_;
return Note.save(diff, { autoTimestamp: false }); return Note.save(diff, { autoTimestamp: false });
} else { } else {
// id: noteResource.id,
// data: decodedData,
// mime: noteResource.mime,
// title: noteResource.filename,
// filename: noteResource.filename,
// CREATE TABLE resources (
// id TEXT PRIMARY KEY,
// title TEXT,
// mime TEXT,
// filename TEXT,
// created_time INT,
// updated_time INT
for (let i = 0; i < note.resources.length; i++) { for (let i = 0; i < note.resources.length; i++) {
let resource = note.resources[i]; let resource = note.resources[i];
let toSave = Object.assign({}, resource); let toSave = Object.assign({}, resource);
delete toSave.data; delete toSave.data;
// The same resource sometimes appear twice in the same enex (exact same ID and file).
// In that case, just skip it - it means two different notes might be linked to the
// same resource.
let existingResource = await Resource.load(toSave.id);
if (existingResource) {
// console.warn('Trying to save: ' + JSON.stringify(toSave));
// console.warn('But duplicate: ' + JSON.stringify(existingResource));
continue;
}
await Resource.save(toSave, { isNew: true }); await Resource.save(toSave, { isNew: true });
await filePutContents(Resource.fullPath(toSave), resource.data); await filePutContents(Resource.fullPath(toSave), resource.data);
} }
@ -646,27 +170,14 @@ function importEnex(parentFolderId, filePath) {
let note = notes.shift(); let note = notes.shift();
const contentStream = stringToStream(note.bodyXml); const contentStream = stringToStream(note.bodyXml);
chain.push(() => { chain.push(() => {
return enexXmlToMd(contentStream, note.resources).then((result) => { return enexXmlToMd(contentStream, note.resources).then((body) => {
delete note.bodyXml; delete note.bodyXml;
let mdLines = result.lines;
let firstAttachment = true;
for (let i = 0; i < result.resources.length; i++) {
let r = result.resources[i];
if (firstAttachment) mdLines.push(NEWLINE);
mdLines.push(NEWLINE);
mdLines = addResourceTag(mdLines, r, r.filename);
firstAttachment = false;
}
note.parent_id = parentFolderId;
note.body = processMdArrayNewLines(result.lines);
note.id = uuid.create(); note.id = uuid.create();
note.parent_id = parentFolderId;
note.body = body;
return saveNoteToStorage(note); return saveNoteToStorage(note);
// SAVE NOTE HERE
// saveNoteToDisk(parentFolder, note);
}); });
}); });
} }
@ -700,6 +211,8 @@ function importEnex(parentFolderId, filePath) {
note.updated_time = dateToTimestamp(text); note.updated_time = dateToTimestamp(text);
} else if (n == 'tag') { } else if (n == 'tag') {
note.tags.push(text); note.tags.push(text);
} else {
console.warn('Unsupported note tag: ' + n);
} }
} }
}) })
@ -763,6 +276,21 @@ function importEnex(parentFolderId, filePath) {
note.longitude = noteAttributes.longitude; note.longitude = noteAttributes.longitude;
note.altitude = noteAttributes.altitude; note.altitude = noteAttributes.altitude;
note.author = noteAttributes.author; note.author = noteAttributes.author;
note.is_todo = !!noteAttributes['reminder-order'];
note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true);
note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true);
note.order = dateToTimestamp(noteAttributes['reminder-order'], true);
note.source = !!noteAttributes.source ? 'evernote.' + noteAttributes.source : 'evernote';
note.source_application = 'joplin.cli';
if (noteAttributes['reminder-time']) {
console.info('======================================================');
console.info(noteAttributes);
console.info('------------------------------------------------------');
console.info(note);
console.info('======================================================');
}
noteAttributes = null; noteAttributes = null;
} else if (n == 'resource') { } else if (n == 'resource') {
let decodedData = null; let decodedData = null;
@ -781,8 +309,6 @@ function importEnex(parentFolderId, filePath) {
filename: noteResource.filename, filename: noteResource.filename,
}; };
// r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE
note.resources.push(r); note.resources.push(r);
noteResource = null; noteResource = null;
} }

4
CliClient/app/main.js
View File

@ -62,7 +62,7 @@ async function main() {
console.info('DELETING ALL DATA'); console.info('DELETING ALL DATA');
await db.exec('DELETE FROM notes'); await db.exec('DELETE FROM notes');
await db.exec('DELETE FROM changes'); await db.exec('DELETE FROM changes');
await db.exec('DELETE FROM folders'); await db.exec('DELETE FROM folders WHERE is_default != 1');
await db.exec('DELETE FROM resources'); await db.exec('DELETE FROM resources');
await db.exec('DELETE FROM deleted_items'); await db.exec('DELETE FROM deleted_items');
await db.exec('DELETE FROM tags'); await db.exec('DELETE FROM tags');
@ -72,7 +72,7 @@ async function main() {
//let folder = await Folder.loadByField('title', 'test'); //let folder = await Folder.loadByField('title', 'test');
await importEnex(folder.id, '/mnt/c/Users/Laurent/Desktop/Laurent.enex'); //'/mnt/c/Users/Laurent/Desktop/Laurent.enex'); await importEnex(folder.id, '/mnt/c/Users/Laurent/Desktop/afaire.enex');
return; return;

3
CliClient/run_test.sh
View File

@ -6,4 +6,5 @@ mkdir -p "$CLIENT_DIR/tests-build/data"
ln -s "$CLIENT_DIR/build/lib" "$CLIENT_DIR/tests-build" ln -s "$CLIENT_DIR/build/lib" "$CLIENT_DIR/tests-build"
#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js #npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js
npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js #npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js
npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/models/folder.js

52
CliClient/tests/models/folder.js Normal file
View File

@ -0,0 +1,52 @@
import { time } from 'lib/time-utils.js';
import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient } from 'test-utils.js';
import { createFoldersAndNotes } from 'test-data.js';
import { Folder } from 'lib/models/folder.js';
import { Note } from 'lib/models/note.js';
import { Setting } from 'lib/models/setting.js';
import { BaseItem } from 'lib/models/base-item.js';
import { BaseModel } from 'lib/base-model.js';
process.on('unhandledRejection', (reason, p) => {
console.error('Unhandled promise rejection at: Promise', p, 'reason:', reason);
});
async function thereIsOnlyOneDefaultFolder() {
let count = 0;
let folders = await Folder.all();
for (let i = 0; i < folders.length; i++) {
if (!!folders[i].is_default) count++;
}
return count === 1;
}
describe('Folder', function() {
beforeEach( async (done) => {
await setupDatabase(1);
switchClient(1);
done();
});
it('should have one default folder only', async (done) => {
let f1 = await Folder.save({ title: 'folder1', is_default: 1 });
let f2 = await Folder.save({ title: 'folder2' });
let f3 = await Folder.save({ title: 'folder3' });
await Folder.save({ id: f2.id, is_default: 1 });
f2 = await Folder.load(f2.id);
expect(f2.is_default).toBe(1);
let r = await thereIsOnlyOneDefaultFolder();
expect(r).toBe(true);
await Folder.save({ id: f2.id, is_default: 0 });
f2 = await Folder.load(f2.id);
expect(f2.is_default).toBe(1);
done();
});
});

44
lib/base-model.js
View File

@ -106,6 +106,7 @@ class BaseModel {
if (!('trackDeleted' in options)) options.trackDeleted = null; if (!('trackDeleted' in options)) options.trackDeleted = null;
if (!('isNew' in options)) options.isNew = 'auto'; if (!('isNew' in options)) options.isNew = 'auto';
if (!('autoTimestamp' in options)) options.autoTimestamp = true; if (!('autoTimestamp' in options)) options.autoTimestamp = true;
if (!('transactionNextQueries' in options)) options.transactionNextQueries = [];
return options; return options;
} }
@ -173,7 +174,7 @@ class BaseModel {
o = temp; o = temp;
let query = {}; let query = {};
let itemId = o.id; let modelId = o.id;
if (options.autoTimestamp && this.hasField('updated_time')) { if (options.autoTimestamp && this.hasField('updated_time')) {
o.updated_time = time.unixMs(); o.updated_time = time.unixMs();
@ -181,8 +182,8 @@ class BaseModel {
if (options.isNew) { if (options.isNew) {
if (this.useUuid() && !o.id) { if (this.useUuid() && !o.id) {
itemId = uuid.create(); modelId = uuid.create();
o.id = itemId; o.id = modelId;
} }
if (!o.created_time && this.hasField('created_time')) { if (!o.created_time && this.hasField('created_time')) {
@ -197,7 +198,7 @@ class BaseModel {
query = Database.updateQuery(this.tableName(), temp, where); query = Database.updateQuery(this.tableName(), temp, where);
} }
query.id = itemId; query.id = modelId;
// Log.info('Saving', JSON.stringify(o)); // Log.info('Saving', JSON.stringify(o));
@ -212,42 +213,17 @@ class BaseModel {
let queries = []; let queries = [];
let saveQuery = this.saveQuery(o, options); let saveQuery = this.saveQuery(o, options);
let itemId = saveQuery.id; let modelId = saveQuery.id;
queries.push(saveQuery); queries.push(saveQuery);
// TODO: DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED for (let i = 0; i < options.transactionNextQueries.length; i++) {
// if (options.trackChanges && this.trackChanges()) { queries.push(options.transactionNextQueries[i]);
// // Cannot import this class the normal way due to cyclical dependencies between Change and BaseModel }
// // which are not handled by React Native.
// const { Change } = require('src/models/change.js');
// if (isNew) {
// let change = Change.newChange();
// change.type = Change.TYPE_CREATE;
// change.item_id = itemId;
// change.item_type = this.itemType();
// queries.push(Change.saveQuery(change));
// } else {
// for (let n in o) {
// if (!o.hasOwnProperty(n)) continue;
// if (n == 'id') continue;
// let change = Change.newChange();
// change.type = Change.TYPE_UPDATE;
// change.item_id = itemId;
// change.item_type = this.itemType();
// change.item_field = n;
// queries.push(Change.saveQuery(change));
// }
// }
// }
return this.db().transactionExecBatch(queries).then(() => { return this.db().transactionExecBatch(queries).then(() => {
o = Object.assign({}, o); o = Object.assign({}, o);
o.id = itemId; o.id = modelId;
o = this.addModelMd(o); o = this.addModelMd(o);
return this.filter(o); return this.filter(o);
}).catch((error) => { }).catch((error) => {

30
lib/database.js
View File

@ -26,12 +26,12 @@ CREATE TABLE notes (
latitude NUMERIC NOT NULL DEFAULT 0, latitude NUMERIC NOT NULL DEFAULT 0,
longitude NUMERIC NOT NULL DEFAULT 0, longitude NUMERIC NOT NULL DEFAULT 0,
altitude NUMERIC NOT NULL DEFAULT 0, altitude NUMERIC NOT NULL DEFAULT 0,
source TEXT NOT NULL DEFAULT "",
author TEXT NOT NULL DEFAULT "", author TEXT NOT NULL DEFAULT "",
source_url TEXT NOT NULL DEFAULT "", source_url TEXT NOT NULL DEFAULT "",
is_todo INT NOT NULL DEFAULT 0, is_todo INT NOT NULL DEFAULT 0,
todo_due INT NOT NULL DEFAULT 0, todo_due INT NOT NULL DEFAULT 0,
todo_completed INT NOT NULL DEFAULT 0, todo_completed INT NOT NULL DEFAULT 0,
source TEXT NOT NULL DEFAULT "",
source_application TEXT NOT NULL DEFAULT "", source_application TEXT NOT NULL DEFAULT "",
application_data TEXT NOT NULL DEFAULT "", application_data TEXT NOT NULL DEFAULT "",
\`order\` INT NOT NULL DEFAULT 0 \`order\` INT NOT NULL DEFAULT 0
@ -163,7 +163,8 @@ class Database {
this.logger().info('Database was open successfully'); this.logger().info('Database was open successfully');
return this.initialize(); return this.initialize();
}).catch((error) => { }).catch((error) => {
this.logger().error('Cannot open database: ', error); this.logger().error('Cannot open database:');
this.logger().error(error);
}); });
} }
@ -192,7 +193,8 @@ class Database {
if (queries.length <= 0) return Promise.resolve(); if (queries.length <= 0) return Promise.resolve();
if (queries.length == 1) { if (queries.length == 1) {
return this.exec(queries[0].sql, queries[0].params); let q = this.wrapQuery(queries[0]);
return this.exec(q.sql, q.params);
} }
// There can be only one transaction running at a time so queue // There can be only one transaction running at a time so queue
@ -282,6 +284,7 @@ class Database {
} }
logQuery(sql, params = null) { logQuery(sql, params = null) {
console.info(sql, params);
if (!this.debugMode()) return; if (!this.debugMode()) return;
if (params !== null) { if (params !== null) {
@ -420,25 +423,6 @@ class Database {
}); });
} }
}); });
// }).then(() => {
// let p = this.exec('DELETE FROM notes').then(() => {
// return this.exec('DELETE FROM folders');
// }).then(() => {
// return this.exec('DELETE FROM changes');
// }).then(() => {
// return this.exec('DELETE FROM settings WHERE `key` = "sync.lastRevId"');
// });
// return p.then(() => {
// return this.exec('UPDATE settings SET `value` = "' + uuid.create() + '" WHERE `key` = "clientId"');
// }).then(() => {
// return this.exec('DELETE FROM settings WHERE `key` != "clientId"');
// });
// return p;
}).catch((error) => { }).catch((error) => {
if (error && error.code != 0 && error.code != 'SQLITE_ERROR') { if (error && error.code != 0 && error.code != 'SQLITE_ERROR') {
this.logger().error(error); this.logger().error(error);
@ -454,7 +438,7 @@ class Database {
let queries = this.wrapQueries(this.sqlStringToLines(structureSql)); let queries = this.wrapQueries(this.sqlStringToLines(structureSql));
queries.push(this.wrapQuery('INSERT INTO settings (`key`, `value`, `type`) VALUES ("clientId", "' + uuid.create() + '", "' + Database.enumId('settings', 'string') + '")')); queries.push(this.wrapQuery('INSERT INTO settings (`key`, `value`, `type`) VALUES ("clientId", "' + uuid.create() + '", "' + Database.enumId('settings', 'string') + '")'));
queries.push(this.wrapQuery('INSERT INTO folders (`id`, `title`, `is_default`, `created_time`) VALUES ("' + uuid.create() + '", "' + _('Default list') + '", 1, ' + Math.round((new Date()).getTime() / 1000) + ')')); queries.push(this.wrapQuery('INSERT INTO folders (`id`, `title`, `is_default`, `created_time`) VALUES ("' + uuid.create() + '", "' + _('Notebook') + '", 1, ' + (new Date()).getTime() + ')'));
return this.transactionExecBatch(queries).then(() => { return this.transactionExecBatch(queries).then(() => {
this.logger().info('Database schema created successfully'); this.logger().info('Database schema created successfully');

10
lib/mime-utils.js
View File

@ -5,7 +5,15 @@ const mime = {
toFileExtension(mimeType) { toFileExtension(mimeType) {
mimeType = mimeType.toLowerCase(); mimeType = mimeType.toLowerCase();
for (let i = 0; i < mimeTypes.length; i++) { for (let i = 0; i < mimeTypes.length; i++) {
if (mimeType == mimeTypes[i].t) return mimeTypes[i].e[0]; const t = mimeTypes[i];
if (mimeType == t.t) {
// Return the first file extension that is 3 characters long
// If none exist return the first one in the list.
for (let j = 0; j < t.e.length; j++) {
if (t.e[j].length == 3) return t.e[j];
}
return t.e[0];
}
} }
return null; return null;
}, },

18
lib/models/folder.js
View File

@ -97,10 +97,28 @@ class Folder extends BaseItem {
return folders.concat(notes); return folders.concat(notes);
} }
static async defaultFolder() {
return this.modelSelectOne('SELECT * FROM folders WHERE is_default = 1');
}
static save(o, options = null) { static save(o, options = null) {
return Folder.loadByField('title', o.title).then((existingFolder) => { return Folder.loadByField('title', o.title).then((existingFolder) => {
if (existingFolder && existingFolder.id != o.id) throw new Error(_('A folder with title "%s" already exists', o.title)); if (existingFolder && existingFolder.id != o.id) throw new Error(_('A folder with title "%s" already exists', o.title));
if ('is_default' in o) {
if (!o.is_default) {
o = Object.assign({}, o);
delete o.is_default;
Log.warn('is_default property cannot be set to 0 directly. Instead, set the folder that should become the default to 1.');
} else {
if (!options) options = {};
if (!options.transactionNextQueries) options.transactionNextQueries = [];
options.transactionNextQueries.push(
{ sql: 'UPDATE folders SET is_default = 0 WHERE id != ?', params: [o.id] },
);
}
}
return super.save(o, options).then((folder) => { return super.save(o, options).then((folder) => {
this.dispatch({ this.dispatch({
type: 'FOLDERS_UPDATE_ONE', type: 'FOLDERS_UPDATE_ONE',

5
lib/models/resource.js
View File

@ -1,6 +1,7 @@
import { BaseModel } from 'lib/base-model.js'; import { BaseModel } from 'lib/base-model.js';
import { Setting } from 'lib/models/setting.js'; import { Setting } from 'lib/models/setting.js';
import { mime } from 'lib/mime-utils.js'; import { mime } from 'lib/mime-utils.js';
import { filename } from 'lib/path-utils.js';
class Resource extends BaseModel { class Resource extends BaseModel {
@ -18,6 +19,10 @@ class Resource extends BaseModel {
return Setting.value('resourceDir') + '/' + resource.id + extension; return Setting.value('resourceDir') + '/' + resource.id + extension;
} }
static pathToId(path) {
return filename(path);
}
} }
export { Resource }; export { Resource };

10
lib/path-utils.js
View File

@ -11,6 +11,16 @@ function basename(path) {
return s[s.length - 1]; return s[s.length - 1];
} }
function filename(path) {
if (!path) throw new Error('Path is empty');
let output = dirname(path);
if (output.indexOf('.') < 0) return output;
output = output.split('.');
output.pop();
return output.join('.');
}
function isHidden(path) { function isHidden(path) {
let b = basename(path); let b = basename(path);
if (!b.length) throw new Error('Path empty or not a valid path: ' + path); if (!b.length) throw new Error('Path empty or not a valid path: ' + path);