self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-09-16 08:56:40 +02:00
Code Issues Releases Activity

Various changezs

Browse Source
This commit is contained in:
Laurent Cozic
2017-06-08 23:24:40 +01:00
parent d822ef8e38
commit 5b9d50947c
12 changed files with 458 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

509
CliClient/app/import-enex.js
View File

@@ -2,11 +2,15 @@ require('app-module-path').addPath(__dirname);
import { uuid } from 'src/uuid.js';
import moment from 'moment';
import { promiseChain } from 'src/promise-chain.js';
import { WebApi } from 'src/web-api.js'
import jsSHA from "jssha";
let webApi = new WebApi('http://joplin.local');
const Promise = require('promise');
const fs = require('fs');
const xml2js = require("xml2js");
const stringToStream = require('string-to-stream')
const BLOCK_OPEN = "<div>";
const BLOCK_CLOSE = "</div>";
@@ -143,7 +147,6 @@ function simplifyString(s) {
function collapseWhiteSpaceAndAppend(lines, state, text) {
if (state.inCode) {
text = "\t" + text;
if (text === undefined) console.info('AAAAAAAAAA');
lines.push(text);
} else {
// Remove all \n and \r from the left and right of the text
@@ -159,7 +162,6 @@ function collapseWhiteSpaceAndAppend(lines, state, text) {
if (!spaceLeft && !spaceRight && text == "") return lines;
if (spaceLeft) lines.push(SPACE);
if (text === undefined) console.info('BBBBBBB');
lines.push(text);
if (spaceRight) lines.push(SPACE);
}
@@ -175,6 +177,7 @@ function isImageMimeType(m) {
function addResourceTag(lines, resource, alt = "") {
let tagAlt = alt == "" ? resource.alt : alt;
if (!tagAlt) tagAlt = '';
if (isImageMimeType(resource.mime)) {
lines.push("![");
lines.push(tagAlt);
@@ -189,9 +192,12 @@ function addResourceTag(lines, resource, alt = "") {
}
function enexXmlToMd(stream) {
function enexXmlToMd(stream, resources) {
resources = resources.slice();
return new Promise((resolve, reject) => {
let output = [];
let state = {
inCode: false,
lists: [],
@@ -235,14 +241,18 @@ function enexXmlToMd(stream) {
}
} else if (isStrongTag(n)) {
output.push("**");
} else if (n == 's') {
// Not supported
} else if (isAnchor(n)) {
state.anchorAttributes.push(node.attributes);
output.push('[');
} else if (isEmTag(n)) {
output.push("*");
} else if (n == "en-todo") {
let x = node.attributes && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' ';
let x = node.attributes && node.attributes.checked && node.attributes.checked.toLowerCase() == 'true' ? 'X' : ' ';
output.push('- [' + x + '] ');
} else if (n == "hr") {
output.push('------------------------------------------------------------------------------');
} else if (n == "h1") {
output.push(BLOCK_OPEN); output.push("# ");
} else if (n == "h2") {
@@ -261,29 +271,80 @@ function enexXmlToMd(stream) {
} else if (n == "br") {
output.push(NEWLINE);
} else if (n == "en-media") {
console.warn('TODO: en-media');
// attrs = attributesLIFO.back();
// QString hash = attrs["hash"];
// Resource resource;
// for (int i = 0; i < state.resources.size(); i++) {
// Resource r = state.resources[i];
// if (r.id == hash) {
// resource = r;
// state.resources.erase(state.resources.begin() + i);
// break;
// }
// }
const hash = node.attributes.hash;
// // If the resource does not appear among the note's resources, it
// // means it's an attachement. It will be appended along with the
// // other remaining resources at the bottom of the markdown text.
// if (resource.id != "") {
// addResourceTag(lines, resource, attrs["alt"]);
// }
let resource = null;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (r.id == hash) {
resource = r;
resources.splice(i, 1);
break;
}
}
if (!resource) {
// This is a bit of a hack. Notes sometime have resources attached to it, but those <resource> tags don't contain
// an "objID" tag, making it impossible to reference the resource. However, in this case the content of the note
// will contain a corresponding <en-media/> tag, which has the ID in the "hash" attribute. All this information
// has been collected above so we now set the resource ID to the hash attribute of the en-media tags. Here's an
// example of note that shows this problem:
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export2.dtd">
// <en-export export-date="20161221T203133Z" application="Evernote/Windows" version="6.x">
// <note>
// <title>Commande</title>
// <content>
// <![CDATA[
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
// <en-note>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media>
// </en-note>
// ]]>
// </content>
// <created>20160921T203424Z</created>
// <updated>20160921T203438Z</updated>
// <note-attributes>
// <reminder-order>20160902T140445Z</reminder-order>
// <reminder-done-time>20160924T101120Z</reminder-done-time>
// </note-attributes>
// <resource>
// <data encoding="base64">........</data>
// <mime>image/png</mime>
// <width>150</width>
// <height>150</height>
// </resource>
// </note>
// </en-export>
let found = false;
for (let i = 0; i < resources.length; i++) {
let r = resources[i];
if (!r.id) {
r.id = hash;
resources[i] = r;
found = true;
break;
}
}
if (!found) {
console.warn('Hash with no associated resource: ' + hash);
}
} else {
// If the resource does not appear among the note's resources, it
// means it's an attachement. It will be appended along with the
// other remaining resources at the bottom of the markdown text.
if (!!resource.id) {
output = addResourceTag(output, resource, node.attributes.alt);
}
}
} else if (n == "span" || n == "font") {
// Ignore
} else {
reject("Unsupported start tag:" + n); // TODO: should be a warning
console.warn("Unsupported start tag: " + n);
}
})
@@ -316,7 +377,7 @@ function enexXmlToMd(stream) {
} else if (isIgnoredEndTag(n)) {
// Skip
} else {
reject("Unsupported end tag:" + n); // TODO: should be a warning
console.warn("Unsupported end tag: " + n);
}
})
@@ -326,7 +387,10 @@ function enexXmlToMd(stream) {
})
saxStream.on('end', function() {
resolve(output);
resolve({
lines: output,
resources: resources,
});
})
stream.pipe(saxStream);
@@ -335,84 +399,58 @@ function enexXmlToMd(stream) {
const path = require('path');
// const path = require('path');
var walk = function (dir, done) {
fs.readdir(dir, function (error, list) {
if (error) return done(error);
var i = 0;
(function next () {
var file = list[i++];
// var walk = function (dir, done) {
// fs.readdir(dir, function (error, list) {
// if (error) return done(error);
// var i = 0;
// (function next () {
// var file = list[i++];
if (!file) return done(null);
file = dir + '/' + file;
// if (!file) return done(null);
// file = dir + '/' + file;
fs.stat(file, function (error, stat) {
if (stat && stat.isDirectory()) {
walk(file, function (error) {
next();
});
} else {
if (path.basename(file) != 'sample4.xml') {
next();
return;
}
// fs.stat(file, function (error, stat) {
// if (stat && stat.isDirectory()) {
// walk(file, function (error) {
// next();
// });
// } else {
// if (path.basename(file) != 'sample4.xml') {
// next();
// return;
// }
if (path.extname(file) == '.xml') {
console.info('Processing: ' + file);
let stream = fs.createReadStream(file);
enexXmlToMd(stream).then((md) => {
console.info(md);
console.info(processMdArrayNewLines(md));
next();
}).catch((error) => {
console.error(error);
return done(error);
});
} else {
next();
}
}
});
})();
});
};
// if (path.extname(file) == '.xml') {
// console.info('Processing: ' + file);
// let stream = fs.createReadStream(file);
// enexXmlToMd(stream).then((md) => {
// console.info(md);
// console.info(processMdArrayNewLines(md));
// next();
// }).catch((error) => {
// console.error(error);
// return done(error);
// });
// } else {
// next();
// }
// }
// });
// })();
// });
// };
walk('/home/laurent/Dropbox/Samples/', function(error) {
if (error) {
throw error;
} else {
console.log('-------------------------------------------------------------');
console.log('finished.');
console.log('-------------------------------------------------------------');
}
});
function parseXml(xml) {
return new Promise((resolve, reject) => {
xml2js.parseString(xml, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
function readFile(path, options = null) {
return new Promise((resolve, reject) => {
fs.readFile(path, options, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
// walk('/home/laurent/Dropbox/Samples/', function(error) {
// if (error) {
// throw error;
// } else {
// console.log('-------------------------------------------------------------');
// console.log('finished.');
// console.log('-------------------------------------------------------------');
// }
// });
function isBlockTag(n) {
return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
@@ -431,7 +469,7 @@ function isAnchor(n) {
}
function isIgnoredEndTag(n) {
return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br";
return n=="en-note" || n=="en-todo" || n=="span" || n=="body" || n=="html" || n=="font" || n=="br" || n=='hr' || n=='s';
}
function isListTag(n) {
@@ -470,63 +508,252 @@ function evernoteXmlToMdArray(xml) {
});
}
function toApiNote(xml) {
let o = {};
o.id = uuid.create();
o.title = xmlNodeText(xml.title);
// o.body = '';
// if (xml.content && xml.content.length) {
// o.body = xmlToMd(xml.content[0]);
// }
o.created_time = dateToTimestamp(xml.created);
o.updated_time = dateToTimestamp(xml.updated);
if (xml['note-attributes'] && xml['note-attributes'].length) {
let attributes = xml['note-attributes'][0];
o.latitude = xmlNodeText(attributes.latitude);
o.longitude = xmlNodeText(attributes.longitude);
o.altitude = xmlNodeText(attributes.altitude);
o.author = xmlNodeText(attributes.author);
function extractRecognitionObjId(recognitionXml) {
const r = recognitionXml.match(/objID="(.*?)"/);
return r && r.length >= 2 ? r[1] : null;
}
o.tags = [];
if (xml.tag && xml.tag.length) o.tags = xml.tag;
function saveNoteToWebApi(note) {
let data = Object.assign({}, note);
delete data.resources;
delete data.tags;
return o;
webApi.post('notes', null, data).then((r) => {
console.info(r);
}).catch((error) => {
console.error("Error for note: " + note.title);
console.error(error);
});
}
function createNoteId(note) {
let shaObj = new jsSHA("SHA-256", "TEXT");
shaObj.update(note.title + '_' + note.body + "_" + note.created_time + "_" + note.updated_time + "_");
let hash = shaObj.getHash("HEX");
return hash.substr(0, 32);
}
function importEnex(parentId, stream) {
return new Promise((resolve, reject) => {
let options = {};
let strict = true;
let saxStream = require('sax').createStream(strict, options);
let nodes = []; // LIFO list of nodes so that we know in which node we are in the onText event
let note = null;
let noteAttributes = null;
let noteResource = null;
let noteResourceAttributes = null;
let noteResourceRecognition = null;
let notes = [];
// readFile('sample.enex', 'utf8').then((content) => {
// return parseXml(content);
// }).then((doc) => {
// let notes = doc['en-export']['note'];
// for (let i = 0; i < notes.length; i++) {
// let note = notes[i];
// let apiNote = toApiNote(note);
function currentNodeName() {
if (!nodes.length) return null;
return nodes[nodes.length - 1].name;
}
function currentNodeAttributes() {
if (!nodes.length) return {};
return nodes[nodes.length - 1].attributes;
}
function processNotes() {
let chain = [];
while (notes.length) {
let note = notes.shift();
const contentStream = stringToStream(note.bodyXml);
chain.push(() => {
return enexXmlToMd(contentStream, note.resources).then((result) => {
delete note.bodyXml;
let mdLines = result.lines;
let firstAttachment = true;
for (let i = 0; i < result.resources.length; i++) {
let r = result.resources[i];
if (firstAttachment) mdLines.push(NEWLINE);
mdLines.push(NEWLINE);
mdLines = addResourceTag(mdLines, r, r.filename);
firstAttachment = false;
}
note.parent_id = parentId;
note.body = processMdArrayNewLines(result.lines);
saveNoteToWebApi(note);
// console.info('======== NOTE ============================================================================');
// let c = note.content;
// delete note.content
// console.info(note);
// console.info('------------------------------------------------------------------------------------------');
// console.info(c);
// if (note.resources.length) {
// console.info('=========================================================');
// console.info(note.content);
// }
// }).catch((error) => {
// console.error('Error reading XML file', error);
// });
});
});
}
return promiseChain(chain);
}
saxStream.on('error', function(e) {
reject(e);
})
saxStream.on('text', function(text) {
let n = currentNodeName();
if (noteAttributes) {
noteAttributes[n] = text;
} else if (noteResourceAttributes) {
noteResourceAttributes[n] = text;
} else if (noteResource) {
if (n == 'data') {
let attr = currentNodeAttributes();
noteResource.dataEncoding = attr.encoding;
}
noteResource[n] = text;
} else if (note) {
if (n == 'title') {
note.title = text;
} else if (n == 'created') {
note.created_time = dateToTimestamp(text);
} else if (n == 'updated') {
note.updated_time = dateToTimestamp(text);
} else if (n == 'tag') {
note.tags.push(text);
}
}
})
saxStream.on('opentag', function(node) {
let n = node.name.toLowerCase();
nodes.push(node);
if (n == 'note') {
note = {
resources: [],
tags: [],
};
} else if (n == 'resource-attributes') {
noteResourceAttributes = {};
} else if (n == 'recognition') {
if (noteResource) noteResourceRecognition = {};
} else if (n == 'note-attributes') {
noteAttributes = {};
} else if (n == 'resource') {
noteResource = {};
}
});
saxStream.on('cdata', function(data) {
let n = currentNodeName();
// import { WebApi } from 'src/web-api.js'
if (noteResourceRecognition) {
noteResourceRecognition.objID = extractRecognitionObjId(data);
} else if (note) {
if (n == 'content') {
note.bodyXml = data;
}
}
});
// let api = new WebApi('http://joplin.local');
saxStream.on('closetag', function(n) {
nodes.pop();
// api.post('sessions', null, {
// email: 'laurent@cozic.net',
// password: '12345678',
// }).then((session) => {
// console.info(session);
// });
if (n == 'note') {
notes.push(note);
if (notes.length >= 10) {
stream.pause();
processNotes().then(() => {
//stream.resume();
}).catch((error) => {
console.info('Error processing note', error);
});
}
note = null;
} else if (n == 'recognition' && noteResource) {
noteResource.id = noteResourceRecognition.objID;
noteResourceRecognition = null;
} else if (n == 'resource-attributes') {
noteResource.filename = noteResourceAttributes['file-name'];
noteResourceAttributes = null;
} else if (n == 'note-attributes') {
note.latitude = noteAttributes.latitude;
note.longitude = noteAttributes.longitude;
note.altitude = noteAttributes.altitude;
note.author = noteAttributes.author;
noteAttributes = null;
} else if (n == 'resource') {
let decodedData = null;
if (noteResource.dataEncoding == 'base64') {
decodedData = Buffer.from(noteResource.data, 'base64');
} else {
reject('Cannot decode resource with encoding: ' + noteResource.dataEncoding);
return;
}
let r = {
id: noteResource.id,
data: decodedData,
mime_type: noteResource.mime,
title: noteResource.filename,
filename: noteResource.filename,
};
r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE
note.resources.push(r);
noteResource = null;
}
});
saxStream.on('end', function() {
processNotes().then(() => { resolve(); });
});
stream.pipe(saxStream);
});
}
// TODO: make it persistent and random
const clientId = 'AB78AB78AB78AB78AB78AB78AB78AB78';
//const folderTitle = 'Laurent';
const folderTitle = 'Voiture';
webApi.post('sessions', null, {
email: 'laurent@cozic.net',
password: '12345678',
client_id: clientId,
}).then((session) => {
webApi.setSession(session.id);
console.info('Got session: ' + session.id);
return webApi.get('folders');
}).then((folders) => {
let folder = null;
for (let i = 0; i < folders.length; i++) {
if (folders[i].title = folderTitle) {
folder = folders[i];
break;
}
}
return folder ? Promise.resolve(folder) : webApi.post('folders', null, { title: folderTitle });
}).then((folder) => {
let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/' + folderTitle + '.enex');
//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/afaire.enex');
//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/testtags.enex');
importEnex(folder.id, fileStream).then(() => {
//console.info('DONE IMPORTING');
}).catch((error) => {
console.error('Cannot import', error);
});
}).catch((error) => {
console.error(error);
});

4
CliClient/package.json
View File

@@ -5,14 +5,14 @@
"dependencies": {
"app-module-path": "^2.2.0",
"form-data": "^2.1.4",
"jssha": "^2.3.0",
"moment": "^2.18.1",
"node-fetch": "^1.7.1",
"promise": "^7.1.1",
"react": "16.0.0-alpha.6",
"sax": "^1.2.2",
"string-to-stream": "^1.1.0",
"uuid": "^3.0.1",
"xml2js": "^0.4.17"
"uuid": "^3.0.1"
},
"devDependencies": {
"babel-changed": "^7.0.0",

4
QtClient/evernote-import/main.cpp
View File

@@ -266,7 +266,9 @@ Note parseNote(QXmlStreamReader& reader) {
// <?xml version="1.0" encoding="UTF-8"?>
// <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
// <en-note>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media></en-note>]]>
// <en-media alt="your QR code" hash="216a16a1bbe007fba4ccf60b118b4ccc" type="image/png"></en-media>
// </en-note>
// ]]>
// </content>
// <created>20160921T203424Z</created>
// <updated>20160921T203438Z</updated>

3
ReactNativeClient/src/web-api.js
View File

@@ -97,7 +97,8 @@ class WebApi {
let r = this.makeRequest(method, path, query, data);
Log.debug(WebApi.toCurl(r, data));
//Log.debug(WebApi.toCurl(r, data));
//console.info(WebApi.toCurl(r, data));
fetch(r.url, r.options).then(function(response) {
let responseClone = response.clone();

8
debug_client/css/style.css
View File

@@ -67,13 +67,19 @@ td {
.form-group label {
width: 200px;
display: inline-block;
vertical-align: top;
}
.form-group input {
width: 300px;
width: 600px;
display: inline-block;
}
.form-group textarea {
width: 600px;
height: 400px;
}
.debug {
color: #777;
font-family: monospace;

4
debug_client/views/item.php
View File

@@ -2,7 +2,11 @@
<?php foreach ($item as $k => $v): ?>
<div class="form-group">
<label><?php echo htmlentities($k); ?></label>
<?php if ($k == 'body'): ?>
<textarea class="form-control" name="item_<?php echo htmlentities($k); ?>"><?php echo htmlentities($v); ?></textarea>
<?php else: ?>
<input type="text" class="form-control" name="item_<?php echo htmlentities($k); ?>" value="<?php echo htmlentities($v); ?>" >
<?php endif; ?>
</div>
<?php endforeach; ?>
<input type="hidden" value="<?php echo htmlentities(json_encode($item)); ?>" name="original_item" />

2
src/AppBundle/Controller/ApiController.php
View File

@@ -175,7 +175,7 @@ abstract class ApiController extends Controller {
if (!isset($_SERVER['CONTENT_TYPE']) || strpos($_SERVER['CONTENT_TYPE'], 'application/x-www-form-urlencoded') === 0) {
parse_str($input, $output);
} else {
throw new \Exception('Only application/x-www-form-urlencoded Content-Type is supported');
throw new \Exception('Only application/x-www-form-urlencoded Content-Type is supported. Not supported: ' . $_SERVER['CONTENT_TYPE']);
}
return $output;

4
src/AppBundle/Controller/FoldersController.php
View File

@@ -17,13 +17,13 @@ class FoldersController extends ApiController {
*/
public function allAction(Request $request) {
if ($request->isMethod('GET')) {
return static::successResponse(Folder::all());
return static::successResponse(Folder::allByOwnerId($this->userId()));
}
if ($request->isMethod('POST')) {
$folder = new Folder();
$folder->fromPublicArray($request->request->all());
$folder->owner_id = $this->user()->id;
$folder->owner_id = $this->userId();
$folder->validate();
$folder->save();
return static::successResponse(Folder::find($folder->id));

7
src/AppBundle/Controller/NotesController.php
View File

@@ -18,7 +18,7 @@ class NotesController extends ApiController {
public function allAction(Request $request) {
if ($request->isMethod('POST')) {
$note = new Note();
$note->fromPublicArray($request->request->all());
$note->fromPublicArray(Note::filter($request->request->all()));
$note->owner_id = $this->user()->id;
$note->save();
return static::successResponse($note->toPublicArray());
@@ -44,8 +44,7 @@ class NotesController extends ApiController {
if ($request->isMethod('PUT')) {
$isNew = !$note;
if ($isNew) $note = new Note();
$data = Note::filter($this->putParameters());
$note->fromPublicArray($data);
$note->fromPublicArray(Note::filter($this->putParameters()));
$note->id = Note::unhex($id);
$note->owner_id = $this->user()->id;
$note->setIsNew($isNew);
@@ -54,7 +53,7 @@ class NotesController extends ApiController {
}
if ($request->isMethod('PATCH')) {
$note->fromPublicArray($this->patchParameters());
$note->fromPublicArray(Note::filter($this->patchParameters()));
$note->save();
return static::successResponse($note);
}

6
src/AppBundle/Model/Note.php
View File

@@ -33,9 +33,9 @@ class Note extends BaseItem {
static public function filter($data, $keepId = false) {
$output = parent::filter($data);
if (array_key_exists('longitude', $output)) $output['longitude'] = (string)number_format($output['longitude'], 8);
if (array_key_exists('latitude', $output)) $output['latitude'] = (string)number_format($output['latitude'], 8);
if (array_key_exists('altitude', $output)) $output['altitude'] = (string)number_format($output['altitude'], 4);
if (array_key_exists('longitude', $output)) $output['longitude'] = (string)number_format((float)$output['longitude'], 8);
if (array_key_exists('latitude', $output)) $output['latitude'] = (string)number_format((float)$output['latitude'], 8);
if (array_key_exists('altitude', $output)) $output['altitude'] = (string)number_format((float)$output['altitude'], 4);
return $output;
}

14
tests/Controller/FoldersControllerTest.php
View File

@@ -43,4 +43,18 @@ class FoldersControllerTest extends BaseControllerTestCase {
$this->assertEquals($f1['id'], $f2['id']);
}
// public function testEmoticonText() {
// $this->loadSession(1, 1);
// var_dump(mb_check_encoding('Voiture 🚘', 'UTF-8'));die();
// $text1 = iconv('UTF-8', 'UCS-2LE', 'Voiture 🚘');
// var_dump($text1);
// die();
// $f1 = $this->request('POST', '/folders', null, array('title' => 'Voiture 🚘'));
// var_dump($f1);
// die();
// }
}

1
web/app.php
View File

@@ -15,6 +15,7 @@ $kernel->loadClassCache();
// When using the HttpCache, you need to call the method in your front controller instead of relying on the configuration parameter
//Request::enableHttpMethodParameterOverride();
try {
$request = Request::createFromGlobals();
$response = $kernel->handle($request);