self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-26 18:58:21 +02:00
Code Issues Releases Activity

Import Evernote tags

Browse Source
This commit is contained in:
Laurent Cozic 2017-07-02 16:46:03 +01:00
parent 1aeedb80f7
commit f04c1c58f1
14 changed files with 127 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
CliClient/app/file-api-test.js
View File

@ -1,93 +0,0 @@
import { FileApi } from 'lib/file-api.js';
import { FileApiDriverLocal } from 'lib/file-api-driver-local.js';
import { Database } from 'lib/database.js';
import { DatabaseDriverNode } from 'lib/database-driver-node.js';
import { Log } from 'lib/log.js';
const fs = require('fs');
// let driver = new FileApiDriverLocal();
// let api = new FileApi('/home/laurent/Temp/TestImport', driver);
// api.list('/').then((items) => {
// console.info(items);
// }).then(() => {
// return api.get('un.txt');
// }).then((content) => {
// console.info(content);
// }).then(() => {
// return api.mkdir('TESTING');
// }).then(() => {
// return api.put('un.txt', 'testing change');
// }).then(() => {
// return api.delete('deux.txt');
// }).catch((error) => {
// console.error('ERROR', error);
// });
Log.setLevel(Log.LEVEL_DEBUG);
let db = new Database(new DatabaseDriverNode());
//db.setDebugMode(true);
db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => {
return db.selectAll('SELECT * FROM table_fields');
}).then((rows) => {
});
//'/home/laurent/Temp/TestImport'
// var sqlite3 = require('sqlite3').verbose();
// var db = new sqlite3.Database(':memory:');
// db.run("CREATE TABLE lorem (info TEXT)", () => {
// db.exec('INSERT INTO lorem VALUES "un"', () => {
// db.exec('INSERT INTO lorem VALUES "deux"', () => {
// let st = db.prepare("SELECT rowid AS id, info FROM lorem", () => {
// st.get((error, row) => {
// console.info(row);
// });
// });
// });
// });
// });
// var stmt = db.prepare("INSERT INTO lorem VALUES (?)");
// for (var i = 0; i < 10; i++) {
// stmt.run("Ipsum " + i);
// }
// stmt.finalize();
// let st = db.prepare("SELECT rowid AS id, info FROM lorem");
// st.get({}, (row) => {
// console.info('xx',row);
// });
// st.finalize();
//db.serialize(function() {
// db.run("CREATE TABLE lorem (info TEXT)");
// var stmt = db.prepare("INSERT INTO lorem VALUES (?)");
// for (var i = 0; i < 10; i++) {
// stmt.run("Ipsum " + i);
// }
// stmt.finalize();
// let st = db.prepare("SELECT rowid AS id, info FROM lorem");
// st.get({}, (row) => {
// console.info('xx',row);
// });
// st.finalize();
// db.each("SELECT rowid AS id, info FROM lorem", function(err, row) {
// console.log(row.id + ": " + row.info);
// });
//});
//db.close();

22
CliClient/app/import-enex.js
View File

@ -4,6 +4,7 @@ import { promiseChain } from 'lib/promise-utils.js';
import { folderItemFilename } from 'lib/string-utils.js' import { folderItemFilename } from 'lib/string-utils.js'
import { BaseModel } from 'lib/base-model.js'; import { BaseModel } from 'lib/base-model.js';
import { Note } from 'lib/models/note.js'; import { Note } from 'lib/models/note.js';
import { Tag } from 'lib/models/tag.js';
import { Resource } from 'lib/models/resource.js'; import { Resource } from 'lib/models/resource.js';
import { Folder } from 'lib/models/folder.js'; import { Folder } from 'lib/models/folder.js';
import { enexXmlToMd } from './import-enex-md-gen.js'; import { enexXmlToMd } from './import-enex-md-gen.js';
@ -74,6 +75,21 @@ async function saveNoteResources(note) {
return resourcesCreated; return resourcesCreated;
} }
async function saveNoteTags(note) {
let noteTagged = 0;
for (let i = 0; i < note.tags.length; i++) {
let tagTitle = note.tags[i];
let tag = await Tag.loadByTitle(tagTitle);
if (!tag) tag = await Tag.save({ title: tagTitle });
await Tag.addNote(tag.id, note.id);
noteTagged++;
}
return noteTagged;
}
async function saveNoteToStorage(note, fuzzyMatching = false) { async function saveNoteToStorage(note, fuzzyMatching = false) {
note = Note.filter(note); note = Note.filter(note);
@ -84,11 +100,15 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
noteUpdated: false, noteUpdated: false,
noteSkipped: false, noteSkipped: false,
resourcesCreated: 0, resourcesCreated: 0,
noteTagged: 0,
}; };
let resourcesCreated = await saveNoteResources(note); let resourcesCreated = await saveNoteResources(note);
result.resourcesCreated += resourcesCreated; result.resourcesCreated += resourcesCreated;
let noteTagged = await saveNoteTags(note);
result.noteTagged += noteTagged;
if (existingNote) { if (existingNote) {
let diff = BaseModel.diffObjects(existingNote, note); let diff = BaseModel.diffObjects(existingNote, note);
delete diff.tags; delete diff.tags;
@ -128,6 +148,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
updated: 0, updated: 0,
skipped: 0, skipped: 0,
resourcesCreated: 0, resourcesCreated: 0,
noteTagged: 0,
}; };
let stream = fs.createReadStream(filePath); let stream = fs.createReadStream(filePath);
@ -192,6 +213,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
progressState.skipped++; progressState.skipped++;
} }
progressState.resourcesCreated += result.resourcesCreated; progressState.resourcesCreated += result.resourcesCreated;
progressState.noteTagged += result.noteTagged;
importOptions.onProgress(progressState); importOptions.onProgress(progressState);
}); });
}); });

34
CliClient/app/main.js
View File

@ -153,33 +153,36 @@ commands.push({
commands.push({ commands.push({
usage: 'cat <title>', usage: 'cat <title>',
description: 'Displays the given item data.', description: 'Displays the given item data.',
action: function(args, end) { action: async function(args, end) {
let title = args['title']; try {
let title = args['title'];
let promise = null; let item = null;
if (!currentFolder) { if (!currentFolder) {
promise = Folder.loadByField('title', title); item = await Folder.loadByField('title', title);
} else { } else {
promise = Note.loadFolderNoteByField(currentFolder.id, 'title', title); item = await Note.loadFolderNoteByField(currentFolder.id, 'title', title);
} }
promise.then((item) => {
if (!item) { if (!item) {
this.log(_('No item with title "%s" found.', title)); this.log(_('No item with title "%s" found.', title));
end(); end();
return; return;
} }
let content = null;
if (!currentFolder) { if (!currentFolder) {
this.log(Folder.serialize(item)); content = await Folder.serialize(item);
} else { } else {
this.log(Note.serialize(item)); content = await Note.serialize(item);
} }
}).catch((error) => {
this.log(content);
} catch(error) {
this.log(error); this.log(error);
}).then(() => { }
end();
}); end();
}, },
autocomplete: autocompleteItems, autocomplete: autocompleteItems,
}); });
@ -467,6 +470,7 @@ commands.push({
if (progressState.updated) line.push(_('Updated: %d.', progressState.updated)); if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped)); if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped));
if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated)); if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
if (progressState.notesTagged) line.push(_('Tagged: %d.', progressState.notesTagged));
redrawnCalled = true; redrawnCalled = true;
vorpal.ui.redraw(line.join(' ')); vorpal.ui.redraw(line.join(' '));
}, },

2
CliClient/tests/synchronizer.js
View File

@ -37,7 +37,7 @@ async function localItemsSameAsRemote(locals, expect) {
expect(remote.updated_time).toBe(dbItem.updated_time); expect(remote.updated_time).toBe(dbItem.updated_time);
let remoteContent = await fileApi().get(path); let remoteContent = await fileApi().get(path);
remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? Note.unserialize(remoteContent) : Folder.unserialize(remoteContent); remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? await Note.unserialize(remoteContent) : await Folder.unserialize(remoteContent);
expect(remoteContent.title).toBe(dbItem.title); expect(remoteContent.title).toBe(dbItem.title);
} }
} catch (error) { } catch (error) {

4
CliClient/tests/test-utils.js
View File

@ -48,7 +48,9 @@ function clearDatabase(id = null) {
'DELETE FROM changes', 'DELETE FROM changes',
'DELETE FROM notes', 'DELETE FROM notes',
'DELETE FROM folders', 'DELETE FROM folders',
'DELETE FROM item_sync_times', 'DELETE FROM resources',
'DELETE FROM tags',
'DELETE FROM note_tags',
]; ];
return databases_[id].transactionExecBatch(queries); return databases_[id].transactionExecBatch(queries);

4
lib/base-model.js
View File

@ -161,6 +161,10 @@ class BaseModel {
return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]); return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]);
} }
static loadByTitle(fieldValue) {
return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `title` = ?', [fieldValue]);
}
static applyPatch(model, patch) { static applyPatch(model, patch) {
model = Object.assign({}, model); model = Object.assign({}, model);
for (let n in patch) { for (let n in patch) {

39
lib/database.js
View File

@ -10,8 +10,8 @@ CREATE TABLE folders (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
parent_id TEXT NOT NULL DEFAULT "", parent_id TEXT NOT NULL DEFAULT "",
title TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "",
created_time INT NOT NULL DEFAULT 0, created_time INT NOT NULL,
updated_time INT NOT NULL DEFAULT 0, updated_time INT NOT NULL,
sync_time INT NOT NULL DEFAULT 0 sync_time INT NOT NULL DEFAULT 0
); );
@ -24,8 +24,8 @@ CREATE TABLE notes (
parent_id TEXT NOT NULL DEFAULT "", parent_id TEXT NOT NULL DEFAULT "",
title TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "",
body TEXT NOT NULL DEFAULT "", body TEXT NOT NULL DEFAULT "",
created_time INT NOT NULL DEFAULT 0, created_time INT NOT NULL,
updated_time INT NOT NULL DEFAULT 0, updated_time INT NOT NULL,
sync_time INT NOT NULL DEFAULT 0, sync_time INT NOT NULL DEFAULT 0,
is_conflict INT NOT NULL DEFAULT 0, is_conflict INT NOT NULL DEFAULT 0,
latitude NUMERIC NOT NULL DEFAULT 0, latitude NUMERIC NOT NULL DEFAULT 0,
@ -58,15 +58,15 @@ CREATE TABLE deleted_items (
CREATE TABLE tags ( CREATE TABLE tags (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
title TEXT, title TEXT NOT NULL DEFAULT "",
created_time INT, created_time INT NOT NULL,
updated_time INT updated_time INT NOT NULL
); );
CREATE TABLE note_tags ( CREATE TABLE note_tags (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
note_id TEXT, note_id TEXT NOT NULL,
tag_id TEXT tag_id TEXT NOT NULL
); );
CREATE TABLE resources ( CREATE TABLE resources (
@ -79,16 +79,6 @@ CREATE TABLE resources (
sync_time INT NOT NULL DEFAULT 0 sync_time INT NOT NULL DEFAULT 0
); );
CREATE TABLE note_resources (
id INTEGER PRIMARY KEY,
note_id TEXT,
resource_id TEXT
);
CREATE TABLE version (
version INT
);
CREATE TABLE changes ( CREATE TABLE changes (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
\`type\` INT, \`type\` INT,
@ -111,10 +101,8 @@ CREATE TABLE table_fields (
field_default TEXT field_default TEXT
); );
CREATE TABLE item_sync_times ( CREATE TABLE version (
id INTEGER PRIMARY KEY, version INT
item_id TEXT,
\`time\` INT
); );
INSERT INTO version (version) VALUES (1); INSERT INTO version (version) VALUES (1);
@ -208,6 +196,11 @@ class Database {
} }
async exec(sql, params = null) { async exec(sql, params = null) {
if (typeof sql === 'object') {
params = sql.params;
sql = sql.sql;
}
let result = null; let result = null;
let waitTime = 50; let waitTime = 50;
let totalWaitTime = 0; let totalWaitTime = 0;

6
lib/models/base-item.js
View File

@ -100,7 +100,7 @@ class BaseItem extends BaseModel {
return propValue; return propValue;
} }
static serialize(item, type = null, shownKeys = null) { static async serialize(item, type = null, shownKeys = null) {
item = this.filter(item); item = this.filter(item);
let output = []; let output = [];
@ -118,7 +118,7 @@ class BaseItem extends BaseModel {
return output.join("\n"); return output.join("\n");
} }
static unserialize(content) { static async unserialize(content) {
let lines = content.split("\n"); let lines = content.split("\n");
let output = {}; let output = {};
let state = 'readingProps'; let state = 'readingProps';
@ -156,7 +156,7 @@ class BaseItem extends BaseModel {
for (let n in output) { for (let n in output) {
if (!output.hasOwnProperty(n)) continue; if (!output.hasOwnProperty(n)) continue;
output[n] = this.unserialize_format(output.type_, n, output[n]); output[n] = await this.unserialize_format(output.type_, n, output[n]);
} }
return output; return output;

2
lib/models/folder.js
View File

@ -14,7 +14,7 @@ class Folder extends BaseItem {
return 'folders'; return 'folders';
} }
static serialize(folder) { static async serialize(folder) {
let fieldNames = this.fieldNames(); let fieldNames = this.fieldNames();
fieldNames.push('type_'); fieldNames.push('type_');
lodash.pull(fieldNames, 'parent_id', 'sync_time'); lodash.pull(fieldNames, 'parent_id', 'sync_time');

39
lib/models/item-sync-time.js
View File

@ -1,39 +0,0 @@
import { BaseModel } from 'lib/base-model.js';
class ItemSyncTime extends BaseModel {
static time(itemId) {
if (itemId in this.cache_) return Promise.resolve(this.cache_[itemId]);
return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => {
this.cache_[itemId] = row ? row.time : 0;
return this.cache_[itemId];
});
}
static setTime(itemId, time) {
return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => {
let p = null;
if (row) {
p = this.db().exec('UPDATE item_sync_times SET `time` = ? WHERE item_id = ?', [time, itemId]);
} else {
p = this.db().exec('INSERT INTO item_sync_times (item_id, `time`) VALUES (?, ?)', [itemId, time]);
}
return p.then(() => {
this.cache_[itemId] = time;
});
});
}
static deleteTime(itemId) {
return this.db().exec('DELETE FROM item_sync_times WHERE item_id = ?', [itemId]).then(() => {
delete this.cache_[itemId];
});
}
}
ItemSyncTime.cache_ = {};
export { ItemSyncTime };

2
lib/models/note.js
View File

@ -13,7 +13,7 @@ class Note extends BaseItem {
return 'notes'; return 'notes';
} }
static serialize(note, type = null, shownKeys = null) { static async serialize(note, type = null, shownKeys = null) {
let fieldNames = this.fieldNames(); let fieldNames = this.fieldNames();
fieldNames.push('type_'); fieldNames.push('type_');
lodash.pull(fieldNames, 'is_conflict', 'sync_time', 'body'); // Exclude 'body' since it's going to be added separately at the top of the note lodash.pull(fieldNames, 'is_conflict', 'sync_time', 'body'); // Exclude 'body' since it's going to be added separately at the top of the note

2
lib/models/resource.js
View File

@ -15,7 +15,7 @@ class Resource extends BaseItem {
return BaseModel.MODEL_TYPE_RESOURCE; return BaseModel.MODEL_TYPE_RESOURCE;
} }
static serialize(item, type = null, shownKeys = null) { static async serialize(item, type = null, shownKeys = null) {
let fieldNames = this.fieldNames(); let fieldNames = this.fieldNames();
fieldNames.push('type_'); fieldNames.push('type_');
lodash.pull(fieldNames, 'sync_time'); lodash.pull(fieldNames, 'sync_time');

52
lib/models/tag.js Normal file
View File

@ -0,0 +1,52 @@
import { BaseModel } from 'lib/base-model.js';
import { Database } from 'lib/database.js';
import { BaseItem } from 'lib/models/base-item.js';
import lodash from 'lodash';
class Tag extends BaseItem {
static tableName() {
return 'tags';
}
static itemType() {
return BaseModel.MODEL_TYPE_TAG;
}
static async serialize(item, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
fieldNames.push(() => {
});
lodash.pull(fieldNames, 'sync_time');
return super.serialize(item, 'tag', fieldNames);
}
static tagNoteIds(tagId) {
return this.db().selectAll('SELECT note_id FROM note_tags WHERE tag_id = ?', [tagId]);
}
static async addNote(tagId, noteId) {
let hasIt = await this.hasNote(tagId, noteId);
if (hasIt) return;
let query = Database.insertQuery('note_tags', {
tag_id: tagId,
note_id: noteId,
});
return this.db().exec(query);
}
static async hasNote(tagId, noteId) {
let r = await this.db().selectOne('SELECT note_id FROM note_tags WHERE tag_id = ? AND note_id = ? LIMIT 1', [tagId, noteId]);
return !!r;
}
static removeNote(tagId, noteId) {
return this.db().exec('DELETE FROM note_tags WHERE tag_id = ? AND note_id = ?', [tagId, noteId]);
}
}
export { Tag };

8
lib/synchronizer.js
View File

@ -145,7 +145,7 @@ class Synchronizer {
if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path)); if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path));
let remote = await this.api().stat(path); let remote = await this.api().stat(path);
let content = ItemClass.serialize(local); let content = await ItemClass.serialize(local);
let action = null; let action = null;
let updateSyncTimeOnly = true; let updateSyncTimeOnly = true;
let reason = ''; let reason = '';
@ -198,7 +198,7 @@ class Synchronizer {
if (remote) { if (remote) {
let remoteContent = await this.api().get(path); let remoteContent = await this.api().get(path);
local = BaseItem.unserialize(remoteContent); local = await BaseItem.unserialize(remoteContent);
local.sync_time = time.unixMs(); local.sync_time = time.unixMs();
await ItemClass.save(local, { autoTimestamp: false }); await ItemClass.save(local, { autoTimestamp: false });
@ -219,7 +219,7 @@ class Synchronizer {
if (remote) { if (remote) {
let remoteContent = await this.api().get(path); let remoteContent = await this.api().get(path);
local = BaseItem.unserialize(remoteContent); local = await BaseItem.unserialize(remoteContent);
local.sync_time = time.unixMs(); local.sync_time = time.unixMs();
await ItemClass.save(local, { autoTimestamp: false }); await ItemClass.save(local, { autoTimestamp: false });
@ -301,7 +301,7 @@ class Synchronizer {
this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path);
continue; continue;
} }
content = BaseItem.unserialize(content); content = await BaseItem.unserialize(content);
let ItemClass = BaseItem.itemClass(content); let ItemClass = BaseItem.itemClass(content);
let newContent = Object.assign({}, content); let newContent = Object.assign({}, content);