self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-26 18:58:21 +02:00
Code Issues Releases Activity

Import Evernote tags

Browse Source
This commit is contained in:
Laurent Cozic 2017-07-02 16:46:03 +01:00
parent 1aeedb80f7
commit f04c1c58f1
14 changed files with 127 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
CliClient/app/file-api-test.js
View File

@ -1,93 +0,0 @@
import { FileApi } from 'lib/file-api.js';
import { FileApiDriverLocal } from 'lib/file-api-driver-local.js';
import { Database } from 'lib/database.js';
import { DatabaseDriverNode } from 'lib/database-driver-node.js';
import { Log } from 'lib/log.js';
const fs = require('fs');
// let driver = new FileApiDriverLocal();
// let api = new FileApi('/home/laurent/Temp/TestImport', driver);
// api.list('/').then((items) => {
// console.info(items);
// }).then(() => {
// return api.get('un.txt');
// }).then((content) => {
// console.info(content);
// }).then(() => {
// return api.mkdir('TESTING');
// }).then(() => {
// return api.put('un.txt', 'testing change');
// }).then(() => {
// return api.delete('deux.txt');
// }).catch((error) => {
// console.error('ERROR', error);
// });
Log.setLevel(Log.LEVEL_DEBUG);
let db = new Database(new DatabaseDriverNode());
//db.setDebugMode(true);
db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => {
return db.selectAll('SELECT * FROM table_fields');
}).then((rows) => {
});
//'/home/laurent/Temp/TestImport'
// var sqlite3 = require('sqlite3').verbose();
// var db = new sqlite3.Database(':memory:');
// db.run("CREATE TABLE lorem (info TEXT)", () => {
// db.exec('INSERT INTO lorem VALUES "un"', () => {
// db.exec('INSERT INTO lorem VALUES "deux"', () => {
// let st = db.prepare("SELECT rowid AS id, info FROM lorem", () => {
// st.get((error, row) => {
// console.info(row);
// });
// });
// });
// });
// });
// var stmt = db.prepare("INSERT INTO lorem VALUES (?)");
// for (var i = 0; i < 10; i++) {
// stmt.run("Ipsum " + i);
// }
// stmt.finalize();
// let st = db.prepare("SELECT rowid AS id, info FROM lorem");
// st.get({}, (row) => {
// console.info('xx',row);
// });
// st.finalize();
//db.serialize(function() {
// db.run("CREATE TABLE lorem (info TEXT)");
// var stmt = db.prepare("INSERT INTO lorem VALUES (?)");
// for (var i = 0; i < 10; i++) {
// stmt.run("Ipsum " + i);
// }
// stmt.finalize();
// let st = db.prepare("SELECT rowid AS id, info FROM lorem");
// st.get({}, (row) => {
// console.info('xx',row);
// });
// st.finalize();
// db.each("SELECT rowid AS id, info FROM lorem", function(err, row) {
// console.log(row.id + ": " + row.info);
// });
//});
//db.close();

22
CliClient/app/import-enex.js
View File

@ -4,6 +4,7 @@ import { promiseChain } from 'lib/promise-utils.js';
import { folderItemFilename } from 'lib/string-utils.js'
import { BaseModel } from 'lib/base-model.js';
import { Note } from 'lib/models/note.js';
import { Tag } from 'lib/models/tag.js';
import { Resource } from 'lib/models/resource.js';
import { Folder } from 'lib/models/folder.js';
import { enexXmlToMd } from './import-enex-md-gen.js';
@ -74,6 +75,21 @@ async function saveNoteResources(note) {
return resourcesCreated;
}
async function saveNoteTags(note) {
let noteTagged = 0;
for (let i = 0; i < note.tags.length; i++) {
let tagTitle = note.tags[i];
let tag = await Tag.loadByTitle(tagTitle);
if (!tag) tag = await Tag.save({ title: tagTitle });
await Tag.addNote(tag.id, note.id);
noteTagged++;
}
return noteTagged;
}
async function saveNoteToStorage(note, fuzzyMatching = false) {
note = Note.filter(note);
@ -84,11 +100,15 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
noteUpdated: false,
noteSkipped: false,
resourcesCreated: 0,
noteTagged: 0,
};
let resourcesCreated = await saveNoteResources(note);
result.resourcesCreated += resourcesCreated;
let noteTagged = await saveNoteTags(note);
result.noteTagged += noteTagged;
if (existingNote) {
let diff = BaseModel.diffObjects(existingNote, note);
delete diff.tags;
@ -128,6 +148,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
updated: 0,
skipped: 0,
resourcesCreated: 0,
noteTagged: 0,
};
let stream = fs.createReadStream(filePath);
@ -192,6 +213,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
progressState.skipped++;
}
progressState.resourcesCreated += result.resourcesCreated;
progressState.noteTagged += result.noteTagged;
importOptions.onProgress(progressState);
});
});

34
CliClient/app/main.js
View File

@ -153,33 +153,36 @@ commands.push({
commands.push({
usage: 'cat <title>',
description: 'Displays the given item data.',
action: function(args, end) {
let title = args['title'];
action: async function(args, end) {
try {
let title = args['title'];
let promise = null;
if (!currentFolder) {
promise = Folder.loadByField('title', title);
} else {
promise = Note.loadFolderNoteByField(currentFolder.id, 'title', title);
}
let item = null;
if (!currentFolder) {
item = await Folder.loadByField('title', title);
} else {
item = await Note.loadFolderNoteByField(currentFolder.id, 'title', title);
}
promise.then((item) => {
if (!item) {
this.log(_('No item with title "%s" found.', title));
end();
return;
}
let content = null;
if (!currentFolder) {
this.log(Folder.serialize(item));
content = await Folder.serialize(item);
} else {
this.log(Note.serialize(item));
content = await Note.serialize(item);
}
}).catch((error) => {
this.log(content);
} catch(error) {
this.log(error);
}).then(() => {
end();
});
}
end();
},
autocomplete: autocompleteItems,
});
@ -467,6 +470,7 @@ commands.push({
if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped));
if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
if (progressState.notesTagged) line.push(_('Tagged: %d.', progressState.notesTagged));
redrawnCalled = true;
vorpal.ui.redraw(line.join(' '));
},

2
CliClient/tests/synchronizer.js
View File

@ -37,7 +37,7 @@ async function localItemsSameAsRemote(locals, expect) {
expect(remote.updated_time).toBe(dbItem.updated_time);
let remoteContent = await fileApi().get(path);
remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? Note.unserialize(remoteContent) : Folder.unserialize(remoteContent);
remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? await Note.unserialize(remoteContent) : await Folder.unserialize(remoteContent);
expect(remoteContent.title).toBe(dbItem.title);
}
} catch (error) {

4
CliClient/tests/test-utils.js
View File

@ -48,7 +48,9 @@ function clearDatabase(id = null) {
'DELETE FROM changes',
'DELETE FROM notes',
'DELETE FROM folders',
'DELETE FROM item_sync_times',
'DELETE FROM resources',
'DELETE FROM tags',
'DELETE FROM note_tags',
];
return databases_[id].transactionExecBatch(queries);

4
lib/base-model.js
View File

@ -161,6 +161,10 @@ class BaseModel {
return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]);
}
static loadByTitle(fieldValue) {
return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `title` = ?', [fieldValue]);
}
static applyPatch(model, patch) {
model = Object.assign({}, model);
for (let n in patch) {

39
lib/database.js
View File

@ -10,8 +10,8 @@ CREATE TABLE folders (
id TEXT PRIMARY KEY,
parent_id TEXT NOT NULL DEFAULT "",
title TEXT NOT NULL DEFAULT "",
created_time INT NOT NULL DEFAULT 0,
updated_time INT NOT NULL DEFAULT 0,
created_time INT NOT NULL,
updated_time INT NOT NULL,
sync_time INT NOT NULL DEFAULT 0
);
@ -24,8 +24,8 @@ CREATE TABLE notes (
parent_id TEXT NOT NULL DEFAULT "",
title TEXT NOT NULL DEFAULT "",
body TEXT NOT NULL DEFAULT "",
created_time INT NOT NULL DEFAULT 0,
updated_time INT NOT NULL DEFAULT 0,
created_time INT NOT NULL,
updated_time INT NOT NULL,
sync_time INT NOT NULL DEFAULT 0,
is_conflict INT NOT NULL DEFAULT 0,
latitude NUMERIC NOT NULL DEFAULT 0,
@ -58,15 +58,15 @@ CREATE TABLE deleted_items (
CREATE TABLE tags (
id TEXT PRIMARY KEY,
title TEXT,
created_time INT,
updated_time INT
title TEXT NOT NULL DEFAULT "",
created_time INT NOT NULL,
updated_time INT NOT NULL
);
CREATE TABLE note_tags (
id INTEGER PRIMARY KEY,
note_id TEXT,
tag_id TEXT
note_id TEXT NOT NULL,
tag_id TEXT NOT NULL
);
CREATE TABLE resources (
@ -79,16 +79,6 @@ CREATE TABLE resources (
sync_time INT NOT NULL DEFAULT 0
);
CREATE TABLE note_resources (
id INTEGER PRIMARY KEY,
note_id TEXT,
resource_id TEXT
);
CREATE TABLE version (
version INT
);
CREATE TABLE changes (
id INTEGER PRIMARY KEY,
\`type\` INT,
@ -111,10 +101,8 @@ CREATE TABLE table_fields (
field_default TEXT
);
CREATE TABLE item_sync_times (
id INTEGER PRIMARY KEY,
item_id TEXT,
\`time\` INT
CREATE TABLE version (
version INT
);
INSERT INTO version (version) VALUES (1);
@ -208,6 +196,11 @@ class Database {
}
async exec(sql, params = null) {
if (typeof sql === 'object') {
params = sql.params;
sql = sql.sql;
}
let result = null;
let waitTime = 50;
let totalWaitTime = 0;

6
lib/models/base-item.js
View File

@ -100,7 +100,7 @@ class BaseItem extends BaseModel {
return propValue;
}
static serialize(item, type = null, shownKeys = null) {
static async serialize(item, type = null, shownKeys = null) {
item = this.filter(item);
let output = [];
@ -118,7 +118,7 @@ class BaseItem extends BaseModel {
return output.join("\n");
}
static unserialize(content) {
static async unserialize(content) {
let lines = content.split("\n");
let output = {};
let state = 'readingProps';
@ -156,7 +156,7 @@ class BaseItem extends BaseModel {
for (let n in output) {
if (!output.hasOwnProperty(n)) continue;
output[n] = this.unserialize_format(output.type_, n, output[n]);
output[n] = await this.unserialize_format(output.type_, n, output[n]);
}
return output;

2
lib/models/folder.js
View File

@ -14,7 +14,7 @@ class Folder extends BaseItem {
return 'folders';
}
static serialize(folder) {
static async serialize(folder) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
lodash.pull(fieldNames, 'parent_id', 'sync_time');

39
lib/models/item-sync-time.js
View File

@ -1,39 +0,0 @@
import { BaseModel } from 'lib/base-model.js';
class ItemSyncTime extends BaseModel {
static time(itemId) {
if (itemId in this.cache_) return Promise.resolve(this.cache_[itemId]);
return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => {
this.cache_[itemId] = row ? row.time : 0;
return this.cache_[itemId];
});
}
static setTime(itemId, time) {
return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => {
let p = null;
if (row) {
p = this.db().exec('UPDATE item_sync_times SET `time` = ? WHERE item_id = ?', [time, itemId]);
} else {
p = this.db().exec('INSERT INTO item_sync_times (item_id, `time`) VALUES (?, ?)', [itemId, time]);
}
return p.then(() => {
this.cache_[itemId] = time;
});
});
}
static deleteTime(itemId) {
return this.db().exec('DELETE FROM item_sync_times WHERE item_id = ?', [itemId]).then(() => {
delete this.cache_[itemId];
});
}
}
ItemSyncTime.cache_ = {};
export { ItemSyncTime };

2
lib/models/note.js
View File

@ -13,7 +13,7 @@ class Note extends BaseItem {
return 'notes';
}
static serialize(note, type = null, shownKeys = null) {
static async serialize(note, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
lodash.pull(fieldNames, 'is_conflict', 'sync_time', 'body'); // Exclude 'body' since it's going to be added separately at the top of the note

2
lib/models/resource.js
View File

@ -15,7 +15,7 @@ class Resource extends BaseItem {
return BaseModel.MODEL_TYPE_RESOURCE;
}
static serialize(item, type = null, shownKeys = null) {
static async serialize(item, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
lodash.pull(fieldNames, 'sync_time');

52
lib/models/tag.js Normal file
View File

@ -0,0 +1,52 @@
import { BaseModel } from 'lib/base-model.js';
import { Database } from 'lib/database.js';
import { BaseItem } from 'lib/models/base-item.js';
import lodash from 'lodash';
class Tag extends BaseItem {
static tableName() {
return 'tags';
}
static itemType() {
return BaseModel.MODEL_TYPE_TAG;
}
static async serialize(item, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
fieldNames.push(() => {
});
lodash.pull(fieldNames, 'sync_time');
return super.serialize(item, 'tag', fieldNames);
}
static tagNoteIds(tagId) {
return this.db().selectAll('SELECT note_id FROM note_tags WHERE tag_id = ?', [tagId]);
}
static async addNote(tagId, noteId) {
let hasIt = await this.hasNote(tagId, noteId);
if (hasIt) return;
let query = Database.insertQuery('note_tags', {
tag_id: tagId,
note_id: noteId,
});
return this.db().exec(query);
}
static async hasNote(tagId, noteId) {
let r = await this.db().selectOne('SELECT note_id FROM note_tags WHERE tag_id = ? AND note_id = ? LIMIT 1', [tagId, noteId]);
return !!r;
}
static removeNote(tagId, noteId) {
return this.db().exec('DELETE FROM note_tags WHERE tag_id = ? AND note_id = ?', [tagId, noteId]);
}
}
export { Tag };

8
lib/synchronizer.js
View File

@ -145,7 +145,7 @@ class Synchronizer {
if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path));
let remote = await this.api().stat(path);
let content = ItemClass.serialize(local);
let content = await ItemClass.serialize(local);
let action = null;
let updateSyncTimeOnly = true;
let reason = '';
@ -198,7 +198,7 @@ class Synchronizer {
if (remote) {
let remoteContent = await this.api().get(path);
local = BaseItem.unserialize(remoteContent);
local = await BaseItem.unserialize(remoteContent);
local.sync_time = time.unixMs();
await ItemClass.save(local, { autoTimestamp: false });
@ -219,7 +219,7 @@ class Synchronizer {
if (remote) {
let remoteContent = await this.api().get(path);
local = BaseItem.unserialize(remoteContent);
local = await BaseItem.unserialize(remoteContent);
local.sync_time = time.unixMs();
await ItemClass.save(local, { autoTimestamp: false });
@ -301,7 +301,7 @@ class Synchronizer {
this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path);
continue;
}
content = BaseItem.unserialize(content);
content = await BaseItem.unserialize(content);
let ItemClass = BaseItem.itemClass(content);
let newContent = Object.assign({}, content);