From f04c1c58f1ab3eb75c560b04519c6a714ca43689 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 2 Jul 2017 16:46:03 +0100 Subject: [PATCH] Import Evernote tags --- CliClient/app/file-api-test.js | 93 --------------------------------- CliClient/app/import-enex.js | 22 ++++++++ CliClient/app/main.js | 34 ++++++------ CliClient/tests/synchronizer.js | 2 +- CliClient/tests/test-utils.js | 4 +- lib/base-model.js | 4 ++ lib/database.js | 39 ++++++-------- lib/models/base-item.js | 6 +-- lib/models/folder.js | 2 +- lib/models/item-sync-time.js | 39 -------------- lib/models/note.js | 2 +- lib/models/resource.js | 2 +- lib/models/tag.js | 52 ++++++++++++++++++ lib/synchronizer.js | 8 +-- 14 files changed, 127 insertions(+), 182 deletions(-) delete mode 100644 CliClient/app/file-api-test.js delete mode 100644 lib/models/item-sync-time.js create mode 100644 lib/models/tag.js diff --git a/CliClient/app/file-api-test.js b/CliClient/app/file-api-test.js deleted file mode 100644 index 3359ba74f..000000000 --- a/CliClient/app/file-api-test.js +++ /dev/null @@ -1,93 +0,0 @@ -import { FileApi } from 'lib/file-api.js'; -import { FileApiDriverLocal } from 'lib/file-api-driver-local.js'; -import { Database } from 'lib/database.js'; -import { DatabaseDriverNode } from 'lib/database-driver-node.js'; -import { Log } from 'lib/log.js'; - -const fs = require('fs'); - -// let driver = new FileApiDriverLocal(); -// let api = new FileApi('/home/laurent/Temp/TestImport', driver); - -// api.list('/').then((items) => { -// console.info(items); -// }).then(() => { -// return api.get('un.txt'); -// }).then((content) => { -// console.info(content); -// }).then(() => { -// return api.mkdir('TESTING'); -// }).then(() => { -// return api.put('un.txt', 'testing change'); -// }).then(() => { -// return api.delete('deux.txt'); -// }).catch((error) => { -// console.error('ERROR', error); -// }); - -Log.setLevel(Log.LEVEL_DEBUG); - -let db = new Database(new DatabaseDriverNode()); -//db.setDebugMode(true); -db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => { - return db.selectAll('SELECT * FROM table_fields'); -}).then((rows) => { - -}); - - //'/home/laurent/Temp/TestImport' - - -// var sqlite3 = require('sqlite3').verbose(); -// var db = new sqlite3.Database(':memory:'); - -// db.run("CREATE TABLE lorem (info TEXT)", () => { -// db.exec('INSERT INTO lorem VALUES "un"', () => { -// db.exec('INSERT INTO lorem VALUES "deux"', () => { -// let st = db.prepare("SELECT rowid AS id, info FROM lorem", () => { -// st.get((error, row) => { -// console.info(row); -// }); -// }); -// }); -// }); -// }); - -// var stmt = db.prepare("INSERT INTO lorem VALUES (?)"); -// for (var i = 0; i < 10; i++) { -// stmt.run("Ipsum " + i); -// } -// stmt.finalize(); - -// let st = db.prepare("SELECT rowid AS id, info FROM lorem"); -// st.get({}, (row) => { -// console.info('xx',row); -// }); - - -// st.finalize(); - - -//db.serialize(function() { - // db.run("CREATE TABLE lorem (info TEXT)"); - - // var stmt = db.prepare("INSERT INTO lorem VALUES (?)"); - // for (var i = 0; i < 10; i++) { - // stmt.run("Ipsum " + i); - // } - // stmt.finalize(); - - // let st = db.prepare("SELECT rowid AS id, info FROM lorem"); - // st.get({}, (row) => { - // console.info('xx',row); - // }); - - - // st.finalize(); - - // db.each("SELECT rowid AS id, info FROM lorem", function(err, row) { - // console.log(row.id + ": " + row.info); - // }); -//}); - -//db.close(); \ No newline at end of file diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index 846be9539..190d267fb 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -4,6 +4,7 @@ import { promiseChain } from 'lib/promise-utils.js'; import { folderItemFilename } from 'lib/string-utils.js' import { BaseModel } from 'lib/base-model.js'; import { Note } from 'lib/models/note.js'; +import { Tag } from 'lib/models/tag.js'; import { Resource } from 'lib/models/resource.js'; import { Folder } from 'lib/models/folder.js'; import { enexXmlToMd } from './import-enex-md-gen.js'; @@ -74,6 +75,21 @@ async function saveNoteResources(note) { return resourcesCreated; } +async function saveNoteTags(note) { + let noteTagged = 0; + for (let i = 0; i < note.tags.length; i++) { + let tagTitle = note.tags[i]; + + let tag = await Tag.loadByTitle(tagTitle); + if (!tag) tag = await Tag.save({ title: tagTitle }); + + await Tag.addNote(tag.id, note.id); + + noteTagged++; + } + return noteTagged; +} + async function saveNoteToStorage(note, fuzzyMatching = false) { note = Note.filter(note); @@ -84,11 +100,15 @@ async function saveNoteToStorage(note, fuzzyMatching = false) { noteUpdated: false, noteSkipped: false, resourcesCreated: 0, + noteTagged: 0, }; let resourcesCreated = await saveNoteResources(note); result.resourcesCreated += resourcesCreated; + let noteTagged = await saveNoteTags(note); + result.noteTagged += noteTagged; + if (existingNote) { let diff = BaseModel.diffObjects(existingNote, note); delete diff.tags; @@ -128,6 +148,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) { updated: 0, skipped: 0, resourcesCreated: 0, + noteTagged: 0, }; let stream = fs.createReadStream(filePath); @@ -192,6 +213,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) { progressState.skipped++; } progressState.resourcesCreated += result.resourcesCreated; + progressState.noteTagged += result.noteTagged; importOptions.onProgress(progressState); }); }); diff --git a/CliClient/app/main.js b/CliClient/app/main.js index f70670139..594ec8744 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -153,33 +153,36 @@ commands.push({ commands.push({ usage: 'cat ', description: 'Displays the given item data.', - action: function(args, end) { - let title = args['title']; + action: async function(args, end) { + try { + let title = args['title']; - let promise = null; - if (!currentFolder) { - promise = Folder.loadByField('title', title); - } else { - promise = Note.loadFolderNoteByField(currentFolder.id, 'title', title); - } + let item = null; + if (!currentFolder) { + item = await Folder.loadByField('title', title); + } else { + item = await Note.loadFolderNoteByField(currentFolder.id, 'title', title); + } - promise.then((item) => { if (!item) { this.log(_('No item with title "%s" found.', title)); end(); return; } + let content = null; if (!currentFolder) { - this.log(Folder.serialize(item)); + content = await Folder.serialize(item); } else { - this.log(Note.serialize(item)); + content = await Note.serialize(item); } - }).catch((error) => { + + this.log(content); + } catch(error) { this.log(error); - }).then(() => { - end(); - }); + } + + end(); }, autocomplete: autocompleteItems, }); @@ -467,6 +470,7 @@ commands.push({ if (progressState.updated) line.push(_('Updated: %d.', progressState.updated)); if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped)); if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated)); + if (progressState.notesTagged) line.push(_('Tagged: %d.', progressState.notesTagged)); redrawnCalled = true; vorpal.ui.redraw(line.join(' ')); }, diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 4fa82e41d..63fe94dd6 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -37,7 +37,7 @@ async function localItemsSameAsRemote(locals, expect) { expect(remote.updated_time).toBe(dbItem.updated_time); let remoteContent = await fileApi().get(path); - remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? Note.unserialize(remoteContent) : Folder.unserialize(remoteContent); + remoteContent = dbItem.type_ == BaseModel.MODEL_TYPE_NOTE ? await Note.unserialize(remoteContent) : await Folder.unserialize(remoteContent); expect(remoteContent.title).toBe(dbItem.title); } } catch (error) { diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index d0fd95c31..7a84ad134 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -48,7 +48,9 @@ function clearDatabase(id = null) { 'DELETE FROM changes', 'DELETE FROM notes', 'DELETE FROM folders', - 'DELETE FROM item_sync_times', + 'DELETE FROM resources', + 'DELETE FROM tags', + 'DELETE FROM note_tags', ]; return databases_[id].transactionExecBatch(queries); diff --git a/lib/base-model.js b/lib/base-model.js index 6b477ffc2..8d7df89b8 100644 --- a/lib/base-model.js +++ b/lib/base-model.js @@ -161,6 +161,10 @@ class BaseModel { return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]); } + static loadByTitle(fieldValue) { + return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `title` = ?', [fieldValue]); + } + static applyPatch(model, patch) { model = Object.assign({}, model); for (let n in patch) { diff --git a/lib/database.js b/lib/database.js index 5da9358a2..ec8654574 100644 --- a/lib/database.js +++ b/lib/database.js @@ -10,8 +10,8 @@ CREATE TABLE folders ( id TEXT PRIMARY KEY, parent_id TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "", - created_time INT NOT NULL DEFAULT 0, - updated_time INT NOT NULL DEFAULT 0, + created_time INT NOT NULL, + updated_time INT NOT NULL, sync_time INT NOT NULL DEFAULT 0 ); @@ -24,8 +24,8 @@ CREATE TABLE notes ( parent_id TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "", body TEXT NOT NULL DEFAULT "", - created_time INT NOT NULL DEFAULT 0, - updated_time INT NOT NULL DEFAULT 0, + created_time INT NOT NULL, + updated_time INT NOT NULL, sync_time INT NOT NULL DEFAULT 0, is_conflict INT NOT NULL DEFAULT 0, latitude NUMERIC NOT NULL DEFAULT 0, @@ -58,15 +58,15 @@ CREATE TABLE deleted_items ( CREATE TABLE tags ( id TEXT PRIMARY KEY, - title TEXT, - created_time INT, - updated_time INT + title TEXT NOT NULL DEFAULT "", + created_time INT NOT NULL, + updated_time INT NOT NULL ); CREATE TABLE note_tags ( id INTEGER PRIMARY KEY, - note_id TEXT, - tag_id TEXT + note_id TEXT NOT NULL, + tag_id TEXT NOT NULL ); CREATE TABLE resources ( @@ -79,16 +79,6 @@ CREATE TABLE resources ( sync_time INT NOT NULL DEFAULT 0 ); -CREATE TABLE note_resources ( - id INTEGER PRIMARY KEY, - note_id TEXT, - resource_id TEXT -); - -CREATE TABLE version ( - version INT -); - CREATE TABLE changes ( id INTEGER PRIMARY KEY, \`type\` INT, @@ -111,10 +101,8 @@ CREATE TABLE table_fields ( field_default TEXT ); -CREATE TABLE item_sync_times ( - id INTEGER PRIMARY KEY, - item_id TEXT, - \`time\` INT +CREATE TABLE version ( + version INT ); INSERT INTO version (version) VALUES (1); @@ -208,6 +196,11 @@ class Database { } async exec(sql, params = null) { + if (typeof sql === 'object') { + params = sql.params; + sql = sql.sql; + } + let result = null; let waitTime = 50; let totalWaitTime = 0; diff --git a/lib/models/base-item.js b/lib/models/base-item.js index 7d21fcf06..38719424d 100644 --- a/lib/models/base-item.js +++ b/lib/models/base-item.js @@ -100,7 +100,7 @@ class BaseItem extends BaseModel { return propValue; } - static serialize(item, type = null, shownKeys = null) { + static async serialize(item, type = null, shownKeys = null) { item = this.filter(item); let output = []; @@ -118,7 +118,7 @@ class BaseItem extends BaseModel { return output.join("\n"); } - static unserialize(content) { + static async unserialize(content) { let lines = content.split("\n"); let output = {}; let state = 'readingProps'; @@ -156,7 +156,7 @@ class BaseItem extends BaseModel { for (let n in output) { if (!output.hasOwnProperty(n)) continue; - output[n] = this.unserialize_format(output.type_, n, output[n]); + output[n] = await this.unserialize_format(output.type_, n, output[n]); } return output; diff --git a/lib/models/folder.js b/lib/models/folder.js index 7b025c019..4f26e1255 100644 --- a/lib/models/folder.js +++ b/lib/models/folder.js @@ -14,7 +14,7 @@ class Folder extends BaseItem { return 'folders'; } - static serialize(folder) { + static async serialize(folder) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); lodash.pull(fieldNames, 'parent_id', 'sync_time'); diff --git a/lib/models/item-sync-time.js b/lib/models/item-sync-time.js deleted file mode 100644 index 056ab0a54..000000000 --- a/lib/models/item-sync-time.js +++ /dev/null @@ -1,39 +0,0 @@ -import { BaseModel } from 'lib/base-model.js'; - -class ItemSyncTime extends BaseModel { - - static time(itemId) { - if (itemId in this.cache_) return Promise.resolve(this.cache_[itemId]); - - return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => { - this.cache_[itemId] = row ? row.time : 0; - return this.cache_[itemId]; - }); - } - - static setTime(itemId, time) { - return this.db().selectOne('SELECT * FROM item_sync_times WHERE item_id = ?', [itemId]).then((row) => { - let p = null; - if (row) { - p = this.db().exec('UPDATE item_sync_times SET `time` = ? WHERE item_id = ?', [time, itemId]); - } else { - p = this.db().exec('INSERT INTO item_sync_times (item_id, `time`) VALUES (?, ?)', [itemId, time]); - } - - return p.then(() => { - this.cache_[itemId] = time; - }); - }); - } - - static deleteTime(itemId) { - return this.db().exec('DELETE FROM item_sync_times WHERE item_id = ?', [itemId]).then(() => { - delete this.cache_[itemId]; - }); - } - -} - -ItemSyncTime.cache_ = {}; - -export { ItemSyncTime }; \ No newline at end of file diff --git a/lib/models/note.js b/lib/models/note.js index a94f974e1..18b4d4729 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -13,7 +13,7 @@ class Note extends BaseItem { return 'notes'; } - static serialize(note, type = null, shownKeys = null) { + static async serialize(note, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); lodash.pull(fieldNames, 'is_conflict', 'sync_time', 'body'); // Exclude 'body' since it's going to be added separately at the top of the note diff --git a/lib/models/resource.js b/lib/models/resource.js index 25719d408..70305de49 100644 --- a/lib/models/resource.js +++ b/lib/models/resource.js @@ -15,7 +15,7 @@ class Resource extends BaseItem { return BaseModel.MODEL_TYPE_RESOURCE; } - static serialize(item, type = null, shownKeys = null) { + static async serialize(item, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); lodash.pull(fieldNames, 'sync_time'); diff --git a/lib/models/tag.js b/lib/models/tag.js new file mode 100644 index 000000000..c9a642548 --- /dev/null +++ b/lib/models/tag.js @@ -0,0 +1,52 @@ +import { BaseModel } from 'lib/base-model.js'; +import { Database } from 'lib/database.js'; +import { BaseItem } from 'lib/models/base-item.js'; +import lodash from 'lodash'; + +class Tag extends BaseItem { + + static tableName() { + return 'tags'; + } + + static itemType() { + return BaseModel.MODEL_TYPE_TAG; + } + + static async serialize(item, type = null, shownKeys = null) { + let fieldNames = this.fieldNames(); + fieldNames.push('type_'); + fieldNames.push(() => { + + }); + lodash.pull(fieldNames, 'sync_time'); + return super.serialize(item, 'tag', fieldNames); + } + + static tagNoteIds(tagId) { + return this.db().selectAll('SELECT note_id FROM note_tags WHERE tag_id = ?', [tagId]); + } + + static async addNote(tagId, noteId) { + let hasIt = await this.hasNote(tagId, noteId); + if (hasIt) return; + + let query = Database.insertQuery('note_tags', { + tag_id: tagId, + note_id: noteId, + }); + return this.db().exec(query); + } + + static async hasNote(tagId, noteId) { + let r = await this.db().selectOne('SELECT note_id FROM note_tags WHERE tag_id = ? AND note_id = ? LIMIT 1', [tagId, noteId]); + return !!r; + } + + static removeNote(tagId, noteId) { + return this.db().exec('DELETE FROM note_tags WHERE tag_id = ? AND note_id = ?', [tagId, noteId]); + } + +} + +export { Tag }; \ No newline at end of file diff --git a/lib/synchronizer.js b/lib/synchronizer.js index 911504bcc..dff572dde 100644 --- a/lib/synchronizer.js +++ b/lib/synchronizer.js @@ -145,7 +145,7 @@ class Synchronizer { if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path)); let remote = await this.api().stat(path); - let content = ItemClass.serialize(local); + let content = await ItemClass.serialize(local); let action = null; let updateSyncTimeOnly = true; let reason = ''; @@ -198,7 +198,7 @@ class Synchronizer { if (remote) { let remoteContent = await this.api().get(path); - local = BaseItem.unserialize(remoteContent); + local = await BaseItem.unserialize(remoteContent); local.sync_time = time.unixMs(); await ItemClass.save(local, { autoTimestamp: false }); @@ -219,7 +219,7 @@ class Synchronizer { if (remote) { let remoteContent = await this.api().get(path); - local = BaseItem.unserialize(remoteContent); + local = await BaseItem.unserialize(remoteContent); local.sync_time = time.unixMs(); await ItemClass.save(local, { autoTimestamp: false }); @@ -301,7 +301,7 @@ class Synchronizer { this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); continue; } - content = BaseItem.unserialize(content); + content = await BaseItem.unserialize(content); let ItemClass = BaseItem.itemClass(content); let newContent = Object.assign({}, content);