From 24f61177d15d8a1175c1863dd12a29ad0f173720 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 16 Jul 2017 13:53:59 +0100 Subject: [PATCH] Improved handling of multiple sync targets --- CliClient/app/app.js | 4 +- CliClient/app/command-ls.js | 3 +- CliClient/app/fuzzing.js | 1 - CliClient/tests/synchronizer.js | 1 - CliClient/tests/test-utils.js | 2 +- ReactNativeClient/lib/base-model.js | 14 +- ReactNativeClient/lib/database.js | 20 ++- .../lib/file-api-driver-local.js | 8 + .../lib/file-api-driver-memory.js | 8 + .../lib/file-api-driver-onedrive.js | 8 + ReactNativeClient/lib/file-api.js | 4 + ReactNativeClient/lib/joplin-database.js | 32 ++-- ReactNativeClient/lib/models/base-item.js | 154 +++++++++++------- ReactNativeClient/lib/models/folder.js | 2 +- ReactNativeClient/lib/models/note-tag.js | 1 - ReactNativeClient/lib/models/note.js | 5 +- ReactNativeClient/lib/models/resource.js | 1 - ReactNativeClient/lib/models/tag.js | 1 - ReactNativeClient/lib/services/report.js | 3 +- ReactNativeClient/lib/synchronizer.js | 54 +++--- joplin.sublime-project | 1 + 21 files changed, 206 insertions(+), 121 deletions(-) diff --git a/CliClient/app/app.js b/CliClient/app/app.js index fa4cb5f41..77ea6913d 100644 --- a/CliClient/app/app.js +++ b/CliClient/app/app.js @@ -239,6 +239,8 @@ class Application { let fileApi = null; + // TODO: create file api based on syncTarget + if (syncTarget == 'onedrive') { const oneDriveApi = reg.oneDriveApi(); let driver = new FileApiDriverOneDrive(oneDriveApi); @@ -258,7 +260,7 @@ class Application { } else if (syncTarget == 'memory') { fileApi = new FileApi('joplin', new FileApiDriverMemory()); fileApi.setLogger(this.logger_); - } else if (syncTarget == 'local') { + } else if (syncTarget == 'file') { let syncDir = Setting.value('sync.local.path'); if (!syncDir) syncDir = Setting.value('profileDir') + '/sync'; this.vorpal().log(_('Synchronizing with directory "%s"', syncDir)); diff --git a/CliClient/app/command-ls.js b/CliClient/app/command-ls.js index 06eff99e8..c269ff558 100644 --- a/CliClient/app/command-ls.js +++ b/CliClient/app/command-ls.js @@ -25,7 +25,7 @@ class Command extends BaseCommand { ['-r, --reverse', 'Reverses the sorting order.'], ['-t, --type ', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'], ['-f, --format ', 'Either "text" or "json"'], - ['-l, --long', 'Use long list format. Format is NOTE_COUNT (for notebook), DATE, NEED_SYNC, TODO_CHECKED (for todos), TITLE'], + ['-l, --long', 'Use long list format. Format is NOTE_COUNT (for notebook), DATE, TODO_CHECKED (for todos), TITLE'], ]; } @@ -90,7 +90,6 @@ class Command extends BaseCommand { } row.push(time.unixMsToLocalDateTime(item.updated_time)); - row.push(item.updated_time > item.sync_time ? '*' : ' '); } let title = item.title + suffix; diff --git a/CliClient/app/fuzzing.js b/CliClient/app/fuzzing.js index 6e0c9267e..5395d97a5 100644 --- a/CliClient/app/fuzzing.js +++ b/CliClient/app/fuzzing.js @@ -208,7 +208,6 @@ function compareItems(item1, item2) { let output = []; for (let n in item1) { if (!item1.hasOwnProperty(n)) continue; - if (n == 'sync_time') continue; let p1 = item1[n]; let p2 = item2[n]; diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index c25bfff0a..f8f10e121 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -177,7 +177,6 @@ describe('Synchronizer', function() { let noteUpdatedFromRemote = await Note.load(note1.id); for (let n in noteUpdatedFromRemote) { if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue; - if (n == 'sync_time') continue; expect(noteUpdatedFromRemote[n]).toBe(note2[n], 'Property: ' + n); } diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index 53830f2c6..9f44b8270 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -91,7 +91,7 @@ function setupDatabase(id = null) { // Don't care if the file doesn't exist }).then(() => { databases_[id] = new JoplinDatabase(new DatabaseDriverNode()); - databases_[id].setLogger(logger); + //databases_[id].setLogger(logger); return databases_[id].open({ name: filePath }).then(() => { BaseModel.db_ = databases_[id]; return setupDatabase(id); diff --git a/ReactNativeClient/lib/base-model.js b/ReactNativeClient/lib/base-model.js index f24300474..a9aea35a1 100644 --- a/ReactNativeClient/lib/base-model.js +++ b/ReactNativeClient/lib/base-model.js @@ -49,8 +49,14 @@ class BaseModel { return fields.indexOf(name) >= 0; } - static fieldNames() { - return this.db().tableFieldNames(this.tableName()); + static fieldNames(withPrefix = false) { + let output = this.db().tableFieldNames(this.tableName()); + if (!withPrefix) return output; + let temp = []; + for (let i = 0; i < output.length; i++) { + temp.push(this.tableName() + '.' + output[i]); + } + return temp; } static fieldType(name) { @@ -228,6 +234,10 @@ class BaseModel { queries.push(saveQuery); + if (options.nextQueries && options.nextQueries.length) { + queries = queries.concat(options.nextQueries); + } + return this.db().transactionExecBatch(queries).then(() => { o = Object.assign({}, o); o.id = modelId; diff --git a/ReactNativeClient/lib/database.js b/ReactNativeClient/lib/database.js index 8d3b5519e..acee77920 100644 --- a/ReactNativeClient/lib/database.js +++ b/ReactNativeClient/lib/database.js @@ -40,7 +40,11 @@ class Database { escapeField(field) { if (field == '*') return '*'; - return '`' + field + '`'; + let p = field.split('.'); + if (p.length == 1) return '`' + field + '`'; + if (p.length == 2) return p[0] + '.`' + p[1] + '`'; + + throw new Error('Invalid field format: ' + field); } escapeFields(fields) { @@ -145,9 +149,23 @@ class Database { if (s == 'INTEGER') s = 'INT'; return this['TYPE_' + s]; } + if (type == 'syncTarget') { + if (s == 'memory') return 1; + if (s == 'file') return 2; + if (s == 'onedrive') return 3; + } throw new Error('Unknown enum type or value: ' + type + ', ' + s); } + static enumName(type, id) { + if (type == 'syncTarget') { + if (id === 1) return 'memory'; + if (id === 2) return 'file'; + if (id === 3) return 'onedrive'; + } + throw new Error('Unknown enum type or id: ' + type + ', ' + id); + } + static formatValue(type, value) { if (value === null || value === undefined) return null; if (type == this.TYPE_INT) return Number(value); diff --git a/ReactNativeClient/lib/file-api-driver-local.js b/ReactNativeClient/lib/file-api-driver-local.js index 0aeeee730..ceb488a08 100644 --- a/ReactNativeClient/lib/file-api-driver-local.js +++ b/ReactNativeClient/lib/file-api-driver-local.js @@ -5,6 +5,14 @@ import { time } from 'lib/time-utils.js'; class FileApiDriverLocal { + syncTargetId() { + return 2; + } + + syncTargetName() { + return 'file'; + } + fsErrorToJsError_(error) { let msg = error.toString(); let output = new Error(msg); diff --git a/ReactNativeClient/lib/file-api-driver-memory.js b/ReactNativeClient/lib/file-api-driver-memory.js index 47694cd7a..49148de0a 100644 --- a/ReactNativeClient/lib/file-api-driver-memory.js +++ b/ReactNativeClient/lib/file-api-driver-memory.js @@ -2,6 +2,14 @@ import { time } from 'lib/time-utils.js'; class FileApiDriverMemory { + syncTargetId() { + return 1; + } + + syncTargetName() { + return 'memory'; + } + constructor() { this.items_ = []; } diff --git a/ReactNativeClient/lib/file-api-driver-onedrive.js b/ReactNativeClient/lib/file-api-driver-onedrive.js index 394c09d3d..f625bc847 100644 --- a/ReactNativeClient/lib/file-api-driver-onedrive.js +++ b/ReactNativeClient/lib/file-api-driver-onedrive.js @@ -9,6 +9,14 @@ class FileApiDriverOneDrive { this.api_ = api; } + syncTargetId() { + return 3; + } + + syncTargetName() { + return 'onedrive'; + } + api() { return this.api_; } diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index 2b6f693ef..b4cf5ea56 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -9,6 +9,10 @@ class FileApi { this.logger_ = new Logger(); } + driver() { + return this.driver_; + } + setLogger(l) { this.logger_ = l; } diff --git a/ReactNativeClient/lib/joplin-database.js b/ReactNativeClient/lib/joplin-database.js index 94feb46f7..4db351972 100644 --- a/ReactNativeClient/lib/joplin-database.js +++ b/ReactNativeClient/lib/joplin-database.js @@ -6,16 +6,13 @@ import { Database } from 'lib/database.js' const structureSql = ` CREATE TABLE folders ( id TEXT PRIMARY KEY, - parent_id TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "", created_time INT NOT NULL, - updated_time INT NOT NULL, - sync_time INT NOT NULL DEFAULT 0 + updated_time INT NOT NULL ); CREATE INDEX folders_title ON folders (title); CREATE INDEX folders_updated_time ON folders (updated_time); -CREATE INDEX folders_sync_time ON folders (sync_time); CREATE TABLE notes ( id TEXT PRIMARY KEY, @@ -24,7 +21,6 @@ CREATE TABLE notes ( body TEXT NOT NULL DEFAULT "", created_time INT NOT NULL, updated_time INT NOT NULL, - sync_time INT NOT NULL DEFAULT 0, is_conflict INT NOT NULL DEFAULT 0, latitude NUMERIC NOT NULL DEFAULT 0, longitude NUMERIC NOT NULL DEFAULT 0, @@ -42,7 +38,6 @@ CREATE TABLE notes ( CREATE INDEX notes_title ON notes (title); CREATE INDEX notes_updated_time ON notes (updated_time); -CREATE INDEX notes_sync_time ON notes (sync_time); CREATE INDEX notes_is_conflict ON notes (is_conflict); CREATE INDEX notes_is_todo ON notes (is_todo); CREATE INDEX notes_order ON notes (\`order\`); @@ -58,27 +53,23 @@ CREATE TABLE tags ( id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT "", created_time INT NOT NULL, - updated_time INT NOT NULL, - sync_time INT NOT NULL DEFAULT 0 + updated_time INT NOT NULL ); CREATE INDEX tags_title ON tags (title); CREATE INDEX tags_updated_time ON tags (updated_time); -CREATE INDEX tags_sync_time ON tags (sync_time); CREATE TABLE note_tags ( id TEXT PRIMARY KEY, note_id TEXT NOT NULL, tag_id TEXT NOT NULL, created_time INT NOT NULL, - updated_time INT NOT NULL, - sync_time INT NOT NULL DEFAULT 0 + updated_time INT NOT NULL ); CREATE INDEX note_tags_note_id ON note_tags (note_id); CREATE INDEX note_tags_tag_id ON note_tags (tag_id); CREATE INDEX note_tags_updated_time ON note_tags (updated_time); -CREATE INDEX note_tags_sync_time ON note_tags (sync_time); CREATE TABLE resources ( id TEXT PRIMARY KEY, @@ -86,13 +77,11 @@ CREATE TABLE resources ( mime TEXT NOT NULL, filename TEXT NOT NULL DEFAULT "", created_time INT NOT NULL, - updated_time INT NOT NULL, - sync_time INT NOT NULL DEFAULT 0 + updated_time INT NOT NULL ); CREATE INDEX resources_title ON resources (title); CREATE INDEX resources_updated_time ON resources (updated_time); -CREATE INDEX resources_sync_time ON resources (sync_time); CREATE TABLE settings ( \`key\` TEXT PRIMARY KEY, @@ -112,6 +101,19 @@ CREATE TABLE version ( version INT NOT NULL ); +CREATE TABLE sync_items ( + id INTEGER PRIMARY KEY, + sync_target INT NOT NULL, + sync_time INT NOT NULL DEFAULT 0, + item_type INT NOT NULL, + item_id TEXT NOT NULL +); + +CREATE INDEX sync_items_sync_time ON sync_items (sync_time); +CREATE INDEX sync_items_sync_target ON sync_items (sync_target); +CREATE INDEX sync_items_item_type ON sync_items (item_type); +CREATE INDEX sync_items_item_id ON sync_items (item_id); + INSERT INTO version (version) VALUES (1); `; diff --git a/ReactNativeClient/lib/models/base-item.js b/ReactNativeClient/lib/models/base-item.js index 7d931aeba..6619a1cb8 100644 --- a/ReactNativeClient/lib/models/base-item.js +++ b/ReactNativeClient/lib/models/base-item.js @@ -1,6 +1,7 @@ import { BaseModel } from 'lib/base-model.js'; import { Database } from 'lib/database.js'; import { time } from 'lib/time-utils.js'; +import { sprintf } from 'sprintf-js'; import moment from 'moment'; class BaseItem extends BaseModel { @@ -31,44 +32,14 @@ class BaseItem extends BaseModel { throw new Error('Invalid class name: ' + name); } - static async stats() { - let output = { - items: {}, - total: {}, - }; - - let itemCount = 0; - let syncedCount = 0; - for (let i = 0; i < BaseItem.syncItemDefinitions_.length; i++) { - let d = BaseItem.syncItemDefinitions_[i]; - let ItemClass = this.getClass(d.className); - let o = { - total: await ItemClass.count(), - synced: await ItemClass.syncedCount(), - }; - output.items[d.className] = o; - itemCount += o.total; - syncedCount += o.synced; - } - - output.total = { - total: itemCount, - synced: syncedCount, - }; - - output.toDelete = { - total: await this.deletedItemCount(), - }; - - return output; - } - static async syncedCount() { - const ItemClass = this.itemClass(this.modelType()); - let sql = 'SELECT count(*) as total FROM `' + ItemClass.tableName() + '` WHERE updated_time <= sync_time'; - if (this.modelType() == BaseModel.TYPE_NOTE) sql += ' AND is_conflict = 0'; - const r = await this.db().selectOne(sql); - return r.total; + // TODO + return 0; + // const ItemClass = this.itemClass(this.modelType()); + // let sql = 'SELECT count(*) as total FROM `' + ItemClass.tableName() + '` WHERE updated_time <= sync_time'; + // if (this.modelType() == BaseModel.TYPE_NOTE) sql += ' AND is_conflict = 0'; + // const r = await this.db().selectOne(sql); + // return r.total; } static systemPath(itemOrId) { @@ -92,13 +63,9 @@ class BaseItem extends BaseModel { } // Returns the IDs of the items that have been synced at least once - static async syncedItems() { - let folders = await this.getClass('Folder').modelSelectAll('SELECT id FROM folders WHERE sync_time > 0'); - let notes = await this.getClass('Note').modelSelectAll('SELECT id FROM notes WHERE is_conflict = 0 AND sync_time > 0'); - let resources = await this.getClass('Resource').modelSelectAll('SELECT id FROM resources WHERE sync_time > 0'); - let tags = await this.getClass('Tag').modelSelectAll('SELECT id FROM tags WHERE sync_time > 0'); - let noteTags = await this.getClass('NoteTag').modelSelectAll('SELECT id FROM note_tags WHERE sync_time > 0'); - return folders.concat(notes).concat(resources).concat(tags).concat(noteTags); + static async syncedItems(syncTarget) { + if (!syncTarget) throw new Error('No syncTarget specified'); + return await this.db().selectAll('SELECT item_id, item_type FROM sync_items WHERE sync_time > 0 AND sync_target = ?', [syncTarget]); } static pathToId(path) { @@ -136,14 +103,6 @@ class BaseItem extends BaseModel { static async delete(id, options = null) { return this.batchDelete([id], options); - // let trackDeleted = true; - // if (options && options.trackDeleted !== null && options.trackDeleted !== undefined) trackDeleted = options.trackDeleted; - - // await super.delete(id, options); - - // if (trackDeleted) { - // await this.db().exec('INSERT INTO deleted_items (item_type, item_id, deleted_time) VALUES (?, ?, ?)', [this.modelType(), id, time.unixMs()]); - // } } static async batchDelete(ids, options = null) { @@ -289,21 +248,54 @@ class BaseItem extends BaseModel { return output; } - static async itemsThatNeedSync(limit = 100) { - let items = await this.getClass('Folder').modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit); - if (items.length) return { hasMore: true, items: items }; + static async itemsThatNeedSync(syncTarget, limit = 100) { + const classNames = this.syncItemClassNames(); - items = await this.getClass('Resource').modelSelectAll('SELECT * FROM resources WHERE sync_time < updated_time LIMIT ' + limit); - if (items.length) return { hasMore: true, items: items }; + for (let i = 0; i < classNames.length; i++) { + const className = classNames[i]; + const ItemClass = this.getClass(className); + const fieldNames = ItemClass.fieldNames(true); + fieldNames.push('sync_time'); - items = await this.getClass('Note').modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time AND is_conflict = 0 LIMIT ' + limit); - if (items.length) return { hasMore: true, items: items }; + let sql = sprintf(` + SELECT %s FROM %s + LEFT JOIN sync_items t ON t.item_id = %s.id + WHERE t.id IS NULL OR t.sync_time < %s.updated_time + LIMIT %d + `, + this.db().escapeFields(fieldNames), + this.db().escapeField(ItemClass.tableName()), + this.db().escapeField(ItemClass.tableName()), + this.db().escapeField(ItemClass.tableName()), + limit); - items = await this.getClass('Tag').modelSelectAll('SELECT * FROM tags WHERE sync_time < updated_time LIMIT ' + limit); - if (items.length) return { hasMore: true, items: items }; + const items = await ItemClass.modelSelectAll(sql); - items = await this.getClass('NoteTag').modelSelectAll('SELECT * FROM note_tags WHERE sync_time < updated_time LIMIT ' + limit); - return { hasMore: items.length >= limit, items: items }; + if (i >= classNames.length - 1) { + return { hasMore: items.length >= limit, items: items }; + } else { + if (items.length) return { hasMore: true, items: items }; + } + } + + throw new Error('Unreachable'); + + //return this.modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit); + + // let items = await this.getClass('Folder').modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit); + // if (items.length) return { hasMore: true, items: items }; + + // items = await this.getClass('Resource').modelSelectAll('SELECT * FROM resources WHERE sync_time < updated_time LIMIT ' + limit); + // if (items.length) return { hasMore: true, items: items }; + + // items = await this.getClass('Note').modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time AND is_conflict = 0 LIMIT ' + limit); + // if (items.length) return { hasMore: true, items: items }; + + // items = await this.getClass('Tag').modelSelectAll('SELECT * FROM tags WHERE sync_time < updated_time LIMIT ' + limit); + // if (items.length) return { hasMore: true, items: items }; + + // items = await this.getClass('NoteTag').modelSelectAll('SELECT * FROM note_tags WHERE sync_time < updated_time LIMIT ' + limit); + // return { hasMore: items.length >= limit, items: items }; } static syncItemClassNames() { @@ -319,6 +311,42 @@ class BaseItem extends BaseModel { throw new Error('Invalid type: ' + type); } + static updateSyncTimeQueries(syncTarget, item, syncTime) { + const itemType = item.type_; + const itemId = item.id; + if (!itemType || !itemId || syncTime === undefined) throw new Error('Invalid parameters in updateSyncTimeQueries()'); + + return [ + { + sql: 'DELETE FROM sync_items WHERE sync_target = ? AND item_type = ? AND item_id = ?', + params: [syncTarget, itemType, itemId], + }, + { + sql: 'INSERT INTO sync_items (sync_target, item_type, item_id, sync_time) VALUES (?, ?, ?, ?)', + params: [syncTarget, itemType, itemId, syncTime], + } + ]; + } + + static async saveSyncTime(syncTarget, item, syncTime) { + const queries = this.updateSyncTimeQueries(syncTarget, item, syncTime); + return this.db().transactionExecBatch(queries); + } + + static async deleteOrphanSyncItems() { + const classNames = this.syncItemClassNames(); + + let queries = []; + for (let i = 0; i < classNames.length; i++) { + const className = classNames[i]; + const ItemClass = this.getClass(className); + + queries.push('DELETE FROM sync_items WHERE item_type = ' + ItemClass.modelType() + ' AND item_id NOT IN (SELECT id FROM ' + ItemClass.tableName() + ')'); + } + + await this.db().transactionExecBatch(queries); + } + } // Also update: diff --git a/ReactNativeClient/lib/models/folder.js b/ReactNativeClient/lib/models/folder.js index 9ed11e493..866a8d918 100644 --- a/ReactNativeClient/lib/models/folder.js +++ b/ReactNativeClient/lib/models/folder.js @@ -19,7 +19,7 @@ class Folder extends BaseItem { static async serialize(folder) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); - lodash.pull(fieldNames, 'parent_id', 'sync_time'); + lodash.pull(fieldNames, 'parent_id'); return super.serialize(folder, 'folder', fieldNames); } diff --git a/ReactNativeClient/lib/models/note-tag.js b/ReactNativeClient/lib/models/note-tag.js index 542001fd9..e9cc4c612 100644 --- a/ReactNativeClient/lib/models/note-tag.js +++ b/ReactNativeClient/lib/models/note-tag.js @@ -15,7 +15,6 @@ class NoteTag extends BaseItem { static async serialize(item, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); - lodash.pull(fieldNames, 'sync_time'); return super.serialize(item, 'note_tag', fieldNames); } diff --git a/ReactNativeClient/lib/models/note.js b/ReactNativeClient/lib/models/note.js index d9d7c579f..7a5eff10b 100644 --- a/ReactNativeClient/lib/models/note.js +++ b/ReactNativeClient/lib/models/note.js @@ -18,8 +18,6 @@ class Note extends BaseItem { static async serialize(note, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); - //lodash.pull(fieldNames, 'is_conflict', 'sync_time'); - lodash.pull(fieldNames, 'sync_time'); return super.serialize(note, 'note', fieldNames); } @@ -68,7 +66,7 @@ class Note extends BaseItem { } static previewFields() { - return ['id', 'title', 'body', 'is_todo', 'todo_completed', 'parent_id', 'updated_time', 'sync_time']; + return ['id', 'title', 'body', 'is_todo', 'todo_completed', 'parent_id', 'updated_time']; } static previewFieldsSql() { @@ -212,7 +210,6 @@ class Note extends BaseItem { let newNote = Object.assign({}, originalNote); delete newNote.id; - newNote.sync_time = 0; for (let n in changes) { if (!changes.hasOwnProperty(n)) continue; diff --git a/ReactNativeClient/lib/models/resource.js b/ReactNativeClient/lib/models/resource.js index 940d52427..90dbddb27 100644 --- a/ReactNativeClient/lib/models/resource.js +++ b/ReactNativeClient/lib/models/resource.js @@ -23,7 +23,6 @@ class Resource extends BaseItem { static async serialize(item, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); - lodash.pull(fieldNames, 'sync_time'); return super.serialize(item, 'resource', fieldNames); } diff --git a/ReactNativeClient/lib/models/tag.js b/ReactNativeClient/lib/models/tag.js index c02469c13..81c10ad56 100644 --- a/ReactNativeClient/lib/models/tag.js +++ b/ReactNativeClient/lib/models/tag.js @@ -18,7 +18,6 @@ class Tag extends BaseItem { static async serialize(item, type = null, shownKeys = null) { let fieldNames = this.fieldNames(); fieldNames.push('type_'); - lodash.pull(fieldNames, 'sync_time'); return super.serialize(item, 'tag', fieldNames); } diff --git a/ReactNativeClient/lib/services/report.js b/ReactNativeClient/lib/services/report.js index 7917b325f..c65482c83 100644 --- a/ReactNativeClient/lib/services/report.js +++ b/ReactNativeClient/lib/services/report.js @@ -19,7 +19,8 @@ class ReportService { let ItemClass = BaseItem.getClass(d.className); let o = { total: await ItemClass.count(), - synced: await ItemClass.syncedCount(), + // synced: await ItemClass.syncedCount(), // TODO + synced: 0, }; output.items[d.className] = o; itemCount += o.total; diff --git a/ReactNativeClient/lib/synchronizer.js b/ReactNativeClient/lib/synchronizer.js index 661d66bc4..e6ba3ccf0 100644 --- a/ReactNativeClient/lib/synchronizer.js +++ b/ReactNativeClient/lib/synchronizer.js @@ -54,6 +54,7 @@ class Synchronizer { if (report.deleteLocal) lines.push(_('Deleted local items: %d.', report.deleteLocal)); if (report.deleteRemote) lines.push(_('Deleted remote items: %d.', report.deleteRemote)); if (report.state) lines.push(_('State: %s.', report.state.replace(/_/g, ' '))); + if (report.errors && report.errors.length) lines.push(_('Last error: %s (stacktrace in log).', report.errors[report.errors.length-1].message)); return lines; } @@ -139,6 +140,8 @@ class Synchronizer { this.onProgress_ = options.onProgress ? options.onProgress : function(o) {}; this.progressReport_ = { errors: [] }; + const syncTargetId = this.api().driver().syncTargetId(); + if (this.state() != 'idle') { this.logger().warn('Synchronization is already in progress. State: ' + this.state()); return; @@ -156,7 +159,7 @@ class Synchronizer { this.state_ = 'in_progress'; - this.logSyncOperation('starting', null, null, 'Starting synchronization... [' + synchronizationId + ']'); + this.logSyncOperation('starting', null, null, 'Starting synchronization to ' + this.api().driver().syncTargetName() + ' (' + syncTargetId + ')... [' + synchronizationId + ']'); try { await this.api().mkdir(this.syncDirName_); @@ -166,7 +169,7 @@ class Synchronizer { while (true) { if (this.cancelling()) break; - let result = await BaseItem.itemsThatNeedSync(); + let result = await BaseItem.itemsThatNeedSync(syncTargetId); let locals = result.items; for (let i = 0; i < locals.length; i++) { @@ -209,7 +212,6 @@ class Synchronizer { this.logSyncOperation(action, local, remote, reason); - if (local.type_ == BaseModel.TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) { let remoteContentPath = this.resourceDirName_ + '/' + local.id; let resourceContent = await Resource.content(local); @@ -236,8 +238,8 @@ class Synchronizer { await this.api().setTimestamp(path, local.updated_time); if (this.randomFailure(options, 1)) return; - - await ItemClass.save({ id: local.id, sync_time: time.unixMs(), type_: local.type_ }, { autoTimestamp: false }); + + await ItemClass.saveSyncTime(syncTargetId, local, time.unixMs()); } else if (action == 'itemConflict') { @@ -245,8 +247,8 @@ class Synchronizer { let remoteContent = await this.api().get(path); local = await BaseItem.unserialize(remoteContent); - local.sync_time = time.unixMs(); - await ItemClass.save(local, { autoTimestamp: false }); + const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs()); + await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries }); } else { await ItemClass.delete(local.id); } @@ -266,8 +268,8 @@ class Synchronizer { let remoteContent = await this.api().get(path); local = await BaseItem.unserialize(remoteContent); - local.sync_time = time.unixMs(); - await ItemClass.save(local, { autoTimestamp: false }); + const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs()); + await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries }); } else { await ItemClass.delete(local.id); } @@ -346,10 +348,10 @@ class Synchronizer { let ItemClass = BaseItem.itemClass(content); let newContent = Object.assign({}, content); - newContent.sync_time = time.unixMs(); let options = { autoTimestamp: false, applyMetadataChanges: true, + nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, newContent, time.unixMs()), }; if (action == 'createLocal') options.isNew = true; @@ -381,37 +383,39 @@ class Synchronizer { let localFoldersToDelete = []; if (!this.cancelling()) { - let items = await BaseItem.syncedItems(); - for (let i = 0; i < items.length; i++) { + let syncItems = await BaseItem.syncedItems(syncTargetId); + for (let i = 0; i < syncItems.length; i++) { if (this.cancelling()) break; - let item = items[i]; - if (remoteIds.indexOf(item.id) < 0) { - if (item.type_ == Folder.modelType()) { - localFoldersToDelete.push(item); + let syncItem = syncItems[i]; + if (remoteIds.indexOf(syncItem.item_id) < 0) { + if (syncItem.item_type == Folder.modelType()) { + localFoldersToDelete.push(syncItem); continue; } - this.logSyncOperation('deleteLocal', { id: item.id }, null, 'remote has been deleted'); + this.logSyncOperation('deleteLocal', { id: syncItem.item_id }, null, 'remote has been deleted'); - let ItemClass = BaseItem.itemClass(item); - await ItemClass.delete(item.id, { trackDeleted: false }); + let ItemClass = BaseItem.itemClass(syncItem.item_type); + await ItemClass.delete(syncItem.item_id, { trackDeleted: false }); } } } if (!this.cancelling()) { for (let i = 0; i < localFoldersToDelete.length; i++) { - const folder = localFoldersToDelete[i]; - const noteIds = await Folder.noteIds(folder.id); + const syncItem = localFoldersToDelete[i]; + const noteIds = await Folder.noteIds(syncItem.item_id); if (noteIds.length) { // CONFLICT - await Folder.markNotesAsConflict(folder.id); - await Folder.delete(folder.id, { deleteChildren: false }); - } else { - await Folder.delete(folder.id); + await Folder.markNotesAsConflict(syncItem.item_id); } + await Folder.delete(syncItem.item_id, { deleteChildren: false }); } } + + if (!this.cancelling()) { + await BaseItem.deleteOrphanSyncItems(); + } } catch (error) { this.logger().error(error); this.progressReport_.errors.push(error); diff --git a/joplin.sublime-project b/joplin.sublime-project index 7a1d4a40c..b32f8b7ab 100755 --- a/joplin.sublime-project +++ b/joplin.sublime-project @@ -20,6 +20,7 @@ "ReactNativeClient/android/local.properties", "ReactNativeClient/ios", "_vieux", + "tests/logs" ], "file_exclude_patterns": [ "*.map",