diff --git a/CliClient/app/command-sync.js b/CliClient/app/command-sync.js index d35465c4c..7dd7b9f1c 100644 --- a/CliClient/app/command-sync.js +++ b/CliClient/app/command-sync.js @@ -7,6 +7,7 @@ import { vorpalUtils } from './vorpal-utils.js'; import { Synchronizer } from 'lib/synchronizer.js'; const locker = require('proper-lockfile'); const fs = require('fs-extra'); +const osTmpdir = require('os-tmpdir'); class Command extends BaseCommand { @@ -34,7 +35,7 @@ class Command extends BaseCommand { static lockFile(filePath) { return new Promise((resolve, reject) => { - locker.lock(filePath, (error, release) => { + locker.lock(filePath, { stale: 1000 * 60 * 5 }, (error, release) => { if (error) { reject(error); return; @@ -61,7 +62,7 @@ class Command extends BaseCommand { async action(args) { this.releaseLockFn_ = null; - const lockFilePath = Setting.value('tempDir') + '/synclock'; + const lockFilePath = osTmpdir() + '/synclock'; if (!await fs.pathExists(lockFilePath)) await fs.writeFile(lockFilePath, 'synclock'); if (await Command.isLocked(lockFilePath)) throw new Error(_('Synchronisation is already in progress.')); @@ -71,7 +72,7 @@ class Command extends BaseCommand { try { this.syncTarget_ = Setting.value('sync.target'); if (args.options.target) this.syncTarget_ = args.options.target; - + let syncInitOptions = {}; if (args.options['filesystem-path']) syncInitOptions['sync.filesystem.path'] = args.options['filesystem-path']; @@ -100,6 +101,7 @@ class Command extends BaseCommand { options.context = context; let newContext = await sync.start(options); Setting.setValue('sync.context', JSON.stringify(newContext)); + vorpalUtils.redrawDone(); await app().refreshCurrentFolder(); diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index 8f3d71005..6a9ab7a8c 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -97,8 +97,8 @@ async function saveNoteResources(note) { let existingResource = await Resource.load(toSave.id); if (existingResource) continue; - await Resource.save(toSave, { isNew: true }); await filePutContents(Resource.fullPath(toSave), resource.data) + await Resource.save(toSave, { isNew: true }); resourcesCreated++; } return resourcesCreated; diff --git a/CliClient/locales/en_GB.po b/CliClient/locales/en_GB.po index 7041f0b0a..61499ce22 100644 --- a/CliClient/locales/en_GB.po +++ b/CliClient/locales/en_GB.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 23:34+0100\n" +"POT-Creation-Date: 2017-07-19 20:00+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -287,40 +287,40 @@ msgstr "" msgid "Displays summary about the notes and notebooks." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26 msgid "Synchronizes with remote storage." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31 msgid "Sync to provided target (defaults to sync.target config value)" msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32 msgid "For \"filesystem\" target only: Path to sync to." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69 msgid "Synchronisation is already in progress." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #, javascript-format msgid "Synchronization target: %s" msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98 msgid "Cannot initialize synchronizer." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100 msgid "Starting synchronization..." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112 msgid "Done." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "" diff --git a/CliClient/locales/fr_FR.po b/CliClient/locales/fr_FR.po index b929ad09d..6c8132399 100644 --- a/CliClient/locales/fr_FR.po +++ b/CliClient/locales/fr_FR.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 23:18+0100\n" +"POT-Creation-Date: 2017-07-19 19:53+0100\n" "PO-Revision-Date: 2017-07-18 13:27+0100\n" "Last-Translator: \n" "Language-Team: \n" @@ -310,42 +310,42 @@ msgstr "Assigner la valeur [value] à la propriété de la donnée msgid "Displays summary about the notes and notebooks." msgstr "Afficher un résumé des notes et carnets." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26 msgid "Synchronizes with remote storage." msgstr "Synchroniser les notes et carnets." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31 msgid "Sync to provided target (defaults to sync.target config value)" msgstr "" "Synchroniser avec la cible donnée (par défaut, la valeur de configuration " "`sync.target`)." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32 msgid "For \"filesystem\" target only: Path to sync to." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69 msgid "Synchronisation is already in progress." msgstr "Synchronisation est déjà en cours." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #, javascript-format msgid "Synchronization target: %s" msgstr "Cible de la synchronisation : %s" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98 msgid "Cannot initialize synchronizer." msgstr "Impossible d'initialiser le synchroniseur." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100 msgid "Starting synchronization..." msgstr "Commencement de la synchronisation..." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112 msgid "Done." msgstr "Terminé." -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "Annulation..." diff --git a/CliClient/locales/joplin.pot b/CliClient/locales/joplin.pot index 7041f0b0a..61499ce22 100644 --- a/CliClient/locales/joplin.pot +++ b/CliClient/locales/joplin.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 23:34+0100\n" +"POT-Creation-Date: 2017-07-19 20:00+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -287,40 +287,40 @@ msgstr "" msgid "Displays summary about the notes and notebooks." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26 msgid "Synchronizes with remote storage." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31 msgid "Sync to provided target (defaults to sync.target config value)" msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32 msgid "For \"filesystem\" target only: Path to sync to." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69 msgid "Synchronisation is already in progress." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #, javascript-format msgid "Synchronization target: %s" msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98 msgid "Cannot initialize synchronizer." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100 msgid "Starting synchronization..." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112 msgid "Done." msgstr "" -#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 +#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "" diff --git a/CliClient/package.json b/CliClient/package.json index 8482f0363..122b485ca 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -7,7 +7,7 @@ "url": "https://github.com/laurent22/joplin" }, "url": "git://github.com/laurent22/joplin.git", - "version": "0.8.58", + "version": "0.8.62", "bin": { "joplin": "./main_launcher.js" }, @@ -24,6 +24,7 @@ "moment": "^2.18.1", "moment-timezone": "^0.5.13", "node-fetch": "^1.7.1", + "os-tmpdir": "^1.0.2", "promise": "^7.1.1", "proper-lockfile": "^2.0.1", "query-string": "4.3.4", diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 3e727a645..924baa48c 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -6,6 +6,7 @@ import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, import { Folder } from 'lib/models/folder.js'; import { Note } from 'lib/models/note.js'; import { Tag } from 'lib/models/tag.js'; +import { Database } from 'lib/database.js'; import { Setting } from 'lib/models/setting.js'; import { BaseItem } from 'lib/models/base-item.js'; import { BaseModel } from 'lib/base-model.js'; @@ -16,6 +17,8 @@ process.on('unhandledRejection', (reason, p) => { jasmine.DEFAULT_TIMEOUT_INTERVAL = 9000; // The first test is slow because the database needs to be built +const syncTargetId = Database.enumId('syncTarget', 'memory'); + async function allItems() { let folders = await Folder.all(); let notes = await Note.all(); @@ -237,7 +240,7 @@ describe('Synchronizer', function() { expect(files.length).toBe(1); expect(files[0].path).toBe(Folder.systemPath(folder1)); - let deletedItems = await BaseItem.deletedItems(); + let deletedItems = await BaseItem.deletedItems(syncTargetId); expect(deletedItems.length).toBe(0); done(); @@ -259,7 +262,7 @@ describe('Synchronizer', function() { await synchronizer().start(); let items = await allItems(); expect(items.length).toBe(1); - let deletedItems = await BaseItem.deletedItems(); + let deletedItems = await BaseItem.deletedItems(syncTargetId); expect(deletedItems.length).toBe(0); done(); @@ -565,7 +568,7 @@ describe('Synchronizer', function() { await synchronizer().start(); await Note.save({ id: n1.id, is_conflict: 1 }); await Note.delete(n1.id); - const deletedItems = await BaseItem.deletedItems(); + const deletedItems = await BaseItem.deletedItems(syncTargetId); expect(deletedItems.length).toBe(0); diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index 0e83a0f4f..0229e7c99 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -72,6 +72,9 @@ function clearDatabase(id = null) { 'DELETE FROM resources', 'DELETE FROM tags', 'DELETE FROM note_tags', + + 'DELETE FROM deleted_items', + 'DELETE FROM sync_items', ]; return databases_[id].transactionExecBatch(queries); diff --git a/CliClient/yarn.lock b/CliClient/yarn.lock index 809d16514..b736b7953 100644 --- a/CliClient/yarn.lock +++ b/CliClient/yarn.lock @@ -666,7 +666,7 @@ babel-register@^6.24.1: mkdirp "^0.5.1" source-map-support "^0.4.2" -babel-runtime@^6.18.0, babel-runtime@^6.22.0, babel-runtime@^6.23.0: +babel-runtime@^6.18.0, babel-runtime@^6.22.0: version "6.23.0" resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b" dependencies: @@ -1641,7 +1641,7 @@ os-homedir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" -os-tmpdir@^1.0.0, os-tmpdir@^1.0.1: +os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" diff --git a/ReactNativeClient/android/app/build.gradle b/ReactNativeClient/android/app/build.gradle index 6a4db22b6..c6baf35e9 100644 --- a/ReactNativeClient/android/app/build.gradle +++ b/ReactNativeClient/android/app/build.gradle @@ -90,8 +90,8 @@ android { applicationId "net.cozic.joplin" minSdkVersion 16 targetSdkVersion 22 - versionCode 21 - versionName "0.9.8" + versionCode 23 + versionName "0.9.10" ndk { abiFilters "armeabi-v7a", "x86" } diff --git a/ReactNativeClient/lib/base-model.js b/ReactNativeClient/lib/base-model.js index df3356f6b..24bbebda7 100644 --- a/ReactNativeClient/lib/base-model.js +++ b/ReactNativeClient/lib/base-model.js @@ -52,10 +52,13 @@ class BaseModel { static fieldNames(withPrefix = false) { let output = this.db().tableFieldNames(this.tableName()); if (!withPrefix) return output; + + let p = withPrefix === true ? this.tableName() : withPrefix; let temp = []; for (let i = 0; i < output.length; i++) { - temp.push(this.tableName() + '.' + output[i]); + temp.push(p + '.' + output[i]); } + return temp; } diff --git a/ReactNativeClient/lib/components/side-menu-content.js b/ReactNativeClient/lib/components/side-menu-content.js index ea9465aa6..aab86f28c 100644 --- a/ReactNativeClient/lib/components/side-menu-content.js +++ b/ReactNativeClient/lib/components/side-menu-content.js @@ -73,7 +73,7 @@ class SideMenuContentComponent extends Component { sync.cancel(); } else { if (reg.oneDriveApi().auth()) { - sync.start(); + reg.scheduleSync(1); } else { this.props.dispatch({ type: 'SIDE_MENU_CLOSE' }); diff --git a/ReactNativeClient/lib/database.js b/ReactNativeClient/lib/database.js index bdb6e00cd..e1d6a4997 100644 --- a/ReactNativeClient/lib/database.js +++ b/ReactNativeClient/lib/database.js @@ -166,6 +166,11 @@ class Database { throw new Error('Unknown enum type or id: ' + type + ', ' + id); } + static enumIds(type) { + if (type == 'syncTarget') return [1,2,3]; + throw new Error('Unknown enum type: ' + type); + } + static formatValue(type, value) { if (value === null || value === undefined) return null; if (type == this.TYPE_INT) return Number(value); @@ -182,7 +187,8 @@ class Database { var line = lines[i]; if (line == '') continue; if (line.substr(0, 2) == "--") continue; - statement += line; + statement += line.trim(); + if (line[line.length - 1] == ',') statement += ' '; if (line[line.length - 1] == ';') { output.push(statement); statement = ''; diff --git a/ReactNativeClient/lib/joplin-database.js b/ReactNativeClient/lib/joplin-database.js index 1e54c8e94..d779794c0 100644 --- a/ReactNativeClient/lib/joplin-database.js +++ b/ReactNativeClient/lib/joplin-database.js @@ -42,13 +42,6 @@ CREATE INDEX notes_is_conflict ON notes (is_conflict); CREATE INDEX notes_is_todo ON notes (is_todo); CREATE INDEX notes_order ON notes (\`order\`); -CREATE TABLE deleted_items ( - id INTEGER PRIMARY KEY, - item_type INT NOT NULL, - item_id TEXT NOT NULL, - deleted_time INT NOT NULL -); - CREATE TABLE tags ( id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT "", @@ -97,10 +90,6 @@ CREATE TABLE table_fields ( field_default TEXT ); -CREATE TABLE version ( - version INT NOT NULL -); - CREATE TABLE sync_items ( id INTEGER PRIMARY KEY, sync_target INT NOT NULL, @@ -114,6 +103,17 @@ CREATE INDEX sync_items_sync_target ON sync_items (sync_target); CREATE INDEX sync_items_item_type ON sync_items (item_type); CREATE INDEX sync_items_item_id ON sync_items (item_id); +CREATE TABLE deleted_items ( + id INTEGER PRIMARY KEY, + item_type INT NOT NULL, + item_id TEXT NOT NULL, + deleted_time INT NOT NULL +); + +CREATE TABLE version ( + version INT NOT NULL +); + INSERT INTO version (version) VALUES (1); `; @@ -187,28 +187,59 @@ class JoplinDatabase extends Database { }); } + async upgradeDatabase(fromVersion) { + // INSTRUCTIONS TO UPGRADE THE DATABASE: + // + // 1. Add the new version number to the existingDatabaseVersions array + // 2. Add the upgrade logic to the "switch (targetVersion)" statement below + + const existingDatabaseVersions = [1, 2]; + + let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion); + if (currentVersionIndex == existingDatabaseVersions.length - 1) return false; + + while (currentVersionIndex < existingDatabaseVersions.length - 1) { + const targetVersion = existingDatabaseVersions[currentVersionIndex + 1]; + this.logger().info("Converting database to version " + targetVersion); + + let queries = []; + + if (targetVersion == 2) { + const newTableSql = ` + CREATE TABLE deleted_items ( + id INTEGER PRIMARY KEY, + item_type INT NOT NULL, + item_id TEXT NOT NULL, + deleted_time INT NOT NULL, + sync_target INT NOT NULL + ); + `; + + queries.push({ sql: 'DROP TABLE deleted_items' }); + queries.push({ sql: this.sqlStringToLines(newTableSql)[0] }); + queries.push({ sql: "CREATE INDEX deleted_items_sync_target ON deleted_items (sync_target)" }); + } + + queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] }); + await this.transactionExecBatch(queries); + + currentVersionIndex++; + } + + return true; + } + async initialize() { this.logger().info('Checking for database schema update...'); for (let initLoopCount = 1; initLoopCount <= 2; initLoopCount++) { try { - // await this.exec('DROP TABLE folders'); - // await this.exec('DROP TABLE notes'); - // await this.exec('DROP TABLE deleted_items'); - // await this.exec('DROP TABLE tags'); - // await this.exec('DROP TABLE note_tags'); - // await this.exec('DROP TABLE resources'); - // await this.exec('DROP TABLE settings'); - // await this.exec('DROP TABLE table_fields'); - // await this.exec('DROP TABLE version'); - // await this.exec('DROP TABLE sync_items'); - let row = await this.selectOne('SELECT * FROM version LIMIT 1'); - this.logger().info('Current database version', row); + let currentVersion = row.version; + this.logger().info('Current database version', currentVersion); - // TODO: version update logic - // TODO: only do this if db has been updated: - // return this.refreshTableFields(); + const upgraded = await this.upgradeDatabase(currentVersion); + if (upgraded) await this.refreshTableFields(); } catch (error) { if (error && error.code != 0 && error.code != 'SQLITE_ERROR') throw this.sqliteErrorToJsError(error); diff --git a/ReactNativeClient/lib/models/base-item.js b/ReactNativeClient/lib/models/base-item.js index a88a47519..22abbd979 100644 --- a/ReactNativeClient/lib/models/base-item.js +++ b/ReactNativeClient/lib/models/base-item.js @@ -131,31 +131,36 @@ class BaseItem extends BaseModel { await super.batchDelete(ids, options); if (trackDeleted) { + const syncTargetIds = Database.enumIds('syncTarget'); let queries = []; let now = time.unixMs(); for (let i = 0; i < ids.length; i++) { if (conflictNoteIds.indexOf(ids[i]) >= 0) continue; - queries.push({ - sql: 'INSERT INTO deleted_items (item_type, item_id, deleted_time) VALUES (?, ?, ?)', - params: [this.modelType(), ids[i], now], - }); + // For each deleted item, for each sync target, we need to add an entry in deleted_items. + // That way, each target can later delete the remote item. + for (let j = 0; j < syncTargetIds.length; j++) { + queries.push({ + sql: 'INSERT INTO deleted_items (item_type, item_id, deleted_time, sync_target) VALUES (?, ?, ?, ?)', + params: [this.modelType(), ids[i], now, syncTargetIds[j]], + }); + } } await this.db().transactionExecBatch(queries); } } - static deletedItems() { - return this.db().selectAll('SELECT * FROM deleted_items'); + static deletedItems(syncTarget) { + return this.db().selectAll('SELECT * FROM deleted_items WHERE sync_target = ?', [syncTarget]); } - static async deletedItemCount() { - let r = await this.db().selectOne('SELECT count(*) as total FROM deleted_items'); + static async deletedItemCount(syncTarget) { + let r = await this.db().selectOne('SELECT count(*) as total FROM deleted_items WHERE sync_target = ?', [syncTarget]); return r['total']; } - static remoteDeletedItem(itemId) { - return this.db().exec('DELETE FROM deleted_items WHERE item_id = ?', [itemId]); + static remoteDeletedItem(syncTarget, itemId) { + return this.db().exec('DELETE FROM deleted_items WHERE item_id = ? AND sync_target = ?', [itemId, syncTarget]); } static serialize_format(propName, propValue) { @@ -275,33 +280,131 @@ class BaseItem extends BaseModel { for (let i = 0; i < classNames.length; i++) { const className = classNames[i]; const ItemClass = this.getClass(className); - const fieldNames = ItemClass.fieldNames(true); - fieldNames.push('sync_time'); + let fieldNames = ItemClass.fieldNames('items'); + + // // NEVER SYNCED: + // 'SELECT * FROM [ITEMS] WHERE id NOT INT (SELECT item_id FROM sync_items WHERE sync_target = ?)' + + // // CHANGED: + // 'SELECT * FROM [ITEMS] items JOIN sync_items s ON s.item_id = items.id WHERE sync_target = ? AND' let extraWhere = className == 'Note' ? 'AND is_conflict = 0' : ''; + // First get all the items that have never been synced under this sync target + let sql = sprintf(` - SELECT %s FROM %s - LEFT JOIN sync_items t ON t.item_id = %s.id - WHERE - (t.id IS NULL OR t.sync_time < %s.updated_time) - %s + SELECT %s + FROM %s items + WHERE id NOT IN ( + SELECT item_id FROM sync_items WHERE sync_target = %d + ) + %s LIMIT %d `, this.db().escapeFields(fieldNames), this.db().escapeField(ItemClass.tableName()), - this.db().escapeField(ItemClass.tableName()), - this.db().escapeField(ItemClass.tableName()), + Number(syncTarget), extraWhere, limit); - const items = await ItemClass.modelSelectAll(sql); + let neverSyncedItem = await ItemClass.modelSelectAll(sql); + //for (let i = 0; i < neverSyncedItem.length; i++) neverSyncedItem[i].sync_time = 0; + + // console.info(sql); + // console.info('NEVER', neverSyncedItem); + + // Secondly get the items that have been synced under this sync target but that have been changed since then + + const newLimit = limit - neverSyncedItem.length; + + let changedItems = []; + + if (newLimit > 0) { + fieldNames.push('sync_time'); + + let sql = sprintf(` + SELECT %s FROM %s items + JOIN sync_items s ON s.item_id = items.id + WHERE sync_target = %d + AND s.sync_time < items.updated_time + %s + LIMIT %d + `, + this.db().escapeFields(fieldNames), + this.db().escapeField(ItemClass.tableName()), + Number(syncTarget), + extraWhere, + newLimit); + + changedItems = await ItemClass.modelSelectAll(sql); + } + + // console.info('CHANGED', changedItems); + + const items = neverSyncedItem.concat(changedItems); if (i >= classNames.length - 1) { return { hasMore: items.length >= limit, items: items }; } else { if (items.length) return { hasMore: true, items: items }; } + + + + + //let extraWhere = className == 'Note' ? 'AND is_conflict = 0' : ''; + + // First get all the items that have never been synced under this sync target + + // let sql = sprintf(` + // SELECT %s FROM %s items + // LEFT JOIN sync_items t ON t.item_id = items.id + // WHERE (t.id IS NULL OR t.sync_target != %d) %s + // LIMIT %d + // `, + // this.db().escapeFields(fieldNames), + // this.db().escapeField(ItemClass.tableName()), + // Number(syncTarget), + // extraWhere, + // limit); + + // let neverSyncedItem = await ItemClass.modelSelectAll(sql); + // for (let i = 0; i < neverSyncedItem.length; i++) neverSyncedItem[i].sync_time = 0; + + // console.info(sql); + // console.info('NEVER', neverSyncedItem); + + // // Secondly get the items that have been synced under this sync target but that have been changed since then + + // const newLimit = limit - neverSyncedItem.length; + + // let changedItems = []; + + // if (newLimit > 0) { + // let sql = sprintf(` + // SELECT %s FROM %s items + // LEFT JOIN sync_items t ON t.item_id = items.id + // WHERE (t.sync_time < items.updated_time AND t.sync_target = %d) %s + // LIMIT %d + // `, + // this.db().escapeFields(fieldNames), + // this.db().escapeField(ItemClass.tableName()), + // Number(syncTarget), + // extraWhere, + // newLimit); + + // changedItems = await ItemClass.modelSelectAll(sql); + // } + + // console.info('CHANGED', changedItems); + + // const items = neverSyncedItem.concat(changedItems); + + // if (i >= classNames.length - 1) { + // return { hasMore: items.length >= limit, items: items }; + // } else { + // if (items.length) return { hasMore: true, items: items }; + // } } throw new Error('Unreachable'); diff --git a/ReactNativeClient/lib/registry.js b/ReactNativeClient/lib/registry.js index 8c3d3a0e2..02c7141f6 100644 --- a/ReactNativeClient/lib/registry.js +++ b/ReactNativeClient/lib/registry.js @@ -74,7 +74,9 @@ reg.synchronizer = async () => { return reg.synchronizer_; } -reg.scheduleSync = async () => { +reg.scheduleSync = async (delay = null) => { + if (delay === null) delay = 1000 * 10; + if (reg.scheduleSyncId_) { clearTimeout(reg.scheduleSyncId_); reg.scheduleSyncId_ = null; @@ -92,8 +94,12 @@ reg.scheduleSync = async () => { } const sync = await reg.synchronizer(); - sync.start(); - }, 1000 * 10); + + let context = Setting.value('sync.context'); + context = context ? JSON.parse(context) : {}; + let newContext = await sync.start({ context: context }); + Setting.setValue('sync.context', JSON.stringify(newContext)); + }, delay); } reg.setDb = (v) => { diff --git a/ReactNativeClient/lib/services/report.js b/ReactNativeClient/lib/services/report.js index 41af3b9d1..adbf17423 100644 --- a/ReactNativeClient/lib/services/report.js +++ b/ReactNativeClient/lib/services/report.js @@ -34,7 +34,7 @@ class ReportService { }; output.toDelete = { - total: await BaseItem.deletedItemCount(), + total: await BaseItem.deletedItemCount(syncTarget), }; output.conflicted = { diff --git a/ReactNativeClient/lib/synchronizer.js b/ReactNativeClient/lib/synchronizer.js index 92e7bbf06..5fbeebbf8 100644 --- a/ReactNativeClient/lib/synchronizer.js +++ b/ReactNativeClient/lib/synchronizer.js @@ -202,7 +202,7 @@ class Synchronizer { let content = await ItemClass.serialize(local); let action = null; let updateSyncTimeOnly = true; - let reason = ''; + let reason = ''; if (!remote) { if (!local.sync_time) { @@ -230,7 +230,14 @@ class Synchronizer { if (local.type_ == BaseModel.TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) { let remoteContentPath = this.resourceDirName_ + '/' + local.id; - let resourceContent = await Resource.content(local); + let resourceContent = ''; + try { + resourceContent = await Resource.content(local); + } catch (error) { + error.message = 'Cannot read resource content: ' + local.id + ': ' + error.message; + this.logger().error(error); + this.progressReport_.errors.push(error); + } await this.api().put(remoteContentPath, resourceContent); } @@ -302,7 +309,7 @@ class Synchronizer { // Delete the remote items that have been deleted locally. // ------------------------------------------------------------------------ - let deletedItems = await BaseItem.deletedItems(); + let deletedItems = await BaseItem.deletedItems(syncTargetId); for (let i = 0; i < deletedItems.length; i++) { if (this.cancelling()) break; @@ -311,7 +318,7 @@ class Synchronizer { this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted'); await this.api().delete(path); if (this.randomFailure(options, 3)) return; - await BaseItem.remoteDeletedItem(item.item_id); + await BaseItem.remoteDeletedItem(syncTargetId, item.item_id); } // ------------------------------------------------------------------------