You've already forked joplin
mirror of
https://github.com/laurent22/joplin.git
synced 2025-10-31 00:07:48 +02:00
sync
This commit is contained in:
Laurent Cozic
@@ -1,32 +1,141 @@
|
||||
import { time } from 'src/time-utils.js';
|
||||
import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js';
|
||||
import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient } from 'test-utils.js';
|
||||
import { createFoldersAndNotes } from 'test-data.js';
|
||||
import { Folder } from 'src/models/folder.js';
|
||||
import { Note } from 'src/models/note.js';
|
||||
import { BaseItem } from 'src/models/base-item.js';
|
||||
import { BaseModel } from 'src/base-model.js';
|
||||
|
||||
async function localItemsSameAsRemote(locals, expect) {
|
||||
try {
|
||||
for (let i = 0; i < locals.length; i++) {
|
||||
let dbItem = locals[i];
|
||||
let path = BaseItem.systemPath(dbItem);
|
||||
let remote = await fileApi().stat(path);
|
||||
|
||||
// console.info('=======================');
|
||||
// console.info(remote);
|
||||
// console.info(dbItem);
|
||||
// console.info('=======================');
|
||||
|
||||
expect(!!remote).toBe(true);
|
||||
expect(remote.updatedTime).toBe(dbItem.updated_time);
|
||||
|
||||
let remoteContent = await fileApi().get(path);
|
||||
remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.fromFriendlyString(remoteContent) : Folder.fromFriendlyString(remoteContent);
|
||||
expect(remoteContent.title).toBe(dbItem.title);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
describe('Synchronizer', function() {
|
||||
|
||||
beforeEach( async (done) => {
|
||||
await setupDatabaseAndSynchronizer();
|
||||
await setupDatabaseAndSynchronizer(1);
|
||||
await setupDatabaseAndSynchronizer(2);
|
||||
switchClient(1);
|
||||
done();
|
||||
});
|
||||
|
||||
it('should create remote items', async (done) => {
|
||||
let folder = await Folder.save({ title: "folder1" });
|
||||
await Note.save({ title: "un", parent_id: folder.id });
|
||||
// it('should create remote items', async (done) => {
|
||||
// let folder = await Folder.save({ title: "folder1" });
|
||||
// await Note.save({ title: "un", parent_id: folder.id });
|
||||
|
||||
// let all = await Folder.all(true);
|
||||
|
||||
// await synchronizer().start();
|
||||
|
||||
// await localItemsSameAsRemote(all, expect);
|
||||
|
||||
// done();
|
||||
// });
|
||||
|
||||
// it('should update remote item', async (done) => {
|
||||
// let folder = await Folder.save({ title: "folder1" });
|
||||
// let note = await Note.save({ title: "un", parent_id: folder.id });
|
||||
|
||||
// await sleep(1);
|
||||
|
||||
// await Note.save({ title: "un UPDATE", id: note.id });
|
||||
|
||||
// let all = await Folder.all(true);
|
||||
// await synchronizer().start();
|
||||
|
||||
// await localItemsSameAsRemote(all, expect);
|
||||
|
||||
// done();
|
||||
// });
|
||||
|
||||
// it('should create local items', async (done) => {
|
||||
// let folder = await Folder.save({ title: "folder1" });
|
||||
// await Note.save({ title: "un", parent_id: folder.id });
|
||||
// await synchronizer().start();
|
||||
// await clearDatabase();
|
||||
// await synchronizer().start();
|
||||
|
||||
// let all = await Folder.all(true);
|
||||
// await localItemsSameAsRemote(all, expect);
|
||||
|
||||
// done();
|
||||
// });
|
||||
|
||||
// it('should create same items on client 2', async (done) => {
|
||||
// let folder = await Folder.save({ title: "folder1" });
|
||||
// let note = await Note.save({ title: "un", parent_id: folder.id });
|
||||
// await synchronizer().start();
|
||||
|
||||
// await sleep(1);
|
||||
|
||||
// switchClient(2);
|
||||
|
||||
// await synchronizer().start();
|
||||
|
||||
// let folder2 = await Folder.load(folder.id);
|
||||
// let note2 = await Note.load(note.id);
|
||||
|
||||
// expect(!!folder2).toBe(true);
|
||||
// expect(!!note2).toBe(true);
|
||||
|
||||
// expect(folder.title).toBe(folder.title);
|
||||
// expect(folder.updated_time).toBe(folder.updated_time);
|
||||
|
||||
// expect(note.title).toBe(note.title);
|
||||
// expect(note.updated_time).toBe(note.updated_time);
|
||||
// expect(note.body).toBe(note.body);
|
||||
|
||||
// done();
|
||||
// });
|
||||
|
||||
it('should update local items', async (done) => {
|
||||
let folder1 = await Folder.save({ title: "folder1" });
|
||||
let note1 = await Note.save({ title: "un", parent_id: folder1.id });
|
||||
await synchronizer().start();
|
||||
|
||||
await sleep(1);
|
||||
|
||||
switchClient(2);
|
||||
|
||||
await synchronizer().start();
|
||||
|
||||
let note2 = await Note.load(note1.id);
|
||||
note2.title = "Updated on client 2";
|
||||
await Note.save(note2);
|
||||
|
||||
let all = await Folder.all(true);
|
||||
|
||||
await synchronizer().start();
|
||||
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
let dbItem = all[i];
|
||||
let path = BaseItem.systemPath(all[i]);
|
||||
let remote = await fileApi().stat(path);
|
||||
expect(!!remote).toBe(true);
|
||||
expect(remote.updatedTime).toBe(dbItem.updated_time);
|
||||
}
|
||||
switchClient(1);
|
||||
|
||||
await synchronizer().start();
|
||||
|
||||
note1 = await Note.load(note1.id);
|
||||
|
||||
expect(!!note1).toBe(true);
|
||||
expect(note1.title).toBe(note2.title);
|
||||
expect(note1.body).toBe(note2.body);
|
||||
|
||||
done();
|
||||
});
|
||||
|
||||
@@ -2,61 +2,95 @@ import fs from 'fs-extra';
|
||||
import { Database } from 'src/database.js';
|
||||
import { DatabaseDriverNode } from 'src/database-driver-node.js';
|
||||
import { BaseModel } from 'src/base-model.js';
|
||||
import { Folder } from 'src/models/folder.js';
|
||||
import { Note } from 'src/models/note.js';
|
||||
import { BaseItem } from 'src/models/base-item.js';
|
||||
import { Synchronizer } from 'src/synchronizer.js';
|
||||
import { FileApi } from 'src/file-api.js';
|
||||
import { FileApiDriverMemory } from 'src/file-api-driver-memory.js';
|
||||
|
||||
let database_ = null;
|
||||
let synchronizer_ = null;
|
||||
let databases_ = [];
|
||||
let synchronizers_ = [];
|
||||
let fileApi_ = null;
|
||||
let currentClient_ = 1;
|
||||
|
||||
function setupDatabase(done) {
|
||||
if (database_) {
|
||||
let queries = [
|
||||
'DELETE FROM changes',
|
||||
'DELETE FROM notes',
|
||||
'DELETE FROM folders',
|
||||
'DELETE FROM item_sync_times',
|
||||
];
|
||||
function sleep(n) {
|
||||
return new Promise((resolve, reject) => {
|
||||
setTimeout(() => {
|
||||
resolve();
|
||||
}, n * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
return database_.transactionExecBatch(queries).then(() => {
|
||||
if (done) done();
|
||||
});
|
||||
function switchClient(id) {
|
||||
currentClient_ = id;
|
||||
BaseModel.db_ = databases_[id];
|
||||
Folder.db_ = databases_[id];
|
||||
Note.db_ = databases_[id];
|
||||
BaseItem.db_ = databases_[id];
|
||||
}
|
||||
|
||||
function clearDatabase(id = null) {
|
||||
if (id === null) id = currentClient_;
|
||||
|
||||
let queries = [
|
||||
'DELETE FROM changes',
|
||||
'DELETE FROM notes',
|
||||
'DELETE FROM folders',
|
||||
'DELETE FROM item_sync_times',
|
||||
];
|
||||
|
||||
return databases_[id].transactionExecBatch(queries);
|
||||
}
|
||||
|
||||
function setupDatabase(id = null) {
|
||||
if (id === null) id = currentClient_;
|
||||
|
||||
if (databases_[id]) {
|
||||
return clearDatabase(id);
|
||||
}
|
||||
|
||||
const filePath = __dirname + '/data/test.sqlite';
|
||||
const filePath = __dirname + '/data/test-' + id + '.sqlite';
|
||||
return fs.unlink(filePath).catch(() => {
|
||||
// Don't care if the file doesn't exist
|
||||
}).then(() => {
|
||||
database_ = new Database(new DatabaseDriverNode());
|
||||
database_.setDebugEnabled(false);
|
||||
return database_.open({ name: filePath }).then(() => {
|
||||
BaseModel.db_ = database_;
|
||||
return setupDatabase(done);
|
||||
databases_[id] = new Database(new DatabaseDriverNode());
|
||||
databases_[id].setDebugEnabled(false);
|
||||
return databases_[id].open({ name: filePath }).then(() => {
|
||||
BaseModel.db_ = databases_[id];
|
||||
return setupDatabase(id);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function setupDatabaseAndSynchronizer() {
|
||||
await setupDatabase();
|
||||
async function setupDatabaseAndSynchronizer(id = null) {
|
||||
if (id === null) id = currentClient_;
|
||||
|
||||
if (!synchronizer_) {
|
||||
let fileDriver = new FileApiDriverMemory();
|
||||
fileApi_ = new FileApi('/root', fileDriver);
|
||||
synchronizer_ = new Synchronizer(db(), fileApi_);
|
||||
await setupDatabase(id);
|
||||
|
||||
if (!synchronizers_[id]) {
|
||||
synchronizers_[id] = new Synchronizer(db(id), fileApi());
|
||||
}
|
||||
|
||||
await fileApi().format();
|
||||
}
|
||||
|
||||
function db() {
|
||||
return database_;
|
||||
function db(id = null) {
|
||||
if (id === null) id = currentClient_;
|
||||
return databases_[id];
|
||||
}
|
||||
|
||||
function synchronizer() {
|
||||
return synchronizer_;
|
||||
function synchronizer(id = null) {
|
||||
if (id === null) id = currentClient_;
|
||||
console.info('SYNC', id);
|
||||
return synchronizers_[id];
|
||||
}
|
||||
|
||||
function fileApi() {
|
||||
if (fileApi_) return fileApi_;
|
||||
|
||||
fileApi_ = new FileApi('/root', new FileApiDriverMemory());
|
||||
return fileApi_;
|
||||
}
|
||||
|
||||
export { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi };
|
||||
export { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient };
|
||||
@@ -75,18 +75,11 @@ class Folder extends BaseItem {
|
||||
}
|
||||
|
||||
static async all(includeNotes = false) {
|
||||
let folders = await this.modelSelectAll('SELECT * FROM folders');
|
||||
let folders = await Folder.modelSelectAll('SELECT * FROM folders');
|
||||
if (!includeNotes) return folders;
|
||||
|
||||
let output = [];
|
||||
for (let i = 0; i < folders.length; i++) {
|
||||
let folder = folders[i];
|
||||
let notes = await Note.all(folder.id);
|
||||
output.push(folder);
|
||||
output = output.concat(notes);
|
||||
}
|
||||
|
||||
return output;
|
||||
let notes = await Note.modelSelectAll('SELECT * FROM notes');
|
||||
return folders.concat(notes);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,6 @@ class NoteFolderService extends BaseService {
|
||||
toSave.id = item.id;
|
||||
}
|
||||
|
||||
console.info(toSave);
|
||||
|
||||
return ItemClass.save(toSave, options).then((savedItem) => {
|
||||
output = Object.assign(item, savedItem);
|
||||
if (isNew && type == 'note') return Note.updateGeolocation(output.id);
|
||||
|
||||
@@ -106,10 +106,8 @@ class Synchronizer {
|
||||
dbItemToSyncItem(dbItem) {
|
||||
if (!dbItem) return null;
|
||||
|
||||
let itemType = BaseModel.identifyItemType(dbItem);
|
||||
|
||||
return {
|
||||
type: itemType == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
|
||||
type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
|
||||
path: Folder.systemPath(dbItem),
|
||||
syncTime: dbItem.sync_time,
|
||||
updatedTime: dbItem.updated_time,
|
||||
@@ -121,7 +119,7 @@ class Synchronizer {
|
||||
if (!remoteItem) return null;
|
||||
|
||||
return {
|
||||
type: remoteItem.content.type,
|
||||
type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
|
||||
path: remoteItem.path,
|
||||
syncTime: 0,
|
||||
updatedTime: remoteItem.updatedTime,
|
||||
@@ -175,8 +173,8 @@ class Synchronizer {
|
||||
if (this.itemIsStrictlyOlderThan(remote, local.updatedTime)) {
|
||||
action.type = 'update';
|
||||
action.dest = 'remote';
|
||||
action.reason = sprintf('Remote (%s) was modified after last sync of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
|
||||
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
|
||||
action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
|
||||
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) {
|
||||
action.type = 'conflict';
|
||||
action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).',
|
||||
moment.unix(remote.updatedTime).toISOString(),
|
||||
@@ -195,6 +193,10 @@ class Synchronizer {
|
||||
{ type: 'update', dest: 'local' },
|
||||
];
|
||||
}
|
||||
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updatedTime <= local.syncTime) {
|
||||
action.type = 'update';
|
||||
action.dest = 'local';
|
||||
action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString());
|
||||
} else {
|
||||
continue; // Neither local nor remote item have been changed recently
|
||||
}
|
||||
@@ -230,8 +232,11 @@ class Synchronizer {
|
||||
// modified since the last sync, it's been processed in the previous loop.
|
||||
// So throw an exception is this normally impossible condition happens anyway.
|
||||
// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
|
||||
if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) throw new Error('Remote cannot be newer than last sync time.');
|
||||
|
||||
if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
|
||||
console.error('Remote cannot be newer than last sync time', remote, local);
|
||||
throw new Error('Remote cannot be newer than last sync time');
|
||||
}
|
||||
|
||||
if (this.itemIsStrictlyNewerThan(remote, local.updatedTime)) {
|
||||
action.type = 'update';
|
||||
action.dest = 'local';
|
||||
@@ -285,53 +290,49 @@ class Synchronizer {
|
||||
if (action.type == 'create') {
|
||||
if (action.dest == 'remote') {
|
||||
let content = null;
|
||||
let dbItem = syncItem.dbItem;
|
||||
|
||||
if (syncItem.type == 'folder') {
|
||||
content = Folder.toFriendlyString(syncItem.dbItem);
|
||||
content = Folder.toFriendlyString(dbItem);
|
||||
} else {
|
||||
content = Note.toFriendlyString(syncItem.dbItem);
|
||||
content = Note.toFriendlyString(dbItem);
|
||||
}
|
||||
|
||||
return this.api().put(path, content).then(() => {
|
||||
return this.api().setTimestamp(path, syncItem.updatedTime);
|
||||
return this.api().setTimestamp(path, dbItem.updated_time);
|
||||
});
|
||||
|
||||
// TODO: save sync_time
|
||||
} else {
|
||||
let dbItem = syncItem.remoteItem.content;
|
||||
dbItem.sync_time = time.unix();
|
||||
dbItem.updated_time = dbItem.sync_time;
|
||||
dbItem.updated_time = action.remote.updatedTime;
|
||||
if (syncItem.type == 'folder') {
|
||||
return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
|
||||
} else {
|
||||
return Note.save(dbItem, { isNew: true, autoTimestamp: false });
|
||||
}
|
||||
|
||||
// TODO: save sync_time
|
||||
}
|
||||
}
|
||||
|
||||
if (action.type == 'update') {
|
||||
if (action.dest == 'remote') {
|
||||
// let content = null;
|
||||
|
||||
// if (syncItem.type == 'folder') {
|
||||
// content = Folder.toFriendlyString(syncItem.dbItem);
|
||||
// } else {
|
||||
// content = Note.toFriendlyString(syncItem.dbItem);
|
||||
// }
|
||||
|
||||
// return this.api().put(path, content).then(() => {
|
||||
// return this.api().setTimestamp(path, syncItem.updatedTime);
|
||||
// });
|
||||
let dbItem = syncItem.dbItem;
|
||||
let ItemClass = BaseItem.itemClass(dbItem);
|
||||
let content = ItemClass.toFriendlyString(dbItem);
|
||||
//console.info('PUT', content);
|
||||
return this.api().put(path, content).then(() => {
|
||||
return this.api().setTimestamp(path, dbItem.updated_time);
|
||||
}).then(() => {
|
||||
let toSave = { id: dbItem.id, sync_time: time.unix() };
|
||||
return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false });
|
||||
});
|
||||
} else {
|
||||
let dbItem = syncItem.remoteItem.content;
|
||||
let dbItem = Object.assign({}, syncItem.remoteItem.content);
|
||||
dbItem.sync_time = time.unix();
|
||||
dbItem.updated_time = dbItem.sync_time;
|
||||
return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
|
||||
// let dbItem = syncItem.remoteItem.content;
|
||||
// dbItem.sync_time = time.unix();
|
||||
// if (syncItem.type == 'folder') {
|
||||
// return Folder.save(dbItem, { isNew: true });
|
||||
// } else {
|
||||
// return Note.save(dbItem, { isNew: true });
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -346,12 +347,9 @@ class Synchronizer {
|
||||
let action = this.syncAction(localItem, remoteItem, []);
|
||||
await this.processSyncAction(action);
|
||||
|
||||
dbItem.sync_time = time.unix();
|
||||
if (localItem.type == 'folder') {
|
||||
return Folder.save(dbItem);
|
||||
} else {
|
||||
return Note.save(dbItem);
|
||||
}
|
||||
let toSave = Object.assign({}, dbItem);
|
||||
toSave.sync_time = time.unix();
|
||||
return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false });
|
||||
}
|
||||
|
||||
async processRemoteItem(remoteItem) {
|
||||
|
||||
Reference in New Issue
Block a user