1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
This commit is contained in:
Laurent Cozic 2017-06-15 00:14:15 +01:00
parent d8f71e89df
commit efe7d0a45a
16 changed files with 584 additions and 152 deletions

View File

@ -14,6 +14,195 @@ import { uuid } from 'src/uuid.js';
import { sprintf } from 'sprintf-js';
import { _ } from 'src/locale.js';
import { NoteFolderService } from 'src/services/note-folder-service.js';
let db = new Database(new DatabaseDriverNode());
db.setDebugEnabled(false);
// function whilePromise(callback) {
// let isDone = false;
// function done() {
// isDone = true;
// }
// let iterationDone = false;
// let p = callback(done).then(() => {
// iterationDone = true;
// });
// let iid = setInterval(() => {
// if (iterationDone) {
// if (isDone) {
// clearInterval(iid);
// return;
// }
// iterationDone = false;
// callback(done).then(() => {
// iterationDone = true;
// });
// }
// }, 100);
// }
// function myPromise() {
// return new Promise((resolve, reject) => {
// setTimeout(() => {
// resolve();
// }, 500);
// });
// }
// let counter = 0;
// whilePromise((done) => {
// return myPromise().then(() => {
// counter++;
// console.info(counter);
// if (counter == 5) {
// done();
// }
// });
// });
let fileDriver = new FileApiDriverLocal();
let fileApi = new FileApi('/home/laurent/Temp/TestImport', fileDriver);
let synchronizer = new Synchronizer(db, fileApi);
function clearDatabase() {
let queries = [
'DELETE FROM changes',
'DELETE FROM notes',
'DELETE FROM folders',
'DELETE FROM item_sync_times',
];
return db.transactionExecBatch(queries);
}
function createRemoteItems() {
let a = fileApi;
return Promise.all([a.mkdir('test1'), a.mkdir('test2'), a.mkdir('test3')]).then(() => {
return Promise.all([
a.put('test1/un', 'test1_un'),
a.put('test1/deux', 'test1_deux'),
a.put('test2/trois', 'test2_trois'),
a.put('test3/quatre', 'test3_quatre'),
a.put('test3/cinq', 'test3_cinq'),
a.put('test3/six', 'test3_six'),
]);
});
}
function createLocalItems() {
return Folder.save({ title: "folder1" }).then((f) => {
return Promise.all([
Note.save({ title: "un", parent_id: f.id }),
Note.save({ title: "deux", parent_id: f.id }),
Note.save({ title: "trois", parent_id: f.id }),
Note.save({ title: "quatre", parent_id: f.id }),
]);
}).then(() => {
return Folder.save({ title: "folder2" })
}).then((f) => {
return Promise.all([
Note.save({ title: "cinq", parent_id: f.id }),
]);
}).then(() => {
return Folder.save({ title: "folder3" })
}).then(() => {
return Folder.save({ title: "folder4" })
}).then((f) => {
return Promise.all([
Note.save({ title: "six", parent_id: f.id }),
Note.save({ title: "sept", parent_id: f.id }),
Note.save({ title: "huit", parent_id: f.id }),
]);
});
}
db.open({ name: '/home/laurent/Temp/test-sync.sqlite3' }).then(() => {
BaseModel.db_ = db;
return clearDatabase().then(createLocalItems);
}).then(() => {
return synchronizer.start();
}).catch((error) => {
console.error(error);
});
// let fileDriver = new FileApiDriverMemory();
// let fileApi = new FileApi('/root', fileDriver);
// let synchronizer = new Synchronizer(db, fileApi);
// import { ItemSyncTime } from 'src/models/item-sync-time.js';

View File

@ -2,7 +2,7 @@ require('app-module-path').addPath(__dirname);
import { uuid } from 'src/uuid.js';
import moment from 'moment';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import { WebApi } from 'src/web-api.js'
import { folderItemFilename } from 'src/string-utils.js'
import jsSHA from "jssha";

View File

@ -1,9 +1,7 @@
import { time } from 'src/time-utils.js';
import { Note } from 'src/models/note.js';
import { Folder } from 'src/models/folder.js';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import { NoteFolderService } from 'src/services/note-folder-service.js';
import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js';
import { setupDatabaseAndSynchronizer } from 'test-utils.js';
import { createFoldersAndNotes } from 'test-data.js';
describe('NoteFolderServices', function() {
@ -11,57 +9,8 @@ describe('NoteFolderServices', function() {
setupDatabaseAndSynchronizer(done);
});
function createNotes(parentId, id = 1) {
let notes = [];
if (id === 1) {
notes.push({ parent_id: parentId, title: 'note one', body: 'content of note one' });
notes.push({ parent_id: parentId, title: 'note two', body: 'content of note two' });
} else {
throw new Error('Invalid ID: ' + id);
}
let output = [];
let chain = [];
for (let i = 0; i < notes.length; i++) {
chain.push(() => {
return Note.save(notes[i]).then((note) => {
output.push(note);
return output;
});
});
}
return promiseChain(chain, []);
}
function createFolders(id = 1) {
let folders = [];
if (id === 1) {
folders.push({ title: 'myfolder1' });
folders.push({ title: 'myfolder2' });
folders.push({ title: 'myfolder3' });
} else {
throw new Error('Invalid ID: ' + id);
}
let output = [];
let chain = [];
for (let i = 0; i < folders.length; i++) {
chain.push(() => {
return Folder.save(folders[i]).then((folder) => {
output.push(folder);
return output;
});
});
}
return promiseChain(chain, []);
}
it('should retrieve sync items', function(done) {
createFolders().then((folders) => {
return createNotes(folders[0].id);
}).then(() => {
createFoldersAndNotes().then(() => {
return NoteFolderService.itemsThatNeedSync().then((context) => {
expect(context.items.length).toBe(2);
expect(context.hasMore).toBe(true);

View File

@ -1,7 +1,6 @@
import { time } from 'src/time-utils.js';
import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js';
describe('Synchronizer syncActions', function() {
import { createFoldersAndNotes } from 'test-data.js';
// Note: set 1 matches set 1 of createRemoteItems()
function createLocalItems(id, updatedTime, lastSyncTime) {
@ -34,6 +33,8 @@ describe('Synchronizer syncActions', function() {
}
}
describe('Synchronizer syncActions', function() {
beforeEach(function(done) {
setupDatabaseAndSynchronizer(done);
});
@ -155,8 +156,20 @@ describe('Synchronizer syncActions', function() {
});
});
it('should sync items', function(done) {
});
describe('Synchronizer start', function() {
beforeEach(function(done) {
setupDatabaseAndSynchronizer(done);
});
it('should create remote items', function(done) {
createFoldersAndNotes().then(() => {
return synchronizer().start();
}
}).then(() => {
done();
});
});

View File

@ -0,0 +1,58 @@
import { Note } from 'src/models/note.js';
import { Folder } from 'src/models/folder.js';
import { promiseChain } from 'src/promise-utils.js';
function createNotes(id = 1, parentId) {
let notes = [];
if (id === 1) {
notes.push({ parent_id: parentId, title: 'note one', body: 'content of note one' });
notes.push({ parent_id: parentId, title: 'note two', body: 'content of note two' });
} else {
throw new Error('Invalid ID: ' + id);
}
let output = [];
let chain = [];
for (let i = 0; i < notes.length; i++) {
chain.push(() => {
return Note.save(notes[i]).then((note) => {
output.push(note);
return output;
});
});
}
return promiseChain(chain, []);
}
function createFolders(id = 1) {
let folders = [];
if (id === 1) {
folders.push({ title: 'myfolder1' });
folders.push({ title: 'myfolder2' });
folders.push({ title: 'myfolder3' });
} else {
throw new Error('Invalid ID: ' + id);
}
let output = [];
let chain = [];
for (let i = 0; i < folders.length; i++) {
chain.push(() => {
return Folder.save(folders[i]).then((folder) => {
output.push(folder);
return output;
});
});
}
return promiseChain(chain, []);
}
function createFoldersAndNotes(id = 1) {
return createFolders(id).then((folders) => {
return createNotes(id, folders[0].id);
});
}
export { createNotes, createFolders, createFoldersAndNotes };

View File

@ -40,6 +40,12 @@ class BaseModel {
return this.db().tableFields(this.tableName());
}
static identifyItemType(item) {
if ('body' in item || ('parent_id' in item && !!item.parent_id)) return BaseModel.ITEM_TYPE_NOTE;
if ('sync_time' in item) return BaseModel.ITEM_TYPE_FOLDER;
throw new Error('Cannot identify item: ' + JSON.stringify(item));
}
static new() {
let fields = this.fields();
let output = {};
@ -154,7 +160,8 @@ class BaseModel {
queries.push(saveQuery);
if (options.trackChanges && this.trackChanges()) {
// TODO: DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED
if (0&& ptions.trackChanges && this.trackChanges()) {
// Cannot import this class the normal way due to cyclical dependencies between Change and BaseModel
// which are not handled by React Native.
const { Change } = require('src/models/change.js');

View File

@ -1,6 +1,6 @@
import { Log } from 'src/log.js';
import { uuid } from 'src/uuid.js';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import { _ } from 'src/locale.js'
const structureSql = `
@ -106,6 +106,7 @@ class Database {
this.initialized_ = false;
this.tableFields_ = null;
this.driver_ = driver;
this.inTransaction_ = false;
}
setDebugEnabled(v) {
@ -150,6 +151,33 @@ class Database {
}
transactionExecBatch(queries) {
if (queries.length <= 0) return Promise.resolve();
if (queries.length == 1) {
return this.exec(queries[0].sql, queries[0].params);
}
// There can be only one transaction running at a time so queue
// any new transaction here.
if (this.inTransaction_) {
return new Promise((resolve, reject) => {
let iid = setInterval(() => {
console.info('Waiting...');
if (!this.inTransaction_) {
console.info('OKKKKKKKKKKK');
clearInterval(iid);
this.transactionExecBatch(queries).then(() => {
resolve();
}).catch((error) => {
reject(error);
});
}
}, 100);
});
}
this.inTransaction_ = true;
queries.splice(0, 0, 'BEGIN TRANSACTION');
queries.push('COMMIT'); // Note: ROLLBACK is currently not supported
@ -161,7 +189,9 @@ class Database {
});
}
return promiseChain(chain);
return promiseChain(chain).then(() => {
this.inTransaction_ = false;
});
}
static enumId(type, s) {
@ -218,7 +248,11 @@ class Database {
logQuery(sql, params = null) {
if (!this.debugMode()) return;
if (params !== null) {
Log.debug('DB: ' + sql, params);
} else {
Log.debug('DB: ' + sql);
}
}
static insertQuery(tableName, data) {

View File

@ -1,6 +1,6 @@
import fs from 'fs';
import fse from 'fs-extra';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import moment from 'moment';
class FileApiDriverLocal {
@ -9,10 +9,14 @@ class FileApiDriverLocal {
return new Promise((resolve, reject) => {
fs.stat(path, (error, s) => {
if (error) {
if (error.code == 'ENOENT') {
resolve(null);
} else {
reject(error);
}
return;
}
resolve(s);
resolve(this.metadataFromStats_(path, s));
});
});
}
@ -61,8 +65,7 @@ class FileApiDriverLocal {
chain.push((output) => {
if (!output) output = [];
return this.stat(path + '/' + items[i]).then((stat) => {
let md = this.metadataFromStats_(items[i], stat);
output.push(md);
output.push(stat);
return output;
});
});
@ -82,7 +85,13 @@ class FileApiDriverLocal {
return new Promise((resolve, reject) => {
fs.readFile(path, 'utf8', (error, content) => {
if (error) {
if (error.code == 'ENOENT') {
// Return null in this case so that it's possible to get a file
// without checking if it exists first.
resolve(null);
} else {
reject(error);
}
return;
}
return resolve(content);

View File

@ -1,4 +1,4 @@
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
class FileApi {
@ -13,9 +13,19 @@ class FileApi {
return output;
}
list(path = '', recursive = false) {
listDirectories() {
return this.driver_.list(this.fullPath_('')).then((items) => {
let output = [];
for (let i = 0; i < items.length; i++) {
if (items[i].isDir) output.push(items[i]);
}
return output;
});
}
list(path = '', recursive = false, context = null) {
let fullPath = this.fullPath_(path);
return this.driver_.list(fullPath, recursive).then((items) => {
return this.driver_.list(fullPath).then((items) => {
if (recursive) {
let chain = [];
for (let i = 0; i < items.length; i++) {
@ -47,14 +57,26 @@ class FileApi {
}
mkdir(path) {
console.info('mkdir ' + path);
return this.driver_.mkdir(this.fullPath_(path));
}
stat(path) {
console.info('stat ' + path);
return this.driver_.stat(this.fullPath_(path)).then((output) => {
if (!output) return output;
output.path = path;
return output;
});
}
get(path) {
console.info('get ' + path);
return this.driver_.get(this.fullPath_(path));
}
put(path, content) {
console.info('put ' + path);
return this.driver_.put(this.fullPath_(path), content);
}

View File

@ -1,6 +1,6 @@
import { BaseModel } from 'src/base-model.js';
import { Log } from 'src/log.js';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import { Note } from 'src/models/note.js';
import { folderItemFilename } from 'src/string-utils.js'
import { _ } from 'src/locale.js';

View File

@ -1,10 +0,0 @@
function promiseChain(chain, defaultValue = null) {
let output = new Promise((resolve, reject) => { resolve(defaultValue); });
for (let i = 0; i < chain.length; i++) {
let f = chain[i];
output = output.then(f);
}
return output;
}
export { promiseChain }

View File

@ -0,0 +1,37 @@
function promiseChain(chain, defaultValue = null) {
let output = new Promise((resolve, reject) => { resolve(defaultValue); });
for (let i = 0; i < chain.length; i++) {
let f = chain[i];
output = output.then(f);
}
return output;
}
function promiseWhile(callback) {
let isDone = false;
function done() {
isDone = true;
}
let iterationDone = false;
let p = callback(done).then(() => {
iterationDone = true;
});
let iid = setInterval(() => {
if (iterationDone) {
if (isDone) {
clearInterval(iid);
return;
}
iterationDone = false;
callback(done).then(() => {
iterationDone = true;
});
}
}, 100);
}
export { promiseChain, promiseWhile }

View File

@ -74,33 +74,35 @@ class NoteFolderService extends BaseService {
}
static itemsThatNeedSync(context = null, limit = 100) {
let now = time.unix();
if (!context) {
context = {
hasMoreNotes: true,
hasMoreFolders: true,
hasMoreNotes: true,
noteOffset: 0,
folderOffset: 0,
hasMore: true,
items: [],
};
}
if (context.hasMoreNotes) {
return BaseModel.db().selectAll('SELECT * FROM notes WHERE sync_time < ? LIMIT ' + limit + ' OFFSET ' + context.noteOffset, [now]).then((items) => {
context.items = items;
context.hasMoreNotes = items.length >= limit;
context.noteOffset += items.length;
return context;
context.folderOffset = 0;
context.noteOffset = 0;
// Process folder first, then notes so that folders are created before
// adding notes to them. However, it will be the opposite when deleting
// folders (TODO).
if (context.hasMoreFolders) {
return BaseModel.db().selectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit + ' OFFSET ' + context.folderOffset).then((items) => {
context.hasMoreFolders = items.length >= limit;
context.folderOffset += items.length;
return { context: context, items: items };
});
} else {
return BaseModel.db().selectAll('SELECT * FROM folders WHERE sync_time < ? LIMIT ' + limit + ' OFFSET ' + context.folderOffset, [now]).then((items) => {
context.items = items;
context.hasMoreFolders = items.length >= limit;
context.hasMore = context.hasMoreFolders;
context.folderOffset += items.length;
return context;
return BaseModel.db().selectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit + ' OFFSET ' + context.noteOffset).then((items) => {
context.hasMoreNotes = items.length >= limit;
context.noteOffset += items.length;
context.hasMore = context.hasMoreNotes;
return { context: context, items: items };
});
}
}

View File

@ -4,7 +4,10 @@ import { Change } from 'src/models/change.js';
import { Folder } from 'src/models/folder.js';
import { Note } from 'src/models/note.js';
import { BaseModel } from 'src/base-model.js';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
import { NoteFolderService } from 'src/services/note-folder-service.js';
import { time } from 'src/time-utils.js';
//import { promiseWhile } from 'src/promise-utils.js';
import moment from 'moment';
const fs = require('fs');
@ -88,14 +91,6 @@ class Synchronizer {
return null;
}
syncAction(actionType, where, item, isConflict) {
return {
type: actionType,
where: where,
item: item,
};
}
itemIsSameDate(item, date) {
return Math.abs(item.updatedTime - date) <= 1;
}
@ -110,9 +105,45 @@ class Synchronizer {
return item.updatedTime < date;
}
dbItemToSyncItem(dbItem) {
let p = Promise.resolve(null);
let itemType = BaseModel.identifyItemType(dbItem);
let ItemClass = null;
if (itemType == BaseModel.ITEM_TYPE_NOTE) {
ItemClass = Note;
p = Folder.load(dbItem.parent_id);
} else {
ItemClass = Folder;
}
return p.then((dbParent) => {
let path = ItemClass.systemPath(dbParent, dbItem);
return {
isDir: itemType == BaseModel.ITEM_TYPE_FOLDER,
path: path,
syncTime: dbItem.sync_time,
updatedTime: dbItem.updated_time,
dbParent: dbParent,
dbItem: dbItem,
};
});
}
syncAction(localItem, remoteItem, deletedLocalPaths) {
let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths);
if (output.length !== 1) throw new Error('Invalid number of actions returned');
return output[0];
}
// Assumption: it's not possible to, for example, have a directory one the dest
// and a file with the same name on the source. It's not possible because the
// file and directory names are UUID so should be unique.
// Each item must have these properties:
// - path
// - isDir
// - syncTime
// - updatedTime
syncActions(localItems, remoteItems, deletedLocalPaths) {
let output = [];
let donePaths = [];
@ -127,7 +158,7 @@ class Synchronizer {
};
if (!remote) {
if (local.lastSyncTime) {
if (local.syncTime) {
// The item has been synced previously and now is no longer in the dest
// which means it has been deleted.
action.type = 'delete';
@ -139,9 +170,9 @@ class Synchronizer {
action.dest = 'remote';
}
} else {
if (this.itemIsOlderThan(local, local.lastSyncTime)) continue;
if (this.itemIsOlderThan(local, local.syncTime)) continue;
if (this.itemIsOlderThan(remote, local.lastSyncTime)) {
if (this.itemIsOlderThan(remote, local.syncTime)) {
action.type = 'update';
action.dest = 'remote';
} else {
@ -188,7 +219,7 @@ class Synchronizer {
action.dest = 'local';
}
} else {
if (this.itemIsOlderThan(remote, local.lastSyncTime)) continue; // Already have this version
if (this.itemIsOlderThan(remote, local.syncTime)) continue; // Already have this version
// Note: no conflict is possible here since if the local item has been
// modified since the last sync, it's been processed in the previous loop.
action.type = 'update';
@ -201,12 +232,6 @@ class Synchronizer {
return output;
}
processSyncActions(syncActions) {
for (let i = 0; i < syncActions.length; i++) {
}
}
processState_uploadChanges() {
let remoteFiles = [];
let processedChangeIds = [];
@ -548,16 +573,111 @@ class Synchronizer {
}
}
processSyncAction(action) {
// console.info(action);
if (action.type == 'conflict') {
} else {
let item = action[action.dest == 'local' ? 'remote' : 'local'];
let ItemClass = null;
if (item.isDir) {
ItemClass = Folder;
} else {
ItemClass = Note;
}
let path = ItemClass.systemPath(item.dbParent, item.dbItem);
if (action.type == 'create') {
if (action.dest == 'remote') {
if (item.isDir) {
return this.api().mkdir(path);
} else {
return this.api().put(path, Note.toFriendlyString(item.dbItem));
}
}
}
}
return Promise.resolve(); // TODO
}
processLocalItem(dbItem) {
//console.info(dbItem);
let localItem = null;
return this.dbItemToSyncItem(dbItem).then((r) => {
localItem = r;
return this.api().stat(localItem.path);
}).then((remoteItem) => {
let action = this.syncAction(localItem, remoteItem, []);
//console.info(action);
return this.processSyncAction(action);
}).then(() => {
dbItem.sync_time = time.unix();
if (localItem.isDir) {
return Folder.save(dbItem);
} else {
return Note.save(dbItem);
}
});
}
start() {
Log.info('Sync: start');
if (this.state() != 'idle') {
Log.info("Sync: cannot start synchronizer because synchronization already in progress. State: " + this.state());
return;
return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state());
}
this.state_ = 'started';
return this.api().listDirectories().then((items) => {
var context = null;
let limit = 2;
let finishedReading = false;
let isReading = false;
let readItems = () => {
isReading = true;
return NoteFolderService.itemsThatNeedSync(context, limit).then((result) => {
context = result.context;
let chain = [];
for (let i = 0; i < result.items.length; i++) {
let item = result.items[i];
console.info(JSON.stringify(item));
chain.push(() => {
//return Promise.resolve();
return this.processLocalItem(item);
});
}
return promiseChain(chain).then(() => {
console.info('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX');
if (!context.hasMore) finishedReading = true;
isReading = false;
});
}).catch((error) => {
console.error(error);
throw error;
});
}
let iid = setInterval(() => {
if (isReading) return;
if (finishedReading) {
clearInterval(iid);
return;
}
readItems();
}, 100);
}).then(() => {
this.state_ = 'idle';
});
//return NoteFolderService.itemsThatNeedSync
// if (!this.api().session()) {
@ -568,6 +688,8 @@ class Synchronizer {
//return this.processState('uploadChanges');
}
}
export { Synchronizer };

View File

@ -4,7 +4,7 @@ import { Change } from 'src/models/change.js';
import { Folder } from 'src/models/folder.js';
import { Note } from 'src/models/note.js';
import { BaseModel } from 'src/base-model.js';
import { promiseChain } from 'src/promise-chain.js';
import { promiseChain } from 'src/promise-utils.js';
class Synchronizer {