You've already forked joplin
mirror of
https://github.com/laurent22/joplin.git
synced 2025-09-16 08:56:40 +02:00
Testing OneDrive API
This commit is contained in:
Laurent Cozic
3
CliClient/.gitignore
vendored
3
CliClient/.gitignore
vendored
@@ -2,4 +2,5 @@ build/
|
||||
node_modules/
|
||||
app/src
|
||||
tests-build/
|
||||
tests/src
|
||||
tests/src
|
||||
config.json
|
||||
@@ -1,5 +1,4 @@
|
||||
require('source-map-support').install();
|
||||
//require("babel-polyfill");
|
||||
require('babel-plugin-transform-runtime');
|
||||
|
||||
import { FileApi } from 'src/file-api.js';
|
||||
@@ -23,158 +22,6 @@ let fileDriver = new FileApiDriverLocal();
|
||||
let fileApi = new FileApi('/home/laurent/Temp/TestImport', fileDriver);
|
||||
let synchronizer = new Synchronizer(db, fileApi);
|
||||
|
||||
function sleep(n) {
|
||||
return new Promise((resolve, reject) => {
|
||||
setTimeout(() => {
|
||||
resolve();
|
||||
}, n * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
function clearDatabase() {
|
||||
let queries = [
|
||||
'DELETE FROM changes',
|
||||
'DELETE FROM notes',
|
||||
'DELETE FROM folders',
|
||||
'DELETE FROM item_sync_times',
|
||||
];
|
||||
|
||||
return db.transactionExecBatch(queries);
|
||||
}
|
||||
|
||||
async function runTest() {
|
||||
db.setDebugEnabled(!true);
|
||||
await db.open({ name: '/home/laurent/Temp/test-sync.sqlite3' });
|
||||
|
||||
BaseModel.db_ = db;
|
||||
|
||||
await clearDatabase();
|
||||
|
||||
let folder = await Folder.save({ title: "folder1" });
|
||||
//console.info(folder);
|
||||
let note1 = await Note.save({ title: "un", parent_id: folder.id });
|
||||
await Note.save({ title: "deux", parent_id: folder.id });
|
||||
folder = await Folder.save({ title: "folder2" });
|
||||
await Note.save({ title: "trois", parent_id: folder.id });
|
||||
|
||||
await synchronizer.start();
|
||||
|
||||
// note1 = await Note.load(note1.id);
|
||||
// note1.title = 'un update';
|
||||
// //console.info('AVANT', note1);
|
||||
// note1 = await Note.save(note1);
|
||||
// //console.info('APRES', note1);
|
||||
|
||||
// return await synchronizer.start();
|
||||
}
|
||||
|
||||
runTest().catch((error) => {
|
||||
console.error(error);
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
function createRemoteItems() {
|
||||
let a = fileApi;
|
||||
return Promise.all([a.mkdir('test1'), a.mkdir('test2'), a.mkdir('test3')]).then(() => {
|
||||
return Promise.all([
|
||||
a.put('test1/un', 'test1_un'),
|
||||
a.put('test1/deux', 'test1_deux'),
|
||||
a.put('test2/trois', 'test2_trois'),
|
||||
a.put('test3/quatre', 'test3_quatre'),
|
||||
a.put('test3/cinq', 'test3_cinq'),
|
||||
a.put('test3/six', 'test3_six'),
|
||||
]);
|
||||
});
|
||||
}
|
||||
|
||||
async function createLocalItems() {
|
||||
let folder = await Folder.save({ title: "folder1" });
|
||||
await Note.save({ title: "un", parent_id: folder.id });
|
||||
await Note.save({ title: "deux", parent_id: folder.id });
|
||||
|
||||
folder = await Folder.save({ title: "folder2" });
|
||||
await Note.save({ title: "trois", parent_id: folder.id });
|
||||
|
||||
|
||||
// let folder = await Folder.save({ title: "folder1" });
|
||||
// await Note.save({ title: "un", parent_id: folder.id });
|
||||
// await Note.save({ title: "deux", parent_id: folder.id });
|
||||
// await Note.save({ title: "trois", parent_id: folder.id });
|
||||
// await Note.save({ title: "quatre", parent_id: folder.id });
|
||||
|
||||
// folder = await Folder.save({ title: "folder2" });
|
||||
// await Note.save({ title: "cinq", parent_id: folder.id });
|
||||
|
||||
// folder = await Folder.save({ title: "folder3" });
|
||||
|
||||
// folder = await Folder.save({ title: "folder4" });
|
||||
// await Note.save({ title: "six", parent_id: folder.id });
|
||||
// await Note.save({ title: "sept", parent_id: folder.id });
|
||||
// await Note.save({ title: "huit", parent_id: folder.id });
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// db.setDebugEnabled(!true);
|
||||
// db.open({ name: '/home/laurent/Temp/test-sync.sqlite3' }).then(() => {
|
||||
// BaseModel.db_ = db;
|
||||
// //return clearDatabase();
|
||||
// return clearDatabase().then(createLocalItems);
|
||||
// }).then(() => {
|
||||
// return synchronizer.start();
|
||||
// }).catch((error) => {
|
||||
// console.error(error);
|
||||
// });
|
||||
|
||||
|
||||
|
||||
|
||||
// function testingProm() {
|
||||
// return new Promise((resolve, reject) => {
|
||||
// setTimeout(() => {
|
||||
// resolve('mavaler');
|
||||
// }, 2000);
|
||||
// });
|
||||
// }
|
||||
|
||||
// async function doSomething() {
|
||||
// let val = await testingProm();
|
||||
// console.info(val);
|
||||
// }
|
||||
|
||||
// doSomething();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// let fileDriver = new FileApiDriverMemory();
|
||||
// let fileApi = new FileApi('/root', fileDriver);
|
||||
// let synchronizer = new Synchronizer(db, fileApi);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
58
CliClient/app/test-onedrive.js
Normal file
58
CliClient/app/test-onedrive.js
Normal file
@@ -0,0 +1,58 @@
|
||||
require('source-map-support').install();
|
||||
require('babel-plugin-transform-runtime');
|
||||
const MicrosoftGraph = require("@microsoft/microsoft-graph-client");
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
function configContent() {
|
||||
const configFilePath = path.dirname(__dirname) + '/config.json';
|
||||
return fs.readFile(configFilePath, 'utf8').then((content) => {
|
||||
return JSON.parse(content);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async function main() {
|
||||
let config = await configContent();
|
||||
|
||||
var token = '';
|
||||
var client = MicrosoftGraph.Client.init({
|
||||
authProvider: (done) => {
|
||||
done(null, config.oneDriveToken);
|
||||
}
|
||||
});
|
||||
|
||||
// LIST ITEMS
|
||||
|
||||
// client.api('/drive/items/9ADA0EADFA073D0A%21109/children').get((err, res) => {
|
||||
// console.log(err, res);
|
||||
// });
|
||||
|
||||
// SET ITEM CONTENT
|
||||
|
||||
// client.api('/drive/items/9ADA0EADFA073D0A%21109:/test.txt:/content').put('testing', (err, res) => {
|
||||
// console.log(err, res);
|
||||
// });
|
||||
|
||||
// SET ITEM CONTENT
|
||||
|
||||
// client.api('/drive/items/9ADA0EADFA073D0A%21109:/test2.txt:/content').put('testing deux', (err, res) => {
|
||||
// console.log(err, res);
|
||||
// });
|
||||
|
||||
// DELETE ITEM
|
||||
|
||||
// client.api('/drive/items/9ADA0EADFA073D0A%21111').delete((err, res) => {
|
||||
// console.log(err, res);
|
||||
// });
|
||||
|
||||
// GET ITEM METADATA
|
||||
|
||||
client.api('/drive/items/9ADA0EADFA073D0A%21110?select=name,lastModifiedDateTime').get((err, res) => {
|
||||
console.log(err, res);
|
||||
});
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
});
|
||||
@@ -3,6 +3,7 @@
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@microsoft/microsoft-graph-client": "^1.0.0",
|
||||
"app-module-path": "^2.2.0",
|
||||
"babel-plugin-transform-runtime": "^6.23.0",
|
||||
"babel-polyfill": "^6.1.4",
|
||||
|
||||
@@ -4,4 +4,5 @@ CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
rm -f "$CLIENT_DIR/app/src"
|
||||
ln -s "$CLIENT_DIR/../ReactNativeClient/src" "$CLIENT_DIR/app"
|
||||
|
||||
npm run build && NODE_PATH="$CLIENT_DIR/build/" node build/cmd.js
|
||||
#npm run build && NODE_PATH="$CLIENT_DIR/build/" node build/cmd.js
|
||||
npm run build && NODE_PATH="$CLIENT_DIR/build/" node build/test-onedrive.js
|
||||
@@ -143,13 +143,18 @@ class BaseModel {
|
||||
|
||||
static diffObjects(oldModel, newModel) {
|
||||
let output = {};
|
||||
let type = null;
|
||||
for (let n in newModel) {
|
||||
if (n == 'type_') continue;
|
||||
if (n == 'type_') {
|
||||
type = n;
|
||||
continue;
|
||||
}
|
||||
if (!newModel.hasOwnProperty(n)) continue;
|
||||
if (!(n in oldModel) || newModel[n] !== oldModel[n]) {
|
||||
output[n] = newModel[n];
|
||||
}
|
||||
}
|
||||
if (type !== null) output.type_ = type;
|
||||
return output;
|
||||
}
|
||||
|
||||
|
||||
@@ -42,10 +42,17 @@ class FolderScreenComponent extends React.Component {
|
||||
}
|
||||
|
||||
saveFolderButton_press() {
|
||||
NoteFolderService.save('folder', this.state.folder, this.originalFolder).then((folder) => {
|
||||
console.warn('CHANGE NOT TESTED');
|
||||
let toSave = BaseModel.diffObjects(this.originalFolder, this.state.folder);
|
||||
toSave.id = this.state.folder.id;
|
||||
Folder.save(toSave).then((folder) => {
|
||||
this.originalFolder = Object.assign({}, folder);
|
||||
this.setState({ folder: folder });
|
||||
});
|
||||
// NoteFolderService.save('folder', this.state.folder, this.originalFolder).then((folder) => {
|
||||
// this.originalFolder = Object.assign({}, folder);
|
||||
// this.setState({ folder: folder });
|
||||
// });
|
||||
}
|
||||
|
||||
render() {
|
||||
|
||||
@@ -51,10 +51,22 @@ class NoteScreenComponent extends React.Component {
|
||||
}
|
||||
|
||||
saveNoteButton_press() {
|
||||
NoteFolderService.save('note', this.state.note, this.originalNote).then((note) => {
|
||||
|
||||
console.warn('CHANGE NOT TESTED');
|
||||
|
||||
let isNew = !this.state.note.id;
|
||||
let toSave = BaseModel.diffObjects(this.originalNote, this.state.note);
|
||||
toSave.id = this.state.note.id;
|
||||
Note.save(toSave).then((note) => {
|
||||
this.originalNote = Object.assign({}, note);
|
||||
this.setState({ note: note });
|
||||
if (isNew) return Note.updateGeolocation(note.id);
|
||||
});
|
||||
|
||||
// NoteFolderService.save('note', this.state.note, this.originalNote).then((note) => {
|
||||
// this.originalNote = Object.assign({}, note);
|
||||
// this.setState({ note: note });
|
||||
// });
|
||||
}
|
||||
|
||||
deleteNote_onPress(noteId) {
|
||||
|
||||
@@ -46,17 +46,6 @@ class NoteFolderService extends BaseService {
|
||||
});
|
||||
}
|
||||
|
||||
// static setField(type, itemId, fieldName, fieldValue, oldValue = undefined) {
|
||||
// // TODO: not really consistent as the promise will return 'null' while
|
||||
// // this.save will return the note or folder. Currently not used, and maybe not needed.
|
||||
// if (oldValue !== undefined && fieldValue === oldValue) return Promise.resolve();
|
||||
|
||||
// let item = { id: itemId };
|
||||
// item[fieldName] = fieldValue;
|
||||
// let oldItem = { id: itemId };
|
||||
// return this.save(type, item, oldItem);
|
||||
// }
|
||||
|
||||
static openNoteList(folderId) {
|
||||
return Note.previews(folderId).then((notes) => {
|
||||
this.dispatch({
|
||||
@@ -74,29 +63,6 @@ class NoteFolderService extends BaseService {
|
||||
});
|
||||
}
|
||||
|
||||
static itemsThatNeedSync(limit = 100) {
|
||||
// Process folder first, then notes so that folders are created before
|
||||
// adding notes to them. However, it will be the opposite when deleting
|
||||
// folders (TODO).
|
||||
|
||||
function getFolders(limit) {
|
||||
return Folder.modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit);
|
||||
//return BaseModel.db().selectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit);
|
||||
}
|
||||
|
||||
function getNotes(limit) {
|
||||
return Note.modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit);
|
||||
//return BaseModel.db().selectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit);
|
||||
}
|
||||
|
||||
return getFolders(limit).then((items) => {
|
||||
if (items.length) return { hasMore: true, items: items };
|
||||
return getNotes(limit).then((items) => {
|
||||
return { hasMore: items.length >= limit, items: items };
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export { NoteFolderService };
|
||||
@@ -76,13 +76,15 @@ class Synchronizer {
|
||||
|
||||
} else if (action == 'folderConflict') {
|
||||
|
||||
// TODO: if remote has been deleted, delete local too
|
||||
if (remote) {
|
||||
let remoteContent = await this.api().get(path);
|
||||
local = BaseItem.unserialize(remoteContent);
|
||||
|
||||
let remoteContent = await this.api().get(path);
|
||||
local = BaseItem.unserialize(remoteContent);
|
||||
|
||||
local.sync_time = time.unixMs();
|
||||
await ItemClass.save(local, { autoTimestamp: false });
|
||||
local.sync_time = time.unixMs();
|
||||
await ItemClass.save(local, { autoTimestamp: false });
|
||||
} else {
|
||||
await ItemClass.delete(local.id);
|
||||
}
|
||||
|
||||
} else if (action == 'noteConflict') {
|
||||
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
import { Log } from 'src/log.js';
|
||||
import { Setting } from 'src/models/setting.js';
|
||||
import { Change } from 'src/models/change.js';
|
||||
import { Folder } from 'src/models/folder.js';
|
||||
import { Note } from 'src/models/note.js';
|
||||
import { BaseModel } from 'src/base-model.js';
|
||||
import { promiseChain } from 'src/promise-utils.js';
|
||||
|
||||
class Synchronizer {
|
||||
|
||||
constructor(db, api) {
|
||||
this.state_ = 'idle';
|
||||
this.db_ = db;
|
||||
this.api_ = api;
|
||||
}
|
||||
|
||||
state() {
|
||||
return this.state_;
|
||||
}
|
||||
|
||||
db() {
|
||||
return this.db_;
|
||||
}
|
||||
|
||||
api() {
|
||||
return this.api_;
|
||||
}
|
||||
|
||||
loadParentAndItem(change) {
|
||||
if (change.item_type == BaseModel.MODEL_TYPE_NOTE) {
|
||||
return Note.load(change.item_id).then((note) => {
|
||||
return Folder.load(note.parent_id).then((folder) => {
|
||||
return Promise.resolve({ parent: folder, item: note });
|
||||
});
|
||||
});
|
||||
} else {
|
||||
return Folder.load(change.item_id).then((folder) => {
|
||||
return Promise.resolve({ parent: null, item: folder });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
processState_uploadChanges() {
|
||||
Change.all().then((changes) => {
|
||||
let mergedChanges = Change.mergeChanges(changes);
|
||||
let chain = [];
|
||||
let processedChangeIds = [];
|
||||
for (let i = 0; i < mergedChanges.length; i++) {
|
||||
let c = mergedChanges[i];
|
||||
chain.push(() => {
|
||||
let p = null;
|
||||
|
||||
let ItemClass = null;
|
||||
let path = null;
|
||||
if (c.item_type == BaseModel.MODEL_TYPE_FOLDER) {
|
||||
ItemClass = Folder;
|
||||
path = 'folders';
|
||||
} else if (c.item_type == BaseModel.MODEL_TYPE_NOTE) {
|
||||
ItemClass = Note;
|
||||
path = 'notes';
|
||||
}
|
||||
|
||||
if (c.type == Change.TYPE_NOOP) {
|
||||
p = Promise.resolve();
|
||||
} else if (c.type == Change.TYPE_CREATE) {
|
||||
p = this.loadParentAndItem(c).then((result) => {
|
||||
let options = {
|
||||
contents: Note.serialize(result.item),
|
||||
path: Note.systemPath(result.parent, result.item),
|
||||
mode: 'overwrite',
|
||||
// client_modified:
|
||||
};
|
||||
|
||||
return this.api().filesUpload(options).then((result) => {
|
||||
console.info('DROPBOX', result);
|
||||
});
|
||||
});
|
||||
// p = ItemClass.load(c.item_id).then((item) => {
|
||||
|
||||
// console.info(item);
|
||||
// let options = {
|
||||
// contents: Note.serialize(item),
|
||||
// path: Note.systemPath(item),
|
||||
// mode: 'overwrite',
|
||||
// // client_modified:
|
||||
// };
|
||||
|
||||
// // console.info(options);
|
||||
|
||||
// //let content = Note.serialize(item);
|
||||
// //console.info(content);
|
||||
|
||||
// //console.info('SYNC', item);
|
||||
// //return this.api().put(path + '/' + item.id, null, item);
|
||||
// });
|
||||
} else if (c.type == Change.TYPE_UPDATE) {
|
||||
p = ItemClass.load(c.item_id).then((item) => {
|
||||
//return this.api().patch(path + '/' + item.id, null, item);
|
||||
});
|
||||
} else if (c.type == Change.TYPE_DELETE) {
|
||||
p = this.api().delete(path + '/' + c.item_id);
|
||||
}
|
||||
|
||||
return p.then(() => {
|
||||
processedChangeIds = processedChangeIds.concat(c.ids);
|
||||
}).catch((error) => {
|
||||
// Log.warn('Failed applying changes', c.ids, error.message, error.type);
|
||||
// This is fine - trying to apply changes to an object that has been deleted
|
||||
if (error.type == 'NotFoundException') {
|
||||
processedChangeIds = processedChangeIds.concat(c.ids);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return promiseChain(chain).catch((error) => {
|
||||
Log.warn('Synchronization was interrupted due to an error:', error);
|
||||
}).then(() => {
|
||||
// Log.info('IDs to delete: ', processedChangeIds);
|
||||
// Change.deleteMultiple(processedChangeIds);
|
||||
});
|
||||
}).then(() => {
|
||||
this.processState('downloadChanges');
|
||||
});
|
||||
}
|
||||
|
||||
processState_downloadChanges() {
|
||||
let maxRevId = null;
|
||||
let hasMore = false;
|
||||
this.api().get('synchronizer', { rev_id: Setting.value('sync.lastRevId') }).then((syncOperations) => {
|
||||
hasMore = syncOperations.has_more;
|
||||
let chain = [];
|
||||
for (let i = 0; i < syncOperations.items.length; i++) {
|
||||
let syncOp = syncOperations.items[i];
|
||||
if (syncOp.id > maxRevId) maxRevId = syncOp.id;
|
||||
|
||||
let ItemClass = null;
|
||||
if (syncOp.item_type == 'folder') {
|
||||
ItemClass = Folder;
|
||||
} else if (syncOp.item_type == 'note') {
|
||||
ItemClass = Note;
|
||||
}
|
||||
|
||||
if (syncOp.type == 'create') {
|
||||
chain.push(() => {
|
||||
let item = ItemClass.fromApiResult(syncOp.item);
|
||||
// TODO: automatically handle NULL fields by checking type and default value of field
|
||||
if ('parent_id' in item && !item.parent_id) item.parent_id = '';
|
||||
return ItemClass.save(item, { isNew: true, trackChanges: false });
|
||||
});
|
||||
}
|
||||
|
||||
if (syncOp.type == 'update') {
|
||||
chain.push(() => {
|
||||
return ItemClass.load(syncOp.item_id).then((item) => {
|
||||
if (!item) return;
|
||||
item = ItemClass.applyPatch(item, syncOp.item);
|
||||
return ItemClass.save(item, { trackChanges: false });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (syncOp.type == 'delete') {
|
||||
chain.push(() => {
|
||||
return ItemClass.delete(syncOp.item_id, { trackChanges: false });
|
||||
});
|
||||
}
|
||||
}
|
||||
return promiseChain(chain);
|
||||
}).then(() => {
|
||||
Log.info('All items synced. has_more = ', hasMore);
|
||||
if (maxRevId) {
|
||||
Setting.setValue('sync.lastRevId', maxRevId);
|
||||
return Setting.saveAll();
|
||||
}
|
||||
}).then(() => {
|
||||
if (hasMore) {
|
||||
this.processState('downloadChanges');
|
||||
} else {
|
||||
this.processState('idle');
|
||||
}
|
||||
}).catch((error) => {
|
||||
Log.warn('Sync error', error);
|
||||
});
|
||||
}
|
||||
|
||||
processState(state) {
|
||||
Log.info('Sync: processing: ' + state);
|
||||
this.state_ = state;
|
||||
|
||||
if (state == 'uploadChanges') {
|
||||
this.processState_uploadChanges();
|
||||
} else if (state == 'downloadChanges') {
|
||||
this.processState('idle');
|
||||
//this.processState_downloadChanges();
|
||||
} else if (state == 'idle') {
|
||||
// Nothing
|
||||
} else {
|
||||
throw new Error('Invalid state: ' . state);
|
||||
}
|
||||
}
|
||||
|
||||
start() {
|
||||
Log.info('Sync: start');
|
||||
|
||||
if (this.state() != 'idle') {
|
||||
Log.info("Sync: cannot start synchronizer because synchronization already in progress. State: " + this.state());
|
||||
return;
|
||||
}
|
||||
|
||||
// if (!this.api().session()) {
|
||||
// Log.info("Sync: cannot start synchronizer because user is not logged in.");
|
||||
// return;
|
||||
// }
|
||||
|
||||
this.processState('uploadChanges');
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export { Synchronizer };
|
||||
@@ -1,418 +0,0 @@
|
||||
require('babel-plugin-transform-runtime');
|
||||
|
||||
import { Log } from 'src/log.js';
|
||||
import { Setting } from 'src/models/setting.js';
|
||||
import { Change } from 'src/models/change.js';
|
||||
import { Folder } from 'src/models/folder.js';
|
||||
import { Note } from 'src/models/note.js';
|
||||
import { BaseItem } from 'src/models/base-item.js';
|
||||
import { BaseModel } from 'src/base-model.js';
|
||||
import { promiseChain } from 'src/promise-utils.js';
|
||||
import { NoteFolderService } from 'src/services/note-folder-service.js';
|
||||
import { time } from 'src/time-utils.js';
|
||||
import { sprintf } from 'sprintf-js';
|
||||
//import { promiseWhile } from 'src/promise-utils.js';
|
||||
import moment from 'moment';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
class Synchronizer {
|
||||
|
||||
constructor(db, api) {
|
||||
this.state_ = 'idle';
|
||||
this.db_ = db;
|
||||
this.api_ = api;
|
||||
}
|
||||
|
||||
state() {
|
||||
return this.state_;
|
||||
}
|
||||
|
||||
db() {
|
||||
return this.db_;
|
||||
}
|
||||
|
||||
api() {
|
||||
return this.api_;
|
||||
}
|
||||
|
||||
loadParentAndItem(change) {
|
||||
if (change.item_type == BaseModel.MODEL_TYPE_NOTE) {
|
||||
return Note.load(change.item_id).then((note) => {
|
||||
if (!note) return { parent:null, item: null };
|
||||
|
||||
return Folder.load(note.parent_id).then((folder) => {
|
||||
return Promise.resolve({ parent: folder, item: note });
|
||||
});
|
||||
});
|
||||
} else {
|
||||
return Folder.load(change.item_id).then((folder) => {
|
||||
return Promise.resolve({ parent: null, item: folder });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
remoteFileByPath(remoteFiles, path) {
|
||||
for (let i = 0; i < remoteFiles.length; i++) {
|
||||
if (remoteFiles[i].path == path) return remoteFiles[i];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
conflictDir(remoteFiles) {
|
||||
let d = this.remoteFileByPath('Conflicts');
|
||||
if (!d) {
|
||||
return this.api().mkdir('Conflicts').then(() => {
|
||||
return 'Conflicts';
|
||||
});
|
||||
} else {
|
||||
return Promise.resolve('Conflicts');
|
||||
}
|
||||
}
|
||||
|
||||
moveConflict(item) {
|
||||
// No need to handle folder conflicts
|
||||
if (item.type == 'folder') return Promise.resolve();
|
||||
|
||||
return this.conflictDir().then((conflictDirPath) => {
|
||||
let p = path.basename(item.path).split('.');
|
||||
let pos = item.type == 'folder' ? p.length - 1 : p.length - 2;
|
||||
p.splice(pos, 0, moment().format('YYYYMMDDThhmmss'));
|
||||
let newPath = p.join('.');
|
||||
return this.api().move(item.path, conflictDirPath + '/' + newPath);
|
||||
});
|
||||
}
|
||||
|
||||
itemByPath(items, path) {
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
if (items[i].path == path) return items[i];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
itemIsSameDate(item, date) {
|
||||
return item.updated_time === date;
|
||||
}
|
||||
|
||||
itemIsStrictlyNewerThan(item, date) {
|
||||
return item.updated_time > date;
|
||||
}
|
||||
|
||||
itemIsStrictlyOlderThan(item, date) {
|
||||
return item.updated_time < date;
|
||||
}
|
||||
|
||||
dbItemToSyncItem(dbItem) {
|
||||
if (!dbItem) return null;
|
||||
|
||||
return {
|
||||
type: dbItem.type_ == BaseModel.MODEL_TYPE_FOLDER ? 'folder' : 'note',
|
||||
path: Folder.systemPath(dbItem),
|
||||
syncTime: dbItem.sync_time,
|
||||
updated_time: dbItem.updated_time,
|
||||
dbItem: dbItem,
|
||||
};
|
||||
}
|
||||
|
||||
remoteItemToSyncItem(remoteItem) {
|
||||
if (!remoteItem) return null;
|
||||
|
||||
return {
|
||||
type: remoteItem.content.type_ == BaseModel.MODEL_TYPE_FOLDER ? 'folder' : 'note',
|
||||
path: remoteItem.path,
|
||||
syncTime: 0,
|
||||
updated_time: remoteItem.updated_time,
|
||||
remoteItem: remoteItem,
|
||||
};
|
||||
}
|
||||
|
||||
syncAction(localItem, remoteItem, deletedLocalPaths) {
|
||||
let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths);
|
||||
if (output.length > 1) throw new Error('Invalid number of actions returned');
|
||||
return output.length ? output[0] : null;
|
||||
}
|
||||
|
||||
// Assumption: it's not possible to, for example, have a directory one the dest
|
||||
// and a file with the same name on the source. It's not possible because the
|
||||
// file and directory names are UUID so should be unique.
|
||||
// Each item must have these properties:
|
||||
// - path
|
||||
// - type
|
||||
// - syncTime
|
||||
// - updated_time
|
||||
syncActions(localItems, remoteItems, deletedLocalPaths) {
|
||||
let output = [];
|
||||
let donePaths = [];
|
||||
|
||||
// console.info('==================================================');
|
||||
// console.info(localItems, remoteItems);
|
||||
|
||||
for (let i = 0; i < localItems.length; i++) {
|
||||
let local = localItems[i];
|
||||
let remote = this.itemByPath(remoteItems, local.path);
|
||||
|
||||
let action = {
|
||||
local: local,
|
||||
remote: remote,
|
||||
};
|
||||
|
||||
if (!remote) {
|
||||
if (local.syncTime) {
|
||||
action.type = 'delete';
|
||||
action.dest = 'local';
|
||||
action.reason = 'Local has been synced to remote previously, but remote no longer exist, which means remote has been deleted';
|
||||
} else {
|
||||
action.type = 'create';
|
||||
action.dest = 'remote';
|
||||
action.reason = 'Local has never been synced to remote, and remote does not exists, which means remote must be created';
|
||||
}
|
||||
} else {
|
||||
if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue;
|
||||
|
||||
if (this.itemIsStrictlyOlderThan(remote, local.updated_time)) {
|
||||
action.type = 'update';
|
||||
action.dest = 'remote';
|
||||
action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.syncTime).toISOString(),);
|
||||
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) {
|
||||
action.type = 'conflict';
|
||||
action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).',
|
||||
moment.unix(remote.updated_time).toISOString(),
|
||||
moment.unix(local.updated_time).toISOString(),
|
||||
moment.unix(local.syncTime).toISOString()
|
||||
);
|
||||
|
||||
if (local.type == 'folder') {
|
||||
action.solution = [
|
||||
{ type: 'update', dest: 'local' },
|
||||
];
|
||||
} else {
|
||||
action.solution = [
|
||||
{ type: 'copy-to-remote-conflict-dir', dest: 'local' },
|
||||
{ type: 'copy-to-local-conflict-dir', dest: 'local' },
|
||||
{ type: 'update', dest: 'local' },
|
||||
];
|
||||
}
|
||||
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updated_time <= local.syncTime) {
|
||||
action.type = 'update';
|
||||
action.dest = 'local';
|
||||
action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(), moment.unix(local.syncTime).toISOString());
|
||||
} else {
|
||||
continue; // Neither local nor remote item have been changed recently
|
||||
}
|
||||
}
|
||||
|
||||
donePaths.push(local.path);
|
||||
|
||||
output.push(action);
|
||||
}
|
||||
|
||||
for (let i = 0; i < remoteItems.length; i++) {
|
||||
let remote = remoteItems[i];
|
||||
if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop
|
||||
let local = this.itemByPath(localItems, remote.path);
|
||||
|
||||
let action = {
|
||||
local: local,
|
||||
remote: remote,
|
||||
};
|
||||
|
||||
if (!local) {
|
||||
if (deletedLocalPaths.indexOf(remote.path) >= 0) {
|
||||
action.type = 'delete';
|
||||
action.dest = 'remote';
|
||||
} else {
|
||||
action.type = 'create';
|
||||
action.dest = 'local';
|
||||
}
|
||||
} else {
|
||||
if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version
|
||||
|
||||
// Note: no conflict is possible here since if the local item has been
|
||||
// modified since the last sync, it's been processed in the previous loop.
|
||||
// So throw an exception is this normally impossible condition happens anyway.
|
||||
// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
|
||||
if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
|
||||
console.error('Remote cannot be newer than last sync time', remote, local);
|
||||
throw new Error('Remote cannot be newer than last sync time');
|
||||
}
|
||||
|
||||
if (this.itemIsStrictlyNewerThan(remote, local.updated_time)) {
|
||||
action.type = 'update';
|
||||
action.dest = 'local';
|
||||
action.reason = sprintf('Remote (%s) was modified after local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(),);;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
output.push(action);
|
||||
}
|
||||
|
||||
// console.info('-----------------------------------------');
|
||||
// console.info(output);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
processState(state) {
|
||||
Log.info('Sync: processing: ' + state);
|
||||
this.state_ = state;
|
||||
|
||||
if (state == 'uploadChanges') {
|
||||
return this.processState_uploadChanges();
|
||||
} else if (state == 'downloadChanges') {
|
||||
//return this.processState('idle');
|
||||
return this.processState_downloadChanges();
|
||||
} else if (state == 'idle') {
|
||||
// Nothing
|
||||
return Promise.resolve();
|
||||
} else {
|
||||
throw new Error('Invalid state: ' . state);
|
||||
}
|
||||
}
|
||||
|
||||
processSyncAction(action) {
|
||||
//console.info('Sync action: ', action);
|
||||
//console.info('Sync action: ' + JSON.stringify(action));
|
||||
|
||||
if (!action) return Promise.resolve();
|
||||
|
||||
console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason);
|
||||
|
||||
if (action.type == 'conflict') {
|
||||
console.info(action);
|
||||
|
||||
} else {
|
||||
let syncItem = action[action.dest == 'local' ? 'remote' : 'local'];
|
||||
let path = syncItem.path;
|
||||
|
||||
if (action.type == 'create') {
|
||||
if (action.dest == 'remote') {
|
||||
let content = null;
|
||||
let dbItem = syncItem.dbItem;
|
||||
|
||||
if (syncItem.type == 'folder') {
|
||||
content = Folder.serialize(dbItem);
|
||||
} else {
|
||||
content = Note.serialize(dbItem);
|
||||
}
|
||||
|
||||
return this.api().put(path, content).then(() => {
|
||||
return this.api().setTimestamp(path, dbItem.updated_time);
|
||||
});
|
||||
|
||||
// TODO: save sync_time
|
||||
} else {
|
||||
let dbItem = syncItem.remoteItem.content;
|
||||
dbItem.sync_time = time.unix();
|
||||
dbItem.updated_time = action.remote.updated_time;
|
||||
if (syncItem.type == 'folder') {
|
||||
return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
|
||||
} else {
|
||||
return Note.save(dbItem, { isNew: true, autoTimestamp: false });
|
||||
}
|
||||
|
||||
// TODO: save sync_time
|
||||
}
|
||||
}
|
||||
|
||||
if (action.type == 'update') {
|
||||
if (action.dest == 'remote') {
|
||||
let dbItem = syncItem.dbItem;
|
||||
let ItemClass = BaseItem.itemClass(dbItem);
|
||||
let content = ItemClass.serialize(dbItem);
|
||||
//console.info('PUT', content);
|
||||
return this.api().put(path, content).then(() => {
|
||||
return this.api().setTimestamp(path, dbItem.updated_time);
|
||||
}).then(() => {
|
||||
let toSave = { id: dbItem.id, sync_time: time.unix() };
|
||||
return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false });
|
||||
});
|
||||
} else {
|
||||
let dbItem = Object.assign({}, syncItem.remoteItem.content);
|
||||
dbItem.sync_time = time.unix();
|
||||
return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.resolve(); // TODO
|
||||
}
|
||||
|
||||
async processLocalItem(dbItem) {
|
||||
let localItem = this.dbItemToSyncItem(dbItem);
|
||||
|
||||
let remoteItem = await this.api().stat(localItem.path);
|
||||
let action = this.syncAction(localItem, remoteItem, []);
|
||||
await this.processSyncAction(action);
|
||||
|
||||
let toSave = Object.assign({}, dbItem);
|
||||
toSave.sync_time = time.unix();
|
||||
return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false });
|
||||
}
|
||||
|
||||
async processRemoteItem(remoteItem) {
|
||||
let content = await this.api().get(remoteItem.path);
|
||||
if (!content) throw new Error('Cannot get content for: ' + remoteItem.path);
|
||||
remoteItem.content = Note.unserialize(content);
|
||||
let remoteSyncItem = this.remoteItemToSyncItem(remoteItem);
|
||||
|
||||
let dbItem = await BaseItem.loadItemByPath(remoteItem.path);
|
||||
let localSyncItem = this.dbItemToSyncItem(dbItem);
|
||||
|
||||
let action = this.syncAction(localSyncItem, remoteSyncItem, []);
|
||||
return this.processSyncAction(action);
|
||||
}
|
||||
|
||||
async processState_uploadChanges() {
|
||||
while (true) {
|
||||
let result = await NoteFolderService.itemsThatNeedSync(50);
|
||||
console.info('Items that need sync: ' + result.items.length);
|
||||
for (let i = 0; i < result.items.length; i++) {
|
||||
let item = result.items[i];
|
||||
await this.processLocalItem(item);
|
||||
}
|
||||
|
||||
if (!result.hasMore) break;
|
||||
}
|
||||
|
||||
//console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve();
|
||||
|
||||
return this.processState('downloadChanges');
|
||||
}
|
||||
|
||||
async processState_downloadChanges() {
|
||||
let items = await this.api().list();
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
await this.processRemoteItem(items[i]);
|
||||
}
|
||||
|
||||
return this.processState('idle');
|
||||
}
|
||||
|
||||
start() {
|
||||
Log.info('Sync: start');
|
||||
|
||||
if (this.state() != 'idle') {
|
||||
return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state());
|
||||
}
|
||||
|
||||
this.state_ = 'started';
|
||||
|
||||
// if (!this.api().session()) {
|
||||
// Log.info("Sync: cannot start synchronizer because user is not logged in.");
|
||||
// return;
|
||||
// }
|
||||
|
||||
return this.processState('uploadChanges').catch((error) => {
|
||||
console.info('Synchronizer error:', error);
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
export { Synchronizer };
|
||||
Reference in New Issue
Block a user