You've already forked joplin
							
							
				mirror of
				https://github.com/laurent22/joplin.git
				synced 2025-10-31 00:07:48 +02:00 
			
		
		
		
	sync
This commit is contained in:
		| @@ -14,6 +14,195 @@ import { uuid } from 'src/uuid.js'; | ||||
| import { sprintf } from 'sprintf-js'; | ||||
| import { _ } from 'src/locale.js'; | ||||
| import { NoteFolderService } from 'src/services/note-folder-service.js'; | ||||
|  | ||||
|  | ||||
| let db = new Database(new DatabaseDriverNode()); | ||||
| db.setDebugEnabled(false); | ||||
|  | ||||
| // function whilePromise(callback) { | ||||
| // 	let isDone = false; | ||||
|  | ||||
| // 	function done() { | ||||
| // 		isDone = true; | ||||
| // 	} | ||||
|  | ||||
| // 	let iterationDone = false; | ||||
| // 	let p = callback(done).then(() => { | ||||
| // 		iterationDone = true; | ||||
| // 	}); | ||||
|  | ||||
| // 	let iid = setInterval(() => { | ||||
| // 		if (iterationDone) { | ||||
| // 			if (isDone) { | ||||
| // 				clearInterval(iid); | ||||
| // 				return; | ||||
| // 			} | ||||
|  | ||||
| // 			iterationDone = false; | ||||
| // 			callback(done).then(() => { | ||||
| // 				iterationDone = true; | ||||
| // 			}); | ||||
| // 		} | ||||
| // 	}, 100); | ||||
| // } | ||||
|  | ||||
| // function myPromise() { | ||||
| // 	return new Promise((resolve, reject) => { | ||||
| // 		setTimeout(() => { | ||||
| // 			resolve(); | ||||
| // 		}, 500); | ||||
| // 	}); | ||||
| // } | ||||
|  | ||||
| // let counter = 0; | ||||
| // whilePromise((done) => { | ||||
| // 	return myPromise().then(() => { | ||||
| // 		counter++; | ||||
| // 		console.info(counter); | ||||
| // 		if (counter == 5) { | ||||
| // 			done(); | ||||
| // 		} | ||||
| // 	}); | ||||
| // }); | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| let fileDriver = new FileApiDriverLocal(); | ||||
| let fileApi = new FileApi('/home/laurent/Temp/TestImport', fileDriver); | ||||
| let synchronizer = new Synchronizer(db, fileApi); | ||||
|  | ||||
| function clearDatabase() { | ||||
| 	let queries = [ | ||||
| 		'DELETE FROM changes', | ||||
| 		'DELETE FROM notes', | ||||
| 		'DELETE FROM folders', | ||||
| 		'DELETE FROM item_sync_times', | ||||
| 	]; | ||||
|  | ||||
| 	return db.transactionExecBatch(queries); | ||||
| } | ||||
|  | ||||
| function createRemoteItems() { | ||||
| 	let a = fileApi; | ||||
| 	return Promise.all([a.mkdir('test1'), a.mkdir('test2'), a.mkdir('test3')]).then(() => { | ||||
| 		return Promise.all([ | ||||
| 			a.put('test1/un', 'test1_un'), | ||||
| 			a.put('test1/deux', 'test1_deux'), | ||||
| 			a.put('test2/trois', 'test2_trois'), | ||||
| 			a.put('test3/quatre', 'test3_quatre'), | ||||
| 			a.put('test3/cinq', 'test3_cinq'), | ||||
| 			a.put('test3/six', 'test3_six'), | ||||
| 		]); | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| function createLocalItems() { | ||||
| 	return Folder.save({ title: "folder1" }).then((f) => { | ||||
| 		return Promise.all([ | ||||
| 			Note.save({ title: "un", parent_id: f.id }), | ||||
| 			Note.save({ title: "deux", parent_id: f.id }), | ||||
| 			Note.save({ title: "trois", parent_id: f.id }), | ||||
| 			Note.save({ title: "quatre", parent_id: f.id }), | ||||
| 		]); | ||||
| 	}).then(() => { | ||||
| 		return Folder.save({ title: "folder2" }) | ||||
| 	}).then((f) => { | ||||
| 		return Promise.all([ | ||||
| 			Note.save({ title: "cinq", parent_id: f.id }), | ||||
| 		]); | ||||
| 	}).then(() => { | ||||
| 		return Folder.save({ title: "folder3" }) | ||||
| 	}).then(() => { | ||||
| 		return Folder.save({ title: "folder4" }) | ||||
| 	}).then((f) => { | ||||
| 		return Promise.all([ | ||||
| 			Note.save({ title: "six", parent_id: f.id }), | ||||
| 			Note.save({ title: "sept", parent_id: f.id }), | ||||
| 			Note.save({ title: "huit", parent_id: f.id }), | ||||
| 		]); | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| db.open({ name: '/home/laurent/Temp/test-sync.sqlite3' }).then(() => { | ||||
|  	BaseModel.db_ = db; | ||||
|  	return clearDatabase().then(createLocalItems); | ||||
| }).then(() => { | ||||
| 	return synchronizer.start(); | ||||
| }).catch((error) => { | ||||
| 	console.error(error); | ||||
| }); | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| // let fileDriver = new FileApiDriverMemory(); | ||||
| // let fileApi = new FileApi('/root', fileDriver); | ||||
| // let synchronizer = new Synchronizer(db, fileApi); | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| // import { ItemSyncTime } from 'src/models/item-sync-time.js'; | ||||
|  | ||||
|   | ||||
| @@ -2,7 +2,7 @@ require('app-module-path').addPath(__dirname); | ||||
|  | ||||
| import { uuid } from 'src/uuid.js'; | ||||
| import moment from 'moment'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { WebApi } from 'src/web-api.js' | ||||
| import { folderItemFilename } from 'src/string-utils.js' | ||||
| import jsSHA from "jssha"; | ||||
|   | ||||
| @@ -6,4 +6,4 @@ mkdir -p "$CLIENT_DIR/tests-build/data" | ||||
| ln -s "$CLIENT_DIR/build/src" "$CLIENT_DIR/tests-build" | ||||
|  | ||||
| npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js | ||||
| # npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/services/note-folder-service.js | ||||
| #npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/services/note-folder-service.js | ||||
| @@ -1,9 +1,7 @@ | ||||
| import { time } from 'src/time-utils.js'; | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { Folder } from 'src/models/folder.js'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { NoteFolderService } from 'src/services/note-folder-service.js'; | ||||
| import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js'; | ||||
| import { setupDatabaseAndSynchronizer } from 'test-utils.js'; | ||||
| import { createFoldersAndNotes } from 'test-data.js'; | ||||
|  | ||||
| describe('NoteFolderServices', function() { | ||||
|  | ||||
| @@ -11,57 +9,8 @@ describe('NoteFolderServices', function() { | ||||
| 		setupDatabaseAndSynchronizer(done); | ||||
| 	}); | ||||
|  | ||||
| 	function createNotes(parentId, id = 1) { | ||||
| 		let notes = []; | ||||
| 		if (id === 1) { | ||||
| 			notes.push({ parent_id: parentId, title: 'note one', body: 'content of note one' }); | ||||
| 			notes.push({ parent_id: parentId, title: 'note two', body: 'content of note two' }); | ||||
| 		} else { | ||||
| 			throw new Error('Invalid ID: ' + id); | ||||
| 		} | ||||
|  | ||||
| 		let output = []; | ||||
| 		let chain = []; | ||||
| 		for (let i = 0; i < notes.length; i++) { | ||||
| 			chain.push(() => { | ||||
| 				return Note.save(notes[i]).then((note) => { | ||||
| 					output.push(note); | ||||
| 					return output; | ||||
| 				}); | ||||
| 			}); | ||||
| 		} | ||||
|  | ||||
| 		return promiseChain(chain, []); | ||||
| 	} | ||||
|  | ||||
| 	function createFolders(id = 1) { | ||||
| 		let folders = []; | ||||
| 		if (id === 1) { | ||||
| 			folders.push({ title: 'myfolder1' }); | ||||
| 			folders.push({ title: 'myfolder2' }); | ||||
| 			folders.push({ title: 'myfolder3' }); | ||||
| 		} else { | ||||
| 			throw new Error('Invalid ID: ' + id); | ||||
| 		} | ||||
|  | ||||
| 		let output = []; | ||||
| 		let chain = []; | ||||
| 		for (let i = 0; i < folders.length; i++) { | ||||
| 			chain.push(() => { | ||||
| 				return Folder.save(folders[i]).then((folder) => { | ||||
| 					output.push(folder); | ||||
| 					return output; | ||||
| 				}); | ||||
| 			}); | ||||
| 		} | ||||
|  | ||||
| 		return promiseChain(chain, []); | ||||
| 	} | ||||
|  | ||||
| 	it('should retrieve sync items', function(done) { | ||||
| 		createFolders().then((folders) => { | ||||
| 			return createNotes(folders[0].id); | ||||
| 		}).then(() => { | ||||
| 		createFoldersAndNotes().then(() => { | ||||
| 			return NoteFolderService.itemsThatNeedSync().then((context) => { | ||||
| 				expect(context.items.length).toBe(2); | ||||
| 				expect(context.hasMore).toBe(true); | ||||
|   | ||||
| @@ -1,39 +1,40 @@ | ||||
| import { time } from 'src/time-utils.js'; | ||||
| import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js'; | ||||
| import { createFoldersAndNotes } from 'test-data.js'; | ||||
|  | ||||
| // Note: set 1 matches set 1 of createRemoteItems() | ||||
| function createLocalItems(id, updatedTime, lastSyncTime) { | ||||
| 	let output = []; | ||||
| 	if (id === 1) { | ||||
| 		output.push({ path: 'test', isDir: true, updatedTime: updatedTime, lastSyncTime: lastSyncTime }); | ||||
| 		output.push({ path: 'test/un', updatedTime: updatedTime, lastSyncTime: lastSyncTime }); | ||||
| 	} else { | ||||
| 		throw new Error('Invalid ID'); | ||||
| 	} | ||||
| 	return output; | ||||
| } | ||||
|  | ||||
| function createRemoteItems(id = 1, updatedTime = null) { | ||||
| 	if (!updatedTime) updatedTime = time.unix(); | ||||
|  | ||||
| 	if (id === 1) { | ||||
| 		return fileApi().format() | ||||
| 		.then(() => fileApi().mkdir('test')) | ||||
| 		.then(() => fileApi().put('test/un', 'abcd')) | ||||
| 		.then(() => fileApi().list('', true)) | ||||
| 		.then((items) => { | ||||
| 			for (let i = 0; i < items.length; i++) { | ||||
| 				items[i].updatedTime = updatedTime; | ||||
| 			} | ||||
| 			return items; | ||||
| 		}); | ||||
| 	} else { | ||||
| 		throw new Error('Invalid ID'); | ||||
| 	} | ||||
| } | ||||
|  | ||||
| describe('Synchronizer syncActions', function() { | ||||
|  | ||||
| 	// Note: set 1 matches set 1 of createRemoteItems() | ||||
| 	function createLocalItems(id, updatedTime, lastSyncTime) { | ||||
| 		let output = []; | ||||
| 		if (id === 1) { | ||||
| 			output.push({ path: 'test', isDir: true, updatedTime: updatedTime, lastSyncTime: lastSyncTime }); | ||||
| 			output.push({ path: 'test/un', updatedTime: updatedTime, lastSyncTime: lastSyncTime }); | ||||
| 		} else { | ||||
| 			throw new Error('Invalid ID'); | ||||
| 		} | ||||
| 		return output; | ||||
| 	} | ||||
|  | ||||
| 	function createRemoteItems(id = 1, updatedTime = null) { | ||||
| 		if (!updatedTime) updatedTime = time.unix(); | ||||
|  | ||||
| 		if (id === 1) { | ||||
| 			return fileApi().format() | ||||
| 			.then(() => fileApi().mkdir('test')) | ||||
| 			.then(() => fileApi().put('test/un', 'abcd')) | ||||
| 			.then(() => fileApi().list('', true)) | ||||
| 			.then((items) => { | ||||
| 				for (let i = 0; i < items.length; i++) { | ||||
| 					items[i].updatedTime = updatedTime; | ||||
| 				} | ||||
| 				return items; | ||||
| 			}); | ||||
| 		} else { | ||||
| 			throw new Error('Invalid ID'); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	beforeEach(function(done) { | ||||
| 		setupDatabaseAndSynchronizer(done); | ||||
| 	}); | ||||
| @@ -155,8 +156,20 @@ describe('Synchronizer syncActions', function() { | ||||
| 		}); | ||||
| 	}); | ||||
|  | ||||
| 	it('should sync items', function(done) { | ||||
| 		 | ||||
| }); | ||||
|  | ||||
| describe('Synchronizer start', function() { | ||||
|  | ||||
| 	beforeEach(function(done) { | ||||
| 		setupDatabaseAndSynchronizer(done); | ||||
| 	}); | ||||
|  | ||||
| 	it('should create remote items', function(done) { | ||||
| 		createFoldersAndNotes().then(() => { | ||||
| 			return synchronizer().start(); | ||||
| 		} | ||||
| 	}).then(() => { | ||||
| 		done(); | ||||
| 	}); | ||||
|  | ||||
| }); | ||||
							
								
								
									
										58
									
								
								CliClient/tests/test-data.js
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										58
									
								
								CliClient/tests/test-data.js
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,58 @@ | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { Folder } from 'src/models/folder.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
|  | ||||
| function createNotes(id = 1, parentId) { | ||||
| 	let notes = []; | ||||
| 	if (id === 1) { | ||||
| 		notes.push({ parent_id: parentId, title: 'note one', body: 'content of note one' }); | ||||
| 		notes.push({ parent_id: parentId, title: 'note two', body: 'content of note two' }); | ||||
| 	} else { | ||||
| 		throw new Error('Invalid ID: ' + id); | ||||
| 	} | ||||
|  | ||||
| 	let output = []; | ||||
| 	let chain = []; | ||||
| 	for (let i = 0; i < notes.length; i++) { | ||||
| 		chain.push(() => { | ||||
| 			return Note.save(notes[i]).then((note) => { | ||||
| 				output.push(note); | ||||
| 				return output; | ||||
| 			}); | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	return promiseChain(chain, []); | ||||
| } | ||||
|  | ||||
| function createFolders(id = 1) { | ||||
| 	let folders = []; | ||||
| 	if (id === 1) { | ||||
| 		folders.push({ title: 'myfolder1' }); | ||||
| 		folders.push({ title: 'myfolder2' }); | ||||
| 		folders.push({ title: 'myfolder3' }); | ||||
| 	} else { | ||||
| 		throw new Error('Invalid ID: ' + id); | ||||
| 	} | ||||
|  | ||||
| 	let output = []; | ||||
| 	let chain = []; | ||||
| 	for (let i = 0; i < folders.length; i++) { | ||||
| 		chain.push(() => { | ||||
| 			return Folder.save(folders[i]).then((folder) => { | ||||
| 				output.push(folder); | ||||
| 				return output; | ||||
| 			}); | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	return promiseChain(chain, []); | ||||
| } | ||||
|  | ||||
| function createFoldersAndNotes(id = 1) { | ||||
| 	return createFolders(id).then((folders) => { | ||||
| 		return createNotes(id, folders[0].id); | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| export { createNotes, createFolders, createFoldersAndNotes }; | ||||
| @@ -40,6 +40,12 @@ class BaseModel { | ||||
| 		return this.db().tableFields(this.tableName()); | ||||
| 	} | ||||
|  | ||||
| 	static identifyItemType(item) { | ||||
| 		if ('body' in item || ('parent_id' in item && !!item.parent_id)) return BaseModel.ITEM_TYPE_NOTE; | ||||
| 		if ('sync_time' in item) return BaseModel.ITEM_TYPE_FOLDER; | ||||
| 		throw new Error('Cannot identify item: ' + JSON.stringify(item)); | ||||
| 	} | ||||
|  | ||||
| 	static new() { | ||||
| 		let fields = this.fields(); | ||||
| 		let output = {}; | ||||
| @@ -154,7 +160,8 @@ class BaseModel { | ||||
|  | ||||
| 		queries.push(saveQuery); | ||||
|  | ||||
| 		if (options.trackChanges && this.trackChanges()) { | ||||
| 		// TODO: DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED DISABLED  | ||||
| 		if (0&& ptions.trackChanges && this.trackChanges()) { | ||||
| 			// Cannot import this class the normal way due to cyclical dependencies between Change and BaseModel | ||||
| 			// which are not handled by React Native. | ||||
| 			const { Change } = require('src/models/change.js'); | ||||
|   | ||||
| @@ -1,6 +1,6 @@ | ||||
| import { Log } from 'src/log.js'; | ||||
| import { uuid } from 'src/uuid.js'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { _ } from 'src/locale.js' | ||||
|  | ||||
| const structureSql = ` | ||||
| @@ -106,6 +106,7 @@ class Database { | ||||
| 		this.initialized_ = false; | ||||
| 		this.tableFields_ = null; | ||||
| 		this.driver_ = driver; | ||||
| 		this.inTransaction_ = false; | ||||
| 	} | ||||
|  | ||||
| 	setDebugEnabled(v) { | ||||
| @@ -150,6 +151,33 @@ class Database { | ||||
| 	} | ||||
|  | ||||
| 	transactionExecBatch(queries) { | ||||
| 		if (queries.length <= 0) return Promise.resolve(); | ||||
|  | ||||
| 		if (queries.length == 1) { | ||||
| 			return this.exec(queries[0].sql, queries[0].params); | ||||
| 		} | ||||
|  | ||||
| 		// There can be only one transaction running at a time so queue | ||||
| 		// any new transaction here. | ||||
| 		if (this.inTransaction_) { | ||||
| 			return new Promise((resolve, reject) => { | ||||
| 				let iid = setInterval(() => { | ||||
| 					console.info('Waiting...'); | ||||
| 					if (!this.inTransaction_) { | ||||
| 						console.info('OKKKKKKKKKKK'); | ||||
| 						clearInterval(iid); | ||||
| 						this.transactionExecBatch(queries).then(() => { | ||||
| 							resolve(); | ||||
| 						}).catch((error) => { | ||||
| 							reject(error); | ||||
| 						}); | ||||
| 					} | ||||
| 				}, 100); | ||||
| 			}); | ||||
| 		} | ||||
|  | ||||
| 		this.inTransaction_ = true; | ||||
|  | ||||
| 		queries.splice(0, 0, 'BEGIN TRANSACTION'); | ||||
| 		queries.push('COMMIT'); // Note: ROLLBACK is currently not supported | ||||
|  | ||||
| @@ -161,7 +189,9 @@ class Database { | ||||
| 			}); | ||||
| 		} | ||||
|  | ||||
| 		return promiseChain(chain); | ||||
| 		return promiseChain(chain).then(() => { | ||||
| 			this.inTransaction_ = false; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	static enumId(type, s) { | ||||
| @@ -218,7 +248,11 @@ class Database { | ||||
|  | ||||
| 	logQuery(sql, params = null) { | ||||
| 		if (!this.debugMode()) return; | ||||
| 		Log.debug('DB: ' + sql, params); | ||||
| 		if (params !== null) { | ||||
| 			Log.debug('DB: ' + sql, params); | ||||
| 		} else { | ||||
| 			Log.debug('DB: ' + sql); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	static insertQuery(tableName, data) { | ||||
|   | ||||
| @@ -1,6 +1,6 @@ | ||||
| import fs from 'fs'; | ||||
| import fse from 'fs-extra'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import moment from 'moment'; | ||||
|  | ||||
| class FileApiDriverLocal { | ||||
| @@ -9,10 +9,14 @@ class FileApiDriverLocal { | ||||
| 		return new Promise((resolve, reject) => { | ||||
| 			fs.stat(path, (error, s) => { | ||||
| 				if (error) { | ||||
| 					reject(error); | ||||
| 					if (error.code == 'ENOENT') { | ||||
| 						resolve(null); | ||||
| 					} else { | ||||
| 						reject(error); | ||||
| 					} | ||||
| 					return; | ||||
| 				} | ||||
| 				resolve(s); | ||||
| 				resolve(this.metadataFromStats_(path, s)); | ||||
| 			});			 | ||||
| 		}); | ||||
| 	} | ||||
| @@ -61,8 +65,7 @@ class FileApiDriverLocal { | ||||
| 					chain.push((output) => { | ||||
| 						if (!output) output = []; | ||||
| 						return this.stat(path + '/' + items[i]).then((stat) => { | ||||
| 							let md = this.metadataFromStats_(items[i], stat); | ||||
| 							output.push(md); | ||||
| 							output.push(stat); | ||||
| 							return output;							 | ||||
| 						}); | ||||
| 					}); | ||||
| @@ -82,7 +85,13 @@ class FileApiDriverLocal { | ||||
| 		return new Promise((resolve, reject) => { | ||||
| 			fs.readFile(path, 'utf8', (error, content) => { | ||||
| 				if (error) { | ||||
| 					reject(error); | ||||
| 					if (error.code == 'ENOENT') { | ||||
| 						// Return null in this case so that it's possible to get a file | ||||
| 						// without checking if it exists first. | ||||
| 						resolve(null); | ||||
| 					} else { | ||||
| 						reject(error); | ||||
| 					} | ||||
| 					return; | ||||
| 				} | ||||
| 				return resolve(content); | ||||
|   | ||||
| @@ -1,4 +1,4 @@ | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
|  | ||||
| class FileApi { | ||||
|  | ||||
| @@ -13,9 +13,19 @@ class FileApi { | ||||
| 		return output; | ||||
| 	} | ||||
|  | ||||
| 	list(path = '', recursive = false) { | ||||
| 	listDirectories() { | ||||
| 		return this.driver_.list(this.fullPath_('')).then((items) => { | ||||
| 			let output = []; | ||||
| 			for (let i = 0; i < items.length; i++) { | ||||
| 				if (items[i].isDir) output.push(items[i]); | ||||
| 			} | ||||
| 			return output; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	list(path = '', recursive = false, context = null) { | ||||
| 		let fullPath = this.fullPath_(path); | ||||
| 		return this.driver_.list(fullPath, recursive).then((items) => { | ||||
| 		return this.driver_.list(fullPath).then((items) => { | ||||
| 			if (recursive) { | ||||
| 				let chain = []; | ||||
| 				for (let i = 0; i < items.length; i++) { | ||||
| @@ -47,14 +57,26 @@ class FileApi { | ||||
| 	} | ||||
|  | ||||
| 	mkdir(path) { | ||||
| 		console.info('mkdir ' + path); | ||||
| 		return this.driver_.mkdir(this.fullPath_(path)); | ||||
| 	} | ||||
|  | ||||
| 	stat(path) { | ||||
| 		console.info('stat ' + path); | ||||
| 		return this.driver_.stat(this.fullPath_(path)).then((output) => { | ||||
| 			if (!output) return output; | ||||
| 			output.path = path; | ||||
| 			return output; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	get(path) { | ||||
| 		console.info('get ' + path); | ||||
| 		return this.driver_.get(this.fullPath_(path)); | ||||
| 	} | ||||
|  | ||||
| 	put(path, content) { | ||||
| 		console.info('put ' + path); | ||||
| 		return this.driver_.put(this.fullPath_(path), content); | ||||
| 	} | ||||
|  | ||||
|   | ||||
| @@ -1,6 +1,6 @@ | ||||
| import { BaseModel } from 'src/base-model.js'; | ||||
| import { Log } from 'src/log.js'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { folderItemFilename } from 'src/string-utils.js' | ||||
| import { _ } from 'src/locale.js'; | ||||
|   | ||||
| @@ -1,10 +0,0 @@ | ||||
| function promiseChain(chain, defaultValue = null) { | ||||
| 	let output = new Promise((resolve, reject) => { resolve(defaultValue); }); | ||||
| 	for (let i = 0; i < chain.length; i++) { | ||||
| 		let f = chain[i]; | ||||
| 		output = output.then(f); | ||||
| 	} | ||||
| 	return output; | ||||
| } | ||||
|  | ||||
| export { promiseChain } | ||||
							
								
								
									
										37
									
								
								ReactNativeClient/src/promise-utils.js
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										37
									
								
								ReactNativeClient/src/promise-utils.js
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,37 @@ | ||||
| function promiseChain(chain, defaultValue = null) { | ||||
| 	let output = new Promise((resolve, reject) => { resolve(defaultValue); }); | ||||
| 	for (let i = 0; i < chain.length; i++) { | ||||
| 		let f = chain[i]; | ||||
| 		output = output.then(f); | ||||
| 	} | ||||
| 	return output; | ||||
| } | ||||
|  | ||||
| function promiseWhile(callback) { | ||||
| 	let isDone = false; | ||||
|  | ||||
| 	function done() { | ||||
| 		isDone = true; | ||||
| 	} | ||||
|  | ||||
| 	let iterationDone = false; | ||||
| 	let p = callback(done).then(() => { | ||||
| 		iterationDone = true; | ||||
| 	}); | ||||
|  | ||||
| 	let iid = setInterval(() => { | ||||
| 		if (iterationDone) { | ||||
| 			if (isDone) { | ||||
| 				clearInterval(iid); | ||||
| 				return; | ||||
| 			} | ||||
|  | ||||
| 			iterationDone = false; | ||||
| 			callback(done).then(() => { | ||||
| 				iterationDone = true; | ||||
| 			}); | ||||
| 		} | ||||
| 	}, 100); | ||||
| } | ||||
|  | ||||
| export { promiseChain, promiseWhile } | ||||
| @@ -74,33 +74,35 @@ class NoteFolderService extends BaseService { | ||||
| 	} | ||||
|  | ||||
| 	static itemsThatNeedSync(context = null, limit = 100) { | ||||
| 		let now = time.unix(); | ||||
|  | ||||
| 		if (!context) { | ||||
| 			context = { | ||||
| 				hasMoreNotes: true, | ||||
| 				hasMoreFolders: true, | ||||
| 				hasMoreNotes: true, | ||||
| 				noteOffset: 0, | ||||
| 				folderOffset: 0, | ||||
| 				hasMore: true, | ||||
| 				items: [], | ||||
| 			}; | ||||
| 		} | ||||
|  | ||||
| 		if (context.hasMoreNotes) { | ||||
| 			return BaseModel.db().selectAll('SELECT * FROM notes WHERE sync_time < ? LIMIT ' + limit + ' OFFSET ' + context.noteOffset, [now]).then((items) => { | ||||
| 				context.items = items; | ||||
| 				context.hasMoreNotes = items.length >= limit; | ||||
| 				context.noteOffset += items.length; | ||||
| 				return context; | ||||
| 		context.folderOffset = 0; | ||||
| 		context.noteOffset = 0; | ||||
|  | ||||
| 		// Process folder first, then notes so that folders are created before | ||||
| 		// adding notes to them. However, it will be the opposite when deleting | ||||
| 		// folders (TODO). | ||||
|  | ||||
| 		if (context.hasMoreFolders) { | ||||
| 			return BaseModel.db().selectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit + ' OFFSET ' + context.folderOffset).then((items) => { | ||||
| 				context.hasMoreFolders = items.length >= limit; | ||||
| 				context.folderOffset += items.length; | ||||
| 				return { context: context, items: items }; | ||||
| 			}); | ||||
| 		} else { | ||||
| 			return BaseModel.db().selectAll('SELECT * FROM folders WHERE sync_time < ? LIMIT ' + limit + ' OFFSET ' + context.folderOffset, [now]).then((items) => { | ||||
| 				context.items = items; | ||||
| 				context.hasMoreFolders = items.length >= limit; | ||||
| 				context.hasMore = context.hasMoreFolders; | ||||
| 				context.folderOffset += items.length; | ||||
| 				return context; | ||||
| 			return BaseModel.db().selectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit + ' OFFSET ' + context.noteOffset).then((items) => { | ||||
| 				context.hasMoreNotes = items.length >= limit; | ||||
| 				context.noteOffset += items.length; | ||||
| 				context.hasMore = context.hasMoreNotes; | ||||
| 				return { context: context, items: items }; | ||||
| 			}); | ||||
| 		} | ||||
| 	} | ||||
|   | ||||
| @@ -4,7 +4,10 @@ import { Change } from 'src/models/change.js'; | ||||
| import { Folder } from 'src/models/folder.js'; | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { BaseModel } from 'src/base-model.js'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { NoteFolderService } from 'src/services/note-folder-service.js'; | ||||
| import { time } from 'src/time-utils.js'; | ||||
| //import { promiseWhile } from 'src/promise-utils.js'; | ||||
| import moment from 'moment'; | ||||
|  | ||||
| const fs = require('fs'); | ||||
| @@ -88,14 +91,6 @@ class Synchronizer { | ||||
| 		return null; | ||||
| 	} | ||||
|  | ||||
| 	syncAction(actionType, where, item, isConflict) { | ||||
| 		return { | ||||
| 			type: actionType, | ||||
| 			where: where, | ||||
| 			item: item, | ||||
| 		}; | ||||
| 	} | ||||
|  | ||||
| 	itemIsSameDate(item, date) { | ||||
| 		return Math.abs(item.updatedTime - date) <= 1; | ||||
| 	} | ||||
| @@ -110,9 +105,45 @@ class Synchronizer { | ||||
| 		return item.updatedTime < date; | ||||
| 	} | ||||
|  | ||||
| 	dbItemToSyncItem(dbItem) { | ||||
| 		let p = Promise.resolve(null); | ||||
| 		let itemType = BaseModel.identifyItemType(dbItem); | ||||
| 		let ItemClass = null; | ||||
|  | ||||
| 		if (itemType == BaseModel.ITEM_TYPE_NOTE) { | ||||
| 			ItemClass = Note; | ||||
| 			p = Folder.load(dbItem.parent_id); | ||||
| 		} else { | ||||
| 			ItemClass = Folder; | ||||
| 		} | ||||
|  | ||||
| 		return p.then((dbParent) => { | ||||
| 			let path = ItemClass.systemPath(dbParent, dbItem); | ||||
| 			return { | ||||
| 				isDir: itemType == BaseModel.ITEM_TYPE_FOLDER, | ||||
| 				path: path, | ||||
| 				syncTime: dbItem.sync_time, | ||||
| 				updatedTime: dbItem.updated_time, | ||||
| 				dbParent: dbParent, | ||||
| 				dbItem: dbItem, | ||||
| 			}; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	syncAction(localItem, remoteItem, deletedLocalPaths) { | ||||
| 		let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths); | ||||
| 		if (output.length !== 1) throw new Error('Invalid number of actions returned'); | ||||
| 		return output[0]; | ||||
| 	} | ||||
|  | ||||
| 	// Assumption: it's not possible to, for example, have a directory one the dest | ||||
| 	// and a file with the same name on the source. It's not possible because the | ||||
| 	// file and directory names are UUID so should be unique. | ||||
| 	// Each item must have these properties: | ||||
| 	// - path | ||||
| 	// - isDir | ||||
| 	// - syncTime | ||||
| 	// - updatedTime | ||||
| 	syncActions(localItems, remoteItems, deletedLocalPaths) { | ||||
| 		let output = []; | ||||
| 		let donePaths = []; | ||||
| @@ -127,7 +158,7 @@ class Synchronizer { | ||||
| 			}; | ||||
|  | ||||
| 			if (!remote) { | ||||
| 				if (local.lastSyncTime) { | ||||
| 				if (local.syncTime) { | ||||
| 					// The item has been synced previously and now is no longer in the dest | ||||
| 					// which means it has been deleted. | ||||
| 					action.type = 'delete'; | ||||
| @@ -139,9 +170,9 @@ class Synchronizer { | ||||
| 					action.dest = 'remote'; | ||||
| 				} | ||||
| 			} else { | ||||
| 				if (this.itemIsOlderThan(local, local.lastSyncTime)) continue; | ||||
| 				if (this.itemIsOlderThan(local, local.syncTime)) continue; | ||||
|  | ||||
| 				if (this.itemIsOlderThan(remote, local.lastSyncTime)) { | ||||
| 				if (this.itemIsOlderThan(remote, local.syncTime)) { | ||||
| 					action.type = 'update'; | ||||
| 					action.dest = 'remote'; | ||||
| 				} else { | ||||
| @@ -188,7 +219,7 @@ class Synchronizer { | ||||
| 					action.dest = 'local'; | ||||
| 				} | ||||
| 			} else { | ||||
| 				if (this.itemIsOlderThan(remote, local.lastSyncTime)) continue; // Already have this version | ||||
| 				if (this.itemIsOlderThan(remote, local.syncTime)) continue; // Already have this version | ||||
| 				// Note: no conflict is possible here since if the local item has been | ||||
| 				// modified since the last sync, it's been processed in the previous loop. | ||||
| 				action.type = 'update'; | ||||
| @@ -201,12 +232,6 @@ class Synchronizer { | ||||
| 		return output; | ||||
| 	} | ||||
|  | ||||
| 	processSyncActions(syncActions) { | ||||
| 		for (let i = 0; i < syncActions.length; i++) { | ||||
| 			 | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	processState_uploadChanges() { | ||||
| 		let remoteFiles = []; | ||||
| 		let processedChangeIds = []; | ||||
| @@ -548,16 +573,111 @@ class Synchronizer { | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	processSyncAction(action) { | ||||
| 		// console.info(action); | ||||
|  | ||||
| 		if (action.type == 'conflict') { | ||||
|  | ||||
| 		} else { | ||||
| 			let item = action[action.dest == 'local' ? 'remote' : 'local']; | ||||
| 			let ItemClass = null; | ||||
| 			if (item.isDir) { | ||||
| 				ItemClass = Folder; | ||||
| 			} else { | ||||
| 				ItemClass = Note; | ||||
| 			} | ||||
| 			let path = ItemClass.systemPath(item.dbParent, item.dbItem); | ||||
|  | ||||
| 			if (action.type == 'create') { | ||||
| 				if (action.dest == 'remote') { | ||||
| 					if (item.isDir) { | ||||
| 						return this.api().mkdir(path); | ||||
| 					} else { | ||||
| 						return this.api().put(path, Note.toFriendlyString(item.dbItem)); | ||||
| 					} | ||||
| 				} | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		return Promise.resolve(); // TODO | ||||
| 	} | ||||
|  | ||||
| 	processLocalItem(dbItem) { | ||||
| 		//console.info(dbItem); | ||||
| 		let localItem = null; | ||||
| 		return this.dbItemToSyncItem(dbItem).then((r) => { | ||||
| 			localItem = r; | ||||
| 			return this.api().stat(localItem.path); | ||||
| 		}).then((remoteItem) => { | ||||
| 			let action = this.syncAction(localItem, remoteItem, []); | ||||
| 			//console.info(action); | ||||
| 			return this.processSyncAction(action); | ||||
| 		}).then(() => { | ||||
| 			dbItem.sync_time = time.unix(); | ||||
| 			if (localItem.isDir) { | ||||
| 				return Folder.save(dbItem); | ||||
| 			} else { | ||||
| 				return Note.save(dbItem); | ||||
| 			} | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	start() { | ||||
| 		Log.info('Sync: start'); | ||||
|  | ||||
| 		if (this.state() != 'idle') { | ||||
| 			Log.info("Sync: cannot start synchronizer because synchronization already in progress. State: " + this.state()); | ||||
| 			return; | ||||
| 			return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state()); | ||||
| 		} | ||||
|  | ||||
| 		this.state_ = 'started'; | ||||
|  | ||||
| 		return this.api().listDirectories().then((items) => { | ||||
| 			var context = null; | ||||
| 			let limit = 2; | ||||
| 			let finishedReading = false; | ||||
| 			let isReading = false; | ||||
| 			 | ||||
| 			let readItems = () => { | ||||
| 				isReading = true; | ||||
| 				return NoteFolderService.itemsThatNeedSync(context, limit).then((result) => { | ||||
| 					context = result.context; | ||||
|  | ||||
| 					let chain = []; | ||||
| 					for (let i = 0; i < result.items.length; i++) { | ||||
| 						let item = result.items[i]; | ||||
| 						console.info(JSON.stringify(item)); | ||||
| 						chain.push(() => { | ||||
| 							//return Promise.resolve(); | ||||
| 							return this.processLocalItem(item); | ||||
| 						}); | ||||
| 					} | ||||
|  | ||||
| 					return promiseChain(chain).then(() => { | ||||
| 						console.info('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'); | ||||
| 						if (!context.hasMore) finishedReading = true; | ||||
| 						isReading = false; | ||||
| 					}); | ||||
| 				}).catch((error) => { | ||||
| 					console.error(error); | ||||
| 					throw error; | ||||
| 				}); | ||||
| 			} | ||||
|  | ||||
| 			let iid = setInterval(() => { | ||||
| 				if (isReading) return; | ||||
| 				if (finishedReading) { | ||||
| 					clearInterval(iid); | ||||
| 					return; | ||||
| 				} | ||||
| 				readItems(); | ||||
| 			}, 100); | ||||
|  | ||||
| 		}).then(() => { | ||||
| 			this.state_ = 'idle'; | ||||
| 		}); | ||||
|  | ||||
| 		//return NoteFolderService.itemsThatNeedSync | ||||
|  | ||||
| 		 | ||||
|  | ||||
| 		// if (!this.api().session()) { | ||||
| @@ -568,6 +688,8 @@ class Synchronizer { | ||||
| 		//return this.processState('uploadChanges'); | ||||
| 	} | ||||
|  | ||||
| 	 | ||||
|  | ||||
| } | ||||
|  | ||||
| export { Synchronizer }; | ||||
| @@ -4,7 +4,7 @@ import { Change } from 'src/models/change.js'; | ||||
| import { Folder } from 'src/models/folder.js'; | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { BaseModel } from 'src/base-model.js'; | ||||
| import { promiseChain } from 'src/promise-chain.js'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
|  | ||||
| class Synchronizer { | ||||
|  | ||||
|   | ||||
		Reference in New Issue
	
	Block a user