You've already forked joplin
							
							
				mirror of
				https://github.com/laurent22/joplin.git
				synced 2025-10-31 00:07:48 +02:00 
			
		
		
		
	Improved sync reporting
This commit is contained in:
		| @@ -28,7 +28,7 @@ const fs = require('fs'); | ||||
| Log.setLevel(Log.LEVEL_DEBUG); | ||||
|  | ||||
| let db = new Database(new DatabaseDriverNode()); | ||||
| db.setDebugEnabled(true); | ||||
| db.setDebugMode(true); | ||||
| db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => { | ||||
| 	return db.selectAll('SELECT * FROM table_fields'); | ||||
| }).then((rows) => { | ||||
|   | ||||
| @@ -3,12 +3,12 @@ require('app-module-path').addPath(__dirname); | ||||
| import { uuid } from 'src/uuid.js'; | ||||
| import moment from 'moment'; | ||||
| import { promiseChain } from 'src/promise-utils.js'; | ||||
| import { WebApi } from 'src/web-api.js' | ||||
| import { folderItemFilename } from 'src/string-utils.js' | ||||
| import { BaseModel } from 'src/base-model.js'; | ||||
| import { Note } from 'src/models/note.js'; | ||||
| import { Folder } from 'src/models/folder.js'; | ||||
| import jsSHA from "jssha"; | ||||
|  | ||||
| let webApi = new WebApi('http://joplin.local'); | ||||
|  | ||||
| const Promise = require('promise'); | ||||
| const fs = require('fs'); | ||||
| const stringToStream = require('string-to-stream') | ||||
| @@ -398,61 +398,6 @@ function enexXmlToMd(stream, resources) { | ||||
| 	}); | ||||
| } | ||||
|  | ||||
|  | ||||
|  | ||||
| // const path = require('path'); | ||||
|  | ||||
| // var walk = function (dir, done) { | ||||
| // 	fs.readdir(dir, function (error, list) { | ||||
| // 		if (error) return done(error); | ||||
| // 		var i = 0; | ||||
| // 		(function next () { | ||||
| // 			var file = list[i++]; | ||||
|  | ||||
| // 			if (!file) return done(null);             | ||||
| // 			file = dir + '/' + file; | ||||
| 			 | ||||
| // 			fs.stat(file, function (error, stat) { | ||||
| // 				if (stat && stat.isDirectory()) { | ||||
| // 					walk(file, function (error) { | ||||
| // 						next(); | ||||
| // 					}); | ||||
| // 				} else { | ||||
| // 					if (path.basename(file) != 'sample4.xml') { | ||||
| // 						next(); | ||||
| // 						return; | ||||
| // 					} | ||||
|  | ||||
| // 					if (path.extname(file) == '.xml') { | ||||
| // 						console.info('Processing: ' + file); | ||||
| // 						let stream = fs.createReadStream(file); | ||||
| // 						enexXmlToMd(stream).then((md) => { | ||||
| // 							console.info(md); | ||||
| // 							console.info(processMdArrayNewLines(md)); | ||||
| // 							next(); | ||||
| // 						}).catch((error) => { | ||||
| // 							console.error(error); | ||||
| // 							return done(error); | ||||
| // 						}); | ||||
| // 					} else { | ||||
| // 						next(); | ||||
| // 					} | ||||
| // 				} | ||||
| // 			}); | ||||
| // 		})(); | ||||
| // 	}); | ||||
| // }; | ||||
|  | ||||
| // walk('/home/laurent/Dropbox/Samples/', function(error) { | ||||
| // 	if (error) { | ||||
| // 		throw error; | ||||
| // 	} else { | ||||
| // 		console.log('-------------------------------------------------------------'); | ||||
| // 		console.log('finished.'); | ||||
| // 		console.log('-------------------------------------------------------------'); | ||||
| // 	} | ||||
| // }); | ||||
|  | ||||
| function isBlockTag(n) { | ||||
| 	return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody"; | ||||
| } | ||||
| @@ -495,12 +440,31 @@ function xmlNodeText(xmlNode) { | ||||
| 	return xmlNode[0]; | ||||
| } | ||||
|  | ||||
| let existingTimestamps = []; | ||||
|  | ||||
| function uniqueCreatedTimestamp(timestamp) { | ||||
| 	if (existingTimestamps.indexOf(timestamp) < 0) { | ||||
| 		existingTimestamps.push(timestamp); | ||||
| 		return timestamp; | ||||
| 	} | ||||
|  | ||||
| 	for (let i = 1; i <= 999; i++) { | ||||
| 		let t = timestamp + i; | ||||
| 		if (existingTimestamps.indexOf(t) < 0) { | ||||
| 			existingTimestamps.push(t); | ||||
| 			return t; | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return timestamp; | ||||
| } | ||||
|  | ||||
| function dateToTimestamp(s) { | ||||
| 	let m = moment(s, 'YYYYMMDDTHHmmssZ'); | ||||
| 	if (!m.isValid()) { | ||||
| 		throw new Error('Invalid date: ' + s); | ||||
| 	} | ||||
| 	return Math.round(m.toDate().getTime() / 1000); | ||||
| 	return m.toDate().getTime(); | ||||
| } | ||||
|  | ||||
| function evernoteXmlToMdArray(xml) { | ||||
| @@ -514,61 +478,6 @@ function extractRecognitionObjId(recognitionXml) { | ||||
| 	return r && r.length >= 2 ? r[1] : null; | ||||
| } | ||||
|  | ||||
| function saveNoteToWebApi(note) { | ||||
| 	let data = Object.assign({}, note); | ||||
| 	delete data.resources; | ||||
| 	delete data.tags; | ||||
|  | ||||
| 	webApi.post('notes', null, data).then((r) => { | ||||
| 		//console.info(r); | ||||
| 	}).catch((error) => { | ||||
| 		console.error("Error for note: " + note.title); | ||||
| 		console.error(error); | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| function noteserialize_format(propName, propValue) { | ||||
| 	if (['created_time', 'updated_time'].indexOf(propName) >= 0) { | ||||
| 		if (!propValue) return ''; | ||||
| 		propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss'); | ||||
| 	} else if (propValue === null || propValue === undefined) { | ||||
| 		propValue = ''; | ||||
| 	} | ||||
|  | ||||
| 	return propValue; | ||||
| } | ||||
|  | ||||
| function noteserialize(note) { | ||||
| 	let shownKeys = ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time']; | ||||
| 	let output = []; | ||||
|  | ||||
| 	output.push(note.title); | ||||
| 	output.push(""); | ||||
| 	output.push(note.body); | ||||
| 	output.push(''); | ||||
| 	for (let i = 0; i < shownKeys.length; i++) { | ||||
| 		let v = note[shownKeys[i]]; | ||||
| 		v = noteserialize_format(shownKeys[i], v); | ||||
| 		output.push(shownKeys[i] + ': ' + v); | ||||
| 	} | ||||
|  | ||||
| 	return output.join("\n"); | ||||
| } | ||||
|  | ||||
| // function folderItemFilename(item) { | ||||
| // 	let output = escapeFilename(item.title).trim(); | ||||
| // 	if (!output.length) output = '_'; | ||||
| // 	return output + '.' + item.id.substr(0, 7); | ||||
| // } | ||||
|  | ||||
| function noteFilename(note) { | ||||
| 	return folderItemFilename(note) + '.md'; | ||||
| } | ||||
|  | ||||
| function folderFilename(folder) { | ||||
| 	return folderItemFilename(folder); | ||||
| } | ||||
|  | ||||
| function filePutContents(filePath, content) { | ||||
| 	return new Promise((resolve, reject) => { | ||||
| 		fs.writeFile(filePath, content, function(error) { | ||||
| @@ -614,28 +523,15 @@ function createDirectory(path) { | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| const baseNoteDir = '/home/laurent/Temp/TestImport'; | ||||
|  | ||||
| // createDirectory('/home/laurent/Temp/TestImport').then(() => { | ||||
| // 	console.info('OK'); | ||||
| // }).catch((error) => { | ||||
| // 	console.error(error); | ||||
| // }); | ||||
|  | ||||
| function saveNoteToDisk(folder, note) { | ||||
| 	const noteContent = noteserialize(note); | ||||
| 	const notePath = baseNoteDir + '/' + folderFilename(folder) + '/' + noteFilename(note); | ||||
|  | ||||
| 	// console.info('==================================================='); | ||||
| 	// console.info(note);//noteContent); | ||||
| 	return filePutContents(notePath, noteContent).then(() => { | ||||
| 		return setModifiedTime(notePath, note.updated_time ? note.updated_time : note.created_time); | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| function saveFolderToDisk(folder) { | ||||
| 	let path = baseNoteDir + '/' + folderFilename(folder); | ||||
| 	return createDirectory(path); | ||||
| function removeUndefinedProperties(note) { | ||||
| 	let output = {}; | ||||
| 	for (let n in note) { | ||||
| 		if (!note.hasOwnProperty(n)) continue; | ||||
| 		let v = note[n]; | ||||
| 		if (v === undefined || v === null) continue; | ||||
| 		output[n] = v; | ||||
| 	} | ||||
| 	return output; | ||||
| } | ||||
|  | ||||
| function createNoteId(note) { | ||||
| @@ -645,7 +541,56 @@ function createNoteId(note) { | ||||
| 	return hash.substr(0, 32); | ||||
| } | ||||
|  | ||||
| function importEnex(parentFolder, stream) { | ||||
| async function fuzzyMatch(note) { | ||||
| 	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time); | ||||
| 	if (!notes.length) return null; | ||||
| 	if (notes.length === 1) return notes[0]; | ||||
|  | ||||
| 	for (let i = 0; i < notes.length; i++) { | ||||
| 		if (notes[i].title == note.title && note.title.trim() != '') return notes[i]; | ||||
| 	} | ||||
|  | ||||
| 	for (let i = 0; i < notes.length; i++) { | ||||
| 		if (notes[i].body == note.body && note.body.trim() != '') return notes[i]; | ||||
| 	} | ||||
|  | ||||
| 	return null; | ||||
| } | ||||
|  | ||||
| async function saveNoteToDb(note) { | ||||
| 	note = Note.filter(note); | ||||
| 	let existingNote = await fuzzyMatch(note); | ||||
|  | ||||
| 	if (existingNote) { | ||||
| 		let diff = BaseModel.diffObjects(existingNote, note); | ||||
| 		delete diff.tags; | ||||
| 		delete diff.resources; | ||||
| 		delete diff.id; | ||||
|  | ||||
| 		// console.info('======================================'); | ||||
| 		// console.info(note); | ||||
| 		// console.info(existingNote); | ||||
| 		// console.info(diff); | ||||
| 		// console.info('======================================'); | ||||
|  | ||||
| 		if (!Object.getOwnPropertyNames(diff).length) return; | ||||
|  | ||||
| 		diff.id = existingNote.id; | ||||
| 		diff.type_ = existingNote.type_; | ||||
| 		return Note.save(diff, { autoTimestamp: false }); | ||||
| 	} else { | ||||
|  | ||||
| 		console.info('NNNNNNNNNNNNNNNNN4'); | ||||
| 		// return Note.save(note, { | ||||
| 		// 	isNew: true, | ||||
| 		// 	autoTimestamp: false, | ||||
| 		// }); | ||||
| 	} | ||||
| } | ||||
|  | ||||
| function importEnex(db, parentFolderId, resourceDir, filePath) { | ||||
| 	let stream = fs.createReadStream(filePath); | ||||
|  | ||||
| 	return new Promise((resolve, reject) => { | ||||
| 		let options = {}; | ||||
| 		let strict = true; | ||||
| @@ -659,6 +604,10 @@ function importEnex(parentFolder, stream) { | ||||
| 		let noteResourceRecognition = null; | ||||
| 		let notes = []; | ||||
|  | ||||
| 		stream.on('error', (error) => { | ||||
| 			reject(new Error(error.toString())); | ||||
| 		}); | ||||
|  | ||||
| 		function currentNodeName() { | ||||
| 			if (!nodes.length) return null; | ||||
| 			return nodes[nodes.length - 1].name; | ||||
| @@ -688,28 +637,14 @@ function importEnex(parentFolder, stream) { | ||||
| 							firstAttachment = false; | ||||
| 						} | ||||
|  | ||||
| 						note.parent_id = parentFolder.id; | ||||
| 						note.parent_id = parentFolderId; | ||||
| 						note.body = processMdArrayNewLines(result.lines); | ||||
| 						note.id = uuid.create(); | ||||
|  | ||||
| 						saveNoteToDisk(parentFolder, note); | ||||
| 						return saveNoteToDb(note); | ||||
|  | ||||
| 						// console.info(noteserialize(note)); | ||||
| 						// console.info('========================================================================================================================='); | ||||
|  | ||||
| 						//saveNoteToWebApi(note); | ||||
|  | ||||
| 						// console.info('======== NOTE ============================================================================'); | ||||
| 						// let c = note.content; | ||||
| 						// delete note.content | ||||
| 						// console.info(note); | ||||
| 						// console.info('------------------------------------------------------------------------------------------'); | ||||
| 						// console.info(c); | ||||
|  | ||||
| 						// if (note.resources.length) { | ||||
| 						// 	console.info('========================================================='); | ||||
| 						// 	console.info(note.content); | ||||
| 						// } | ||||
| 						// SAVE NOTE HERE | ||||
| 						// saveNoteToDisk(parentFolder, note); | ||||
| 					}); | ||||
| 				}); | ||||
| 			} | ||||
| @@ -718,7 +653,7 @@ function importEnex(parentFolder, stream) { | ||||
| 		} | ||||
|  | ||||
| 		saxStream.on('error', function(e) { | ||||
| 			reject(e); | ||||
| 			reject(new Error(e.toString())); | ||||
| 		}) | ||||
|  | ||||
| 		saxStream.on('text', function(text) { | ||||
| @@ -738,7 +673,7 @@ function importEnex(parentFolder, stream) { | ||||
| 				if (n == 'title') { | ||||
| 					note.title = text; | ||||
| 				} else if (n == 'created') { | ||||
| 					note.created_time = dateToTimestamp(text); | ||||
| 					note.created_time = uniqueCreatedTimestamp(dateToTimestamp(text)); | ||||
| 				} else if (n == 'updated') { | ||||
| 					note.updated_time = dateToTimestamp(text); | ||||
| 				} else if (n == 'tag') { | ||||
| @@ -783,6 +718,8 @@ function importEnex(parentFolder, stream) { | ||||
| 			nodes.pop(); | ||||
|  | ||||
| 			if (n == 'note') { | ||||
| 				note = removeUndefinedProperties(note); | ||||
|  | ||||
| 				notes.push(note); | ||||
| 				if (notes.length >= 10) { | ||||
| 					stream.pause(); | ||||
| @@ -822,7 +759,7 @@ function importEnex(parentFolder, stream) { | ||||
| 					filename: noteResource.filename, | ||||
| 				}; | ||||
|  | ||||
| 				r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE  | ||||
| 				// r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE  | ||||
|  | ||||
| 				note.resources.push(r); | ||||
| 				noteResource = null; | ||||
| @@ -837,45 +774,4 @@ function importEnex(parentFolder, stream) { | ||||
| 	}); | ||||
| } | ||||
|  | ||||
| // TODO: make it persistent and random | ||||
| const clientId = 'AB78AB78AB78AB78AB78AB78AB78AB78'; | ||||
|  | ||||
| const folderTitle = 'Laurent'; | ||||
| //const folderTitle = 'Voiture'; | ||||
|  | ||||
| webApi.post('sessions', null, { | ||||
| 	email: 'laurent@cozic.net', | ||||
| 	password: '12345678', | ||||
| 	client_id: clientId, | ||||
| }).then((session) => { | ||||
| 	webApi.setSession(session.id); | ||||
| 	console.info('Got session: ' + session.id); | ||||
| 	return webApi.get('folders'); | ||||
| }).then((folders) => { | ||||
| 	 | ||||
| 	let folder = null; | ||||
|  | ||||
| 	for (let i = 0; i < folders.length; i++) { | ||||
| 		if (folders[i].title = folderTitle) { | ||||
| 			folder = folders[i]; | ||||
| 			break; | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return folder ? Promise.resolve(folder) : webApi.post('folders', null, { title: folderTitle }); | ||||
| }).then((folder) => { | ||||
| 	return saveFolderToDisk(folder).then(() => { | ||||
| 		return folder; | ||||
| 	}); | ||||
| }).then((folder) => { | ||||
| 	let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/' + folderTitle + '.enex'); | ||||
| 	//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/afaire.enex'); | ||||
| 	//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/testtags.enex'); | ||||
| 	importEnex(folder, fileStream).then(() => { | ||||
| 		//console.info('DONE IMPORTING'); | ||||
| 	}).catch((error) => { | ||||
| 		console.error('Cannot import', error); | ||||
| 	}); | ||||
| }).catch((error) => { | ||||
| 	console.error(error); | ||||
| }); | ||||
| export { importEnex }; | ||||
| @@ -13,12 +13,14 @@ import { Synchronizer } from 'src/synchronizer.js'; | ||||
| import { Logger } from 'src/logger.js'; | ||||
| import { uuid } from 'src/uuid.js'; | ||||
| import { sprintf } from 'sprintf-js'; | ||||
| import { importEnex } from 'import-enex'; | ||||
| import { _ } from 'src/locale.js'; | ||||
| import os from 'os'; | ||||
| import fs from 'fs-extra'; | ||||
|  | ||||
| const APPNAME = 'joplin'; | ||||
| const dataDir = os.homedir() + '/.local/share/' + APPNAME; | ||||
| const resourceDir = dataDir + '/resources'; | ||||
|  | ||||
| process.on('unhandledRejection', (reason, p) => { | ||||
| 	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); | ||||
| @@ -28,22 +30,50 @@ const logger = new Logger(); | ||||
| logger.addTarget('file', { path: dataDir + '/log.txt' }); | ||||
| logger.setLevel(Logger.LEVEL_DEBUG); | ||||
|  | ||||
| const dbLogger = new Logger(); | ||||
| dbLogger.addTarget('file', { path: dataDir + '/log-database.txt' }); | ||||
| dbLogger.setLevel(Logger.LEVEL_DEBUG); | ||||
|  | ||||
| const syncLogger = new Logger(); | ||||
| syncLogger.addTarget('file', { path: dataDir + '/log-sync.txt' }); | ||||
| syncLogger.setLevel(Logger.LEVEL_DEBUG); | ||||
|  | ||||
| let db = new Database(new DatabaseDriverNode()); | ||||
| db.setLogger(logger); | ||||
| db.setDebugMode(true); | ||||
| db.setLogger(dbLogger); | ||||
|  | ||||
| let synchronizer_ = null; | ||||
| const vorpal = require('vorpal')(); | ||||
|  | ||||
| async function main() { | ||||
| 	await fs.mkdirp(dataDir, 0o755); | ||||
| 	await fs.mkdirp(resourceDir, 0o755); | ||||
|  | ||||
| 	await db.open({ name: dataDir + '/database.sqlite' }); | ||||
| 	await db.open({ name: dataDir + '/database2.sqlite' }); | ||||
| 	BaseModel.db_ = db; | ||||
| 	await Setting.load(); | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| 	// console.info('DELETING ALL DATA'); | ||||
| 	// await db.exec('DELETE FROM notes'); | ||||
| 	// await db.exec('DELETE FROM changes'); | ||||
| 	// await db.exec('DELETE FROM folders'); | ||||
| 	// await db.exec('DELETE FROM resources'); | ||||
| 	// await db.exec('DELETE FROM deleted_items'); | ||||
| 	// await db.exec('DELETE FROM tags'); | ||||
| 	// await db.exec('DELETE FROM note_tags'); | ||||
|  | ||||
| 	// // let folder = await Folder.save({ title: 'test' }); | ||||
| 	// let folder = await Folder.loadByField('title', 'test'); | ||||
| 	// await importEnex(db, folder.id, resourceDir, '/mnt/c/Users/Laurent/Desktop/Laurent.enex'); //'/mnt/c/Users/Laurent/Desktop/Laurent.enex'); | ||||
| 	// return; | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| 	let commands = []; | ||||
| 	let currentFolder = null; | ||||
|  | ||||
| @@ -383,6 +413,15 @@ async function main() { | ||||
| 		}, | ||||
| 	}); | ||||
|  | ||||
| 	commands.push({ | ||||
| 		usage: 'import-enex', | ||||
| 		description: _('Imports a .enex file (Evernote export file).'), | ||||
| 		action: function (args, end) { | ||||
| 			 | ||||
| 			end(); | ||||
| 		}, | ||||
| 	}); | ||||
|  | ||||
| 	for (let i = 0; i < commands.length; i++) { | ||||
| 		let c = commands[i]; | ||||
| 		let o = vorpal.command(c.usage, c.description); | ||||
|   | ||||
| @@ -63,7 +63,7 @@ function setupDatabase(id = null) { | ||||
| 		// Don't care if the file doesn't exist | ||||
| 	}).then(() => { | ||||
| 		databases_[id] = new Database(new DatabaseDriverNode()); | ||||
| 		databases_[id].setDebugEnabled(false); | ||||
| 		databases_[id].setDebugMode(false); | ||||
| 		return databases_[id].open({ name: filePath }).then(() => { | ||||
| 			BaseModel.db_ = databases_[id]; | ||||
| 			return setupDatabase(id); | ||||
|   | ||||
| @@ -109,6 +109,12 @@ class BaseModel { | ||||
| 		return options; | ||||
| 	} | ||||
|  | ||||
| 	static count() { | ||||
| 		return this.db().selectOne('SELECT count(*) as total FROM `' + this.tableName() + '`').then((r) => { | ||||
| 			return r ? r['total'] : 0; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	static load(id) { | ||||
| 		return this.loadByField('id', id); | ||||
| 	} | ||||
| @@ -116,19 +122,19 @@ class BaseModel { | ||||
| 	static modelSelectOne(sql, params = null) { | ||||
| 		if (params === null) params = []; | ||||
| 		return this.db().selectOne(sql, params).then((model) => { | ||||
| 			return this.addModelMd(model); | ||||
| 			return this.filter(this.addModelMd(model)); | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	static modelSelectAll(sql, params = null) { | ||||
| 		if (params === null) params = []; | ||||
| 		return this.db().selectAll(sql, params).then((models) => { | ||||
| 			return this.addModelMd(models); | ||||
| 			return this.filterArray(this.addModelMd(models)); | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	static loadByField(fieldName, fieldValue) {	 | ||||
| 		return this.modelSelectOne('SELECT * FROM ' + this.tableName() + ' WHERE `' + fieldName + '` = ?', [fieldValue]); | ||||
| 		return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]); | ||||
| 	} | ||||
|  | ||||
| 	static applyPatch(model, patch) { | ||||
| @@ -200,9 +206,10 @@ class BaseModel { | ||||
|  | ||||
| 	static save(o, options = null) { | ||||
| 		options = this.modOptions(options); | ||||
|  | ||||
| 		options.isNew = options.isNew == 'auto' ? !o.id : options.isNew; | ||||
|  | ||||
| 		o = this.filter(o); | ||||
|  | ||||
| 		let queries = []; | ||||
| 		let saveQuery = this.saveQuery(o, options); | ||||
| 		let itemId = saveQuery.id; | ||||
| @@ -242,7 +249,7 @@ class BaseModel { | ||||
| 			o = Object.assign({}, o); | ||||
| 			o.id = itemId; | ||||
| 			o = this.addModelMd(o); | ||||
| 			return o; | ||||
| 			return this.filter(o); | ||||
| 		}).catch((error) => { | ||||
| 			Log.error('Cannot save model', error); | ||||
| 		}); | ||||
| @@ -256,6 +263,18 @@ class BaseModel { | ||||
| 		return this.db().exec('DELETE FROM deleted_items WHERE item_id = ?', [itemId]); | ||||
| 	} | ||||
|  | ||||
| 	static filterArray(models) { | ||||
| 		let output = []; | ||||
| 		for (let i = 0; i < models.length; i++) { | ||||
| 			output.push(this.filter(models[i])); | ||||
| 		} | ||||
| 		return output; | ||||
| 	} | ||||
|  | ||||
| 	static filter(model) { | ||||
| 		return model; | ||||
| 	} | ||||
|  | ||||
| 	static delete(id, options = null) { | ||||
| 		options = this.modOptions(options); | ||||
|  | ||||
|   | ||||
| @@ -15,7 +15,7 @@ class DatabaseDriverNode { | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	setDebugEnabled(v) { | ||||
| 	setDebugMode(v) { | ||||
| 		// ?? | ||||
| 	} | ||||
|  | ||||
|   | ||||
| @@ -13,8 +13,8 @@ class DatabaseDriverReactNative { | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	setDebugEnabled(v) { | ||||
| 		SQLite.DEBUG(v); | ||||
| 	setDebugMode(v) { | ||||
| 		//SQLite.DEBUG(v); | ||||
| 	} | ||||
|  | ||||
| 	selectOne(sql, params = null) { | ||||
|   | ||||
| @@ -141,8 +141,8 @@ class Database { | ||||
| 		return this.logger_; | ||||
| 	} | ||||
|  | ||||
| 	setDebugEnabled(v) { | ||||
| 		this.driver_.setDebugEnabled(v); | ||||
| 	setDebugMode(v) { | ||||
| 		//this.driver_.setDebugMode(v); | ||||
| 		this.debugMode_ = v; | ||||
| 	} | ||||
|  | ||||
| @@ -283,6 +283,7 @@ class Database { | ||||
|  | ||||
| 	logQuery(sql, params = null) { | ||||
| 		if (!this.debugMode()) return; | ||||
|  | ||||
| 		if (params !== null) { | ||||
| 			this.logger().debug('DB: ' + sql, JSON.stringify(params)); | ||||
| 		} else { | ||||
|   | ||||
| @@ -74,6 +74,16 @@ class Note extends BaseItem { | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	static filter(note) { | ||||
| 		if (!note) return note; | ||||
|  | ||||
| 		let output = Object.assign({}, note); | ||||
| 		if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8); | ||||
| 		if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8); | ||||
| 		if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4); | ||||
| 		return output; | ||||
| 	} | ||||
|  | ||||
| 	static all(parentId) { | ||||
| 		return this.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND parent_id = ?', [parentId]); | ||||
| 	} | ||||
|   | ||||
| @@ -192,7 +192,7 @@ class AppComponent extends React.Component { | ||||
| 	componentDidMount() { | ||||
| 		let db = new Database(new DatabaseDriverReactNative()); | ||||
| 		//db.setDebugEnabled(Registry.debugMode()); | ||||
| 		db.setDebugEnabled(false); | ||||
| 		db.setDebugMode(false); | ||||
|  | ||||
| 		BaseModel.dispatch = this.props.dispatch; | ||||
| 		BaseModel.db_ = db; | ||||
|   | ||||
| @@ -12,12 +12,17 @@ import moment from 'moment'; | ||||
| class Synchronizer { | ||||
|  | ||||
| 	constructor(db, api) { | ||||
| 		this.state_ = 'idle'; | ||||
| 		this.db_ = db; | ||||
| 		this.api_ = api; | ||||
| 		this.syncDirName_ = '.sync'; | ||||
| 		this.logger_ = new Logger(); | ||||
| 	} | ||||
|  | ||||
| 	state() { | ||||
| 		return this.state_; | ||||
| 	} | ||||
|  | ||||
| 	db() { | ||||
| 		return this.db_; | ||||
| 	} | ||||
| @@ -34,6 +39,38 @@ class Synchronizer { | ||||
| 		return this.logger_; | ||||
| 	} | ||||
|  | ||||
| 	logSyncOperation(action, local, remote, reason) { | ||||
| 		let line = ['Sync']; | ||||
| 		line.push(action); | ||||
| 		line.push(reason); | ||||
|  | ||||
| 		if (local) { | ||||
| 			let s = []; | ||||
| 			s.push(local.id); | ||||
| 			if ('title' in local) s.push('"' + local.title + '"'); | ||||
| 			line.push('(Local ' + s.join(', ') + ')'); | ||||
| 		} | ||||
|  | ||||
| 		if (remote) { | ||||
| 			let s = []; | ||||
| 			s.push(remote.id); | ||||
| 			if ('title' in remote) s.push('"' + remote.title + '"'); | ||||
| 			line.push('(Remote ' + s.join(', ') + ')'); | ||||
| 		} | ||||
|  | ||||
| 		this.logger().debug(line.join(': ')); | ||||
| 	} | ||||
|  | ||||
| 	async logSyncSummary(report) { | ||||
| 		for (let n in report) { | ||||
| 			this.logger().info(n + ': ' + (report[n] ? report[n] : '-')); | ||||
| 		} | ||||
| 		let folderCount = await Folder.count(); | ||||
| 		let noteCount = await Note.count(); | ||||
| 		this.logger().info('Total folders: ' + folderCount); | ||||
| 		this.logger().info('Total notes: ' + noteCount); | ||||
| 	} | ||||
|  | ||||
| 	async createWorkDir() { | ||||
| 		if (this.syncWorkDir_) return this.syncWorkDir_; | ||||
| 		let dir = await this.api().mkdir(this.syncDirName_); | ||||
| @@ -41,12 +78,31 @@ class Synchronizer { | ||||
| 	} | ||||
|  | ||||
| 	async start() { | ||||
| 		if (this.state() != 'idle') { | ||||
| 			this.logger().warn('Synchronization is already in progress. State: ' + this.state()); | ||||
| 			return; | ||||
| 		}	 | ||||
|  | ||||
| 		// ------------------------------------------------------------------------ | ||||
| 		// First, find all the items that have been changed since the | ||||
| 		// last sync and apply the changes to remote. | ||||
| 		// ------------------------------------------------------------------------ | ||||
|  | ||||
| 		this.logger().info('Starting synchronization...'); | ||||
| 		let synchronizationId = time.unixMs().toString(); | ||||
| 		this.logger().info('Starting synchronization... [' + synchronizationId + ']'); | ||||
|  | ||||
| 		this.state_ = 'started'; | ||||
|  | ||||
| 		let report = { | ||||
| 			createLocal: 0, | ||||
| 			updateLocal: 0, | ||||
| 			deleteLocal: 0, | ||||
| 			createRemote: 0, | ||||
| 			updateRemote: 0, | ||||
| 			deleteRemote: 0, | ||||
| 			folderConflict: 0, | ||||
| 			noteConflict: 0, | ||||
| 		}; | ||||
|  | ||||
| 		await this.createWorkDir(); | ||||
|  | ||||
| @@ -67,13 +123,16 @@ class Synchronizer { | ||||
| 				let content = ItemClass.serialize(local); | ||||
| 				let action = null; | ||||
| 				let updateSyncTimeOnly = true; | ||||
| 				let reason = ''; | ||||
|  | ||||
| 				if (!remote) { | ||||
| 					if (!local.sync_time) { | ||||
| 						action = 'createRemote'; | ||||
| 						reason = 'remote does not exist, and local is new and has never been synced'; | ||||
| 					} else { | ||||
| 						// Note or folder was modified after having been deleted remotely | ||||
| 						action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict'; | ||||
| 						reason = 'remote has been deleted, but local has changes'; | ||||
| 					} | ||||
| 				} else { | ||||
| 					if (remote.updated_time > local.sync_time) { | ||||
| @@ -81,18 +140,20 @@ class Synchronizer { | ||||
| 						// remote has been modified after the sync time, it means both notes have been | ||||
| 						// modified and so there's a conflict. | ||||
| 						action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict'; | ||||
| 						reason = 'both remote and local have changes'; | ||||
| 					} else { | ||||
| 						action = 'updateRemote'; | ||||
| 						reason = 'local has changes'; | ||||
| 					} | ||||
| 				} | ||||
|  | ||||
| 				this.logger().debug('Sync action (1): ' + action); | ||||
| 				this.logSyncOperation(action, local, remote, reason); | ||||
|  | ||||
| 				if (action == 'createRemote' || action == 'updateRemote') { | ||||
|  | ||||
| 					// Make the operation atomic by doing the work on a copy of the file | ||||
| 					// and then copying it back to the original location. | ||||
| 					let tempPath = this.syncDirName_ + '/' + path; | ||||
| 					let tempPath = this.syncDirName_ + '/' + path + '_' + time.unixMs(); | ||||
| 					 | ||||
| 					await this.api().put(tempPath, content); | ||||
| 					await this.api().setTimestamp(tempPath, local.updated_time); | ||||
| @@ -131,6 +192,8 @@ class Synchronizer { | ||||
|  | ||||
| 				} | ||||
|  | ||||
| 				report[action]++; | ||||
|  | ||||
| 				donePaths.push(path); | ||||
| 			} | ||||
|  | ||||
| @@ -145,9 +208,11 @@ class Synchronizer { | ||||
| 		for (let i = 0; i < deletedItems.length; i++) { | ||||
| 			let item = deletedItems[i]; | ||||
| 			let path = BaseItem.systemPath(item.item_id) | ||||
| 			this.logger().debug('Sync action (2): deleteRemote'); | ||||
| 			this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted'); | ||||
| 			await this.api().delete(path); | ||||
| 			await BaseModel.remoteDeletedItem(item.item_id); | ||||
|  | ||||
| 			report['deleteRemote']++; | ||||
| 		} | ||||
|  | ||||
| 		// ------------------------------------------------------------------------ | ||||
| @@ -172,33 +237,37 @@ class Synchronizer { | ||||
| 			let local = await BaseItem.loadItemByPath(path); | ||||
| 			if (!local) { | ||||
| 				action = 'createLocal'; | ||||
| 				reason = 'Local exists but remote does not'; | ||||
| 				reason = 'remote exists but local does not'; | ||||
| 			} else { | ||||
| 				if (remote.updated_time > local.updated_time) { | ||||
| 					action = 'updateLocal'; | ||||
| 					reason = sprintf('Remote (%s) is more recent than local (%s)', time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time)); | ||||
| 					reason = sprintf('remote is more recent than local'); // , time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time) | ||||
| 				} | ||||
| 			} | ||||
|  | ||||
| 			if (!action) continue; | ||||
|  | ||||
| 			this.logger().debug('Sync action (3): ' + action); | ||||
| 			this.logger().debug('Reason: ' + reason); | ||||
|  | ||||
| 			if (action == 'createLocal' || action == 'updateLocal') { | ||||
| 				let content = await this.api().get(path); | ||||
| 				if (!content) { | ||||
| 					this.logger().warn('Remote item has been deleted between now and the list() call? In that case it will handled during the next sync: ' + path); | ||||
| 					this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); | ||||
| 					continue; | ||||
| 				} | ||||
| 				content = BaseItem.unserialize(content); | ||||
| 				let ItemClass = BaseItem.itemClass(content); | ||||
|  | ||||
| 				content.sync_time = time.unixMs(); | ||||
| 				let newContent = Object.assign({}, content); | ||||
| 				newContent.sync_time = time.unixMs(); | ||||
| 				let options = { autoTimestamp: false }; | ||||
| 				if (action == 'createLocal') options.isNew = true; | ||||
| 				await ItemClass.save(content, options); | ||||
| 				await ItemClass.save(newContent, options); | ||||
|  | ||||
| 				this.logSyncOperation(action, local, content, reason); | ||||
| 			} else { | ||||
| 				this.logSyncOperation(action, local, remote, reason); | ||||
| 			} | ||||
|  | ||||
| 			report[action]++; | ||||
| 		} | ||||
|  | ||||
| 		// ------------------------------------------------------------------------ | ||||
| @@ -208,14 +277,18 @@ class Synchronizer { | ||||
|  | ||||
| 		let noteIds = await Folder.syncedNoteIds(); | ||||
| 		for (let i = 0; i < noteIds.length; i++) { | ||||
| 			if (remoteIds.indexOf(noteIds[i]) < 0) { | ||||
| 				this.logger().debug('Sync action (4): deleteLocal: ' + noteIds[i]); | ||||
| 				await Note.delete(noteIds[i], { trackDeleted: false }); | ||||
| 			let noteId = noteIds[i]; | ||||
| 			if (remoteIds.indexOf(noteId) < 0) { | ||||
| 				this.logSyncOperation('deleteLocal', { id: noteId }, null, 'remote has been deleted'); | ||||
| 				await Note.delete(noteId, { trackDeleted: false }); | ||||
| 				report['deleteLocal']++; | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		// Number of sync items (Created, updated, deleted Local/Remote) | ||||
| 		// Total number of items | ||||
| 		this.logger().info('Synchronization complete [' + synchronizationId + ']:'); | ||||
| 		await this.logSyncSummary(report); | ||||
|  | ||||
| 		this.state_ = 'idle'; | ||||
|  | ||||
| 		return Promise.resolve(); | ||||
| 	} | ||||
|   | ||||
		Reference in New Issue
	
	Block a user