mirror of
https://github.com/laurent22/joplin.git
synced 2024-11-24 08:12:24 +02:00
Improved sync reporting
This commit is contained in:
Laurent Cozic
parent
e89447dd85
commit
e521ca9427
@ -28,7 +28,7 @@ const fs = require('fs');
|
||||
Log.setLevel(Log.LEVEL_DEBUG);
|
||||
|
||||
let db = new Database(new DatabaseDriverNode());
|
||||
db.setDebugEnabled(true);
|
||||
db.setDebugMode(true);
|
||||
db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => {
|
||||
return db.selectAll('SELECT * FROM table_fields');
|
||||
}).then((rows) => {
|
||||
|
||||
@ -3,12 +3,12 @@ require('app-module-path').addPath(__dirname);
|
||||
import { uuid } from 'src/uuid.js';
|
||||
import moment from 'moment';
|
||||
import { promiseChain } from 'src/promise-utils.js';
|
||||
import { WebApi } from 'src/web-api.js'
|
||||
import { folderItemFilename } from 'src/string-utils.js'
|
||||
import { BaseModel } from 'src/base-model.js';
|
||||
import { Note } from 'src/models/note.js';
|
||||
import { Folder } from 'src/models/folder.js';
|
||||
import jsSHA from "jssha";
|
||||
|
||||
let webApi = new WebApi('http://joplin.local');
|
||||
|
||||
const Promise = require('promise');
|
||||
const fs = require('fs');
|
||||
const stringToStream = require('string-to-stream')
|
||||
@ -398,61 +398,6 @@ function enexXmlToMd(stream, resources) {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
// const path = require('path');
|
||||
|
||||
// var walk = function (dir, done) {
|
||||
// fs.readdir(dir, function (error, list) {
|
||||
// if (error) return done(error);
|
||||
// var i = 0;
|
||||
// (function next () {
|
||||
// var file = list[i++];
|
||||
|
||||
// if (!file) return done(null);
|
||||
// file = dir + '/' + file;
|
||||
|
||||
// fs.stat(file, function (error, stat) {
|
||||
// if (stat && stat.isDirectory()) {
|
||||
// walk(file, function (error) {
|
||||
// next();
|
||||
// });
|
||||
// } else {
|
||||
// if (path.basename(file) != 'sample4.xml') {
|
||||
// next();
|
||||
// return;
|
||||
// }
|
||||
|
||||
// if (path.extname(file) == '.xml') {
|
||||
// console.info('Processing: ' + file);
|
||||
// let stream = fs.createReadStream(file);
|
||||
// enexXmlToMd(stream).then((md) => {
|
||||
// console.info(md);
|
||||
// console.info(processMdArrayNewLines(md));
|
||||
// next();
|
||||
// }).catch((error) => {
|
||||
// console.error(error);
|
||||
// return done(error);
|
||||
// });
|
||||
// } else {
|
||||
// next();
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// })();
|
||||
// });
|
||||
// };
|
||||
|
||||
// walk('/home/laurent/Dropbox/Samples/', function(error) {
|
||||
// if (error) {
|
||||
// throw error;
|
||||
// } else {
|
||||
// console.log('-------------------------------------------------------------');
|
||||
// console.log('finished.');
|
||||
// console.log('-------------------------------------------------------------');
|
||||
// }
|
||||
// });
|
||||
|
||||
function isBlockTag(n) {
|
||||
return n=="div" || n=="p" || n=="dl" || n=="dd" || n=="center" || n=="table" || n=="tr" || n=="td" || n=="th" || n=="tbody";
|
||||
}
|
||||
@ -495,12 +440,31 @@ function xmlNodeText(xmlNode) {
|
||||
return xmlNode[0];
|
||||
}
|
||||
|
||||
let existingTimestamps = [];
|
||||
|
||||
function uniqueCreatedTimestamp(timestamp) {
|
||||
if (existingTimestamps.indexOf(timestamp) < 0) {
|
||||
existingTimestamps.push(timestamp);
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
for (let i = 1; i <= 999; i++) {
|
||||
let t = timestamp + i;
|
||||
if (existingTimestamps.indexOf(t) < 0) {
|
||||
existingTimestamps.push(t);
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
function dateToTimestamp(s) {
|
||||
let m = moment(s, 'YYYYMMDDTHHmmssZ');
|
||||
if (!m.isValid()) {
|
||||
throw new Error('Invalid date: ' + s);
|
||||
}
|
||||
return Math.round(m.toDate().getTime() / 1000);
|
||||
return m.toDate().getTime();
|
||||
}
|
||||
|
||||
function evernoteXmlToMdArray(xml) {
|
||||
@ -514,61 +478,6 @@ function extractRecognitionObjId(recognitionXml) {
|
||||
return r && r.length >= 2 ? r[1] : null;
|
||||
}
|
||||
|
||||
function saveNoteToWebApi(note) {
|
||||
let data = Object.assign({}, note);
|
||||
delete data.resources;
|
||||
delete data.tags;
|
||||
|
||||
webApi.post('notes', null, data).then((r) => {
|
||||
//console.info(r);
|
||||
}).catch((error) => {
|
||||
console.error("Error for note: " + note.title);
|
||||
console.error(error);
|
||||
});
|
||||
}
|
||||
|
||||
function noteserialize_format(propName, propValue) {
|
||||
if (['created_time', 'updated_time'].indexOf(propName) >= 0) {
|
||||
if (!propValue) return '';
|
||||
propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss');
|
||||
} else if (propValue === null || propValue === undefined) {
|
||||
propValue = '';
|
||||
}
|
||||
|
||||
return propValue;
|
||||
}
|
||||
|
||||
function noteserialize(note) {
|
||||
let shownKeys = ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time'];
|
||||
let output = [];
|
||||
|
||||
output.push(note.title);
|
||||
output.push("");
|
||||
output.push(note.body);
|
||||
output.push('');
|
||||
for (let i = 0; i < shownKeys.length; i++) {
|
||||
let v = note[shownKeys[i]];
|
||||
v = noteserialize_format(shownKeys[i], v);
|
||||
output.push(shownKeys[i] + ': ' + v);
|
||||
}
|
||||
|
||||
return output.join("\n");
|
||||
}
|
||||
|
||||
// function folderItemFilename(item) {
|
||||
// let output = escapeFilename(item.title).trim();
|
||||
// if (!output.length) output = '_';
|
||||
// return output + '.' + item.id.substr(0, 7);
|
||||
// }
|
||||
|
||||
function noteFilename(note) {
|
||||
return folderItemFilename(note) + '.md';
|
||||
}
|
||||
|
||||
function folderFilename(folder) {
|
||||
return folderItemFilename(folder);
|
||||
}
|
||||
|
||||
function filePutContents(filePath, content) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.writeFile(filePath, content, function(error) {
|
||||
@ -614,28 +523,15 @@ function createDirectory(path) {
|
||||
});
|
||||
}
|
||||
|
||||
const baseNoteDir = '/home/laurent/Temp/TestImport';
|
||||
|
||||
// createDirectory('/home/laurent/Temp/TestImport').then(() => {
|
||||
// console.info('OK');
|
||||
// }).catch((error) => {
|
||||
// console.error(error);
|
||||
// });
|
||||
|
||||
function saveNoteToDisk(folder, note) {
|
||||
const noteContent = noteserialize(note);
|
||||
const notePath = baseNoteDir + '/' + folderFilename(folder) + '/' + noteFilename(note);
|
||||
|
||||
// console.info('===================================================');
|
||||
// console.info(note);//noteContent);
|
||||
return filePutContents(notePath, noteContent).then(() => {
|
||||
return setModifiedTime(notePath, note.updated_time ? note.updated_time : note.created_time);
|
||||
});
|
||||
}
|
||||
|
||||
function saveFolderToDisk(folder) {
|
||||
let path = baseNoteDir + '/' + folderFilename(folder);
|
||||
return createDirectory(path);
|
||||
function removeUndefinedProperties(note) {
|
||||
let output = {};
|
||||
for (let n in note) {
|
||||
if (!note.hasOwnProperty(n)) continue;
|
||||
let v = note[n];
|
||||
if (v === undefined || v === null) continue;
|
||||
output[n] = v;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function createNoteId(note) {
|
||||
@ -645,7 +541,56 @@ function createNoteId(note) {
|
||||
return hash.substr(0, 32);
|
||||
}
|
||||
|
||||
function importEnex(parentFolder, stream) {
|
||||
async function fuzzyMatch(note) {
|
||||
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time);
|
||||
if (!notes.length) return null;
|
||||
if (notes.length === 1) return notes[0];
|
||||
|
||||
for (let i = 0; i < notes.length; i++) {
|
||||
if (notes[i].title == note.title && note.title.trim() != '') return notes[i];
|
||||
}
|
||||
|
||||
for (let i = 0; i < notes.length; i++) {
|
||||
if (notes[i].body == note.body && note.body.trim() != '') return notes[i];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function saveNoteToDb(note) {
|
||||
note = Note.filter(note);
|
||||
let existingNote = await fuzzyMatch(note);
|
||||
|
||||
if (existingNote) {
|
||||
let diff = BaseModel.diffObjects(existingNote, note);
|
||||
delete diff.tags;
|
||||
delete diff.resources;
|
||||
delete diff.id;
|
||||
|
||||
// console.info('======================================');
|
||||
// console.info(note);
|
||||
// console.info(existingNote);
|
||||
// console.info(diff);
|
||||
// console.info('======================================');
|
||||
|
||||
if (!Object.getOwnPropertyNames(diff).length) return;
|
||||
|
||||
diff.id = existingNote.id;
|
||||
diff.type_ = existingNote.type_;
|
||||
return Note.save(diff, { autoTimestamp: false });
|
||||
} else {
|
||||
|
||||
console.info('NNNNNNNNNNNNNNNNN4');
|
||||
// return Note.save(note, {
|
||||
// isNew: true,
|
||||
// autoTimestamp: false,
|
||||
// });
|
||||
}
|
||||
}
|
||||
|
||||
function importEnex(db, parentFolderId, resourceDir, filePath) {
|
||||
let stream = fs.createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let options = {};
|
||||
let strict = true;
|
||||
@ -659,6 +604,10 @@ function importEnex(parentFolder, stream) {
|
||||
let noteResourceRecognition = null;
|
||||
let notes = [];
|
||||
|
||||
stream.on('error', (error) => {
|
||||
reject(new Error(error.toString()));
|
||||
});
|
||||
|
||||
function currentNodeName() {
|
||||
if (!nodes.length) return null;
|
||||
return nodes[nodes.length - 1].name;
|
||||
@ -688,28 +637,14 @@ function importEnex(parentFolder, stream) {
|
||||
firstAttachment = false;
|
||||
}
|
||||
|
||||
note.parent_id = parentFolder.id;
|
||||
note.parent_id = parentFolderId;
|
||||
note.body = processMdArrayNewLines(result.lines);
|
||||
note.id = uuid.create();
|
||||
|
||||
saveNoteToDisk(parentFolder, note);
|
||||
return saveNoteToDb(note);
|
||||
|
||||
// console.info(noteserialize(note));
|
||||
// console.info('=========================================================================================================================');
|
||||
|
||||
//saveNoteToWebApi(note);
|
||||
|
||||
// console.info('======== NOTE ============================================================================');
|
||||
// let c = note.content;
|
||||
// delete note.content
|
||||
// console.info(note);
|
||||
// console.info('------------------------------------------------------------------------------------------');
|
||||
// console.info(c);
|
||||
|
||||
// if (note.resources.length) {
|
||||
// console.info('=========================================================');
|
||||
// console.info(note.content);
|
||||
// }
|
||||
// SAVE NOTE HERE
|
||||
// saveNoteToDisk(parentFolder, note);
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -718,7 +653,7 @@ function importEnex(parentFolder, stream) {
|
||||
}
|
||||
|
||||
saxStream.on('error', function(e) {
|
||||
reject(e);
|
||||
reject(new Error(e.toString()));
|
||||
})
|
||||
|
||||
saxStream.on('text', function(text) {
|
||||
@ -738,7 +673,7 @@ function importEnex(parentFolder, stream) {
|
||||
if (n == 'title') {
|
||||
note.title = text;
|
||||
} else if (n == 'created') {
|
||||
note.created_time = dateToTimestamp(text);
|
||||
note.created_time = uniqueCreatedTimestamp(dateToTimestamp(text));
|
||||
} else if (n == 'updated') {
|
||||
note.updated_time = dateToTimestamp(text);
|
||||
} else if (n == 'tag') {
|
||||
@ -783,6 +718,8 @@ function importEnex(parentFolder, stream) {
|
||||
nodes.pop();
|
||||
|
||||
if (n == 'note') {
|
||||
note = removeUndefinedProperties(note);
|
||||
|
||||
notes.push(note);
|
||||
if (notes.length >= 10) {
|
||||
stream.pause();
|
||||
@ -822,7 +759,7 @@ function importEnex(parentFolder, stream) {
|
||||
filename: noteResource.filename,
|
||||
};
|
||||
|
||||
r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE
|
||||
// r.data = noteResource.data.substr(0, 20); // TODO: REMOVE REMOVE REMOVE REMOVE REMOVE REMOVE
|
||||
|
||||
note.resources.push(r);
|
||||
noteResource = null;
|
||||
@ -837,45 +774,4 @@ function importEnex(parentFolder, stream) {
|
||||
});
|
||||
}
|
||||
|
||||
// TODO: make it persistent and random
|
||||
const clientId = 'AB78AB78AB78AB78AB78AB78AB78AB78';
|
||||
|
||||
const folderTitle = 'Laurent';
|
||||
//const folderTitle = 'Voiture';
|
||||
|
||||
webApi.post('sessions', null, {
|
||||
email: 'laurent@cozic.net',
|
||||
password: '12345678',
|
||||
client_id: clientId,
|
||||
}).then((session) => {
|
||||
webApi.setSession(session.id);
|
||||
console.info('Got session: ' + session.id);
|
||||
return webApi.get('folders');
|
||||
}).then((folders) => {
|
||||
|
||||
let folder = null;
|
||||
|
||||
for (let i = 0; i < folders.length; i++) {
|
||||
if (folders[i].title = folderTitle) {
|
||||
folder = folders[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return folder ? Promise.resolve(folder) : webApi.post('folders', null, { title: folderTitle });
|
||||
}).then((folder) => {
|
||||
return saveFolderToDisk(folder).then(() => {
|
||||
return folder;
|
||||
});
|
||||
}).then((folder) => {
|
||||
let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/' + folderTitle + '.enex');
|
||||
//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/afaire.enex');
|
||||
//let fileStream = fs.createReadStream('/mnt/c/Users/Laurent/Desktop/testtags.enex');
|
||||
importEnex(folder, fileStream).then(() => {
|
||||
//console.info('DONE IMPORTING');
|
||||
}).catch((error) => {
|
||||
console.error('Cannot import', error);
|
||||
});
|
||||
}).catch((error) => {
|
||||
console.error(error);
|
||||
});
|
||||
export { importEnex };
|
||||
@ -13,12 +13,14 @@ import { Synchronizer } from 'src/synchronizer.js';
|
||||
import { Logger } from 'src/logger.js';
|
||||
import { uuid } from 'src/uuid.js';
|
||||
import { sprintf } from 'sprintf-js';
|
||||
import { importEnex } from 'import-enex';
|
||||
import { _ } from 'src/locale.js';
|
||||
import os from 'os';
|
||||
import fs from 'fs-extra';
|
||||
|
||||
const APPNAME = 'joplin';
|
||||
const dataDir = os.homedir() + '/.local/share/' + APPNAME;
|
||||
const resourceDir = dataDir + '/resources';
|
||||
|
||||
process.on('unhandledRejection', (reason, p) => {
|
||||
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
|
||||
@ -28,22 +30,50 @@ const logger = new Logger();
|
||||
logger.addTarget('file', { path: dataDir + '/log.txt' });
|
||||
logger.setLevel(Logger.LEVEL_DEBUG);
|
||||
|
||||
const dbLogger = new Logger();
|
||||
dbLogger.addTarget('file', { path: dataDir + '/log-database.txt' });
|
||||
dbLogger.setLevel(Logger.LEVEL_DEBUG);
|
||||
|
||||
const syncLogger = new Logger();
|
||||
syncLogger.addTarget('file', { path: dataDir + '/log-sync.txt' });
|
||||
syncLogger.setLevel(Logger.LEVEL_DEBUG);
|
||||
|
||||
let db = new Database(new DatabaseDriverNode());
|
||||
db.setLogger(logger);
|
||||
db.setDebugMode(true);
|
||||
db.setLogger(dbLogger);
|
||||
|
||||
let synchronizer_ = null;
|
||||
const vorpal = require('vorpal')();
|
||||
|
||||
async function main() {
|
||||
await fs.mkdirp(dataDir, 0o755);
|
||||
await fs.mkdirp(resourceDir, 0o755);
|
||||
|
||||
await db.open({ name: dataDir + '/database.sqlite' });
|
||||
await db.open({ name: dataDir + '/database2.sqlite' });
|
||||
BaseModel.db_ = db;
|
||||
await Setting.load();
|
||||
|
||||
|
||||
|
||||
|
||||
// console.info('DELETING ALL DATA');
|
||||
// await db.exec('DELETE FROM notes');
|
||||
// await db.exec('DELETE FROM changes');
|
||||
// await db.exec('DELETE FROM folders');
|
||||
// await db.exec('DELETE FROM resources');
|
||||
// await db.exec('DELETE FROM deleted_items');
|
||||
// await db.exec('DELETE FROM tags');
|
||||
// await db.exec('DELETE FROM note_tags');
|
||||
|
||||
// // let folder = await Folder.save({ title: 'test' });
|
||||
// let folder = await Folder.loadByField('title', 'test');
|
||||
// await importEnex(db, folder.id, resourceDir, '/mnt/c/Users/Laurent/Desktop/Laurent.enex'); //'/mnt/c/Users/Laurent/Desktop/Laurent.enex');
|
||||
// return;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
let commands = [];
|
||||
let currentFolder = null;
|
||||
|
||||
@ -383,6 +413,15 @@ async function main() {
|
||||
},
|
||||
});
|
||||
|
||||
commands.push({
|
||||
usage: 'import-enex',
|
||||
description: _('Imports a .enex file (Evernote export file).'),
|
||||
action: function (args, end) {
|
||||
|
||||
end();
|
||||
},
|
||||
});
|
||||
|
||||
for (let i = 0; i < commands.length; i++) {
|
||||
let c = commands[i];
|
||||
let o = vorpal.command(c.usage, c.description);
|
||||
|
||||
@ -63,7 +63,7 @@ function setupDatabase(id = null) {
|
||||
// Don't care if the file doesn't exist
|
||||
}).then(() => {
|
||||
databases_[id] = new Database(new DatabaseDriverNode());
|
||||
databases_[id].setDebugEnabled(false);
|
||||
databases_[id].setDebugMode(false);
|
||||
return databases_[id].open({ name: filePath }).then(() => {
|
||||
BaseModel.db_ = databases_[id];
|
||||
return setupDatabase(id);
|
||||
|
||||
@ -109,6 +109,12 @@ class BaseModel {
|
||||
return options;
|
||||
}
|
||||
|
||||
static count() {
|
||||
return this.db().selectOne('SELECT count(*) as total FROM `' + this.tableName() + '`').then((r) => {
|
||||
return r ? r['total'] : 0;
|
||||
});
|
||||
}
|
||||
|
||||
static load(id) {
|
||||
return this.loadByField('id', id);
|
||||
}
|
||||
@ -116,19 +122,19 @@ class BaseModel {
|
||||
static modelSelectOne(sql, params = null) {
|
||||
if (params === null) params = [];
|
||||
return this.db().selectOne(sql, params).then((model) => {
|
||||
return this.addModelMd(model);
|
||||
return this.filter(this.addModelMd(model));
|
||||
});
|
||||
}
|
||||
|
||||
static modelSelectAll(sql, params = null) {
|
||||
if (params === null) params = [];
|
||||
return this.db().selectAll(sql, params).then((models) => {
|
||||
return this.addModelMd(models);
|
||||
return this.filterArray(this.addModelMd(models));
|
||||
});
|
||||
}
|
||||
|
||||
static loadByField(fieldName, fieldValue) {
|
||||
return this.modelSelectOne('SELECT * FROM ' + this.tableName() + ' WHERE `' + fieldName + '` = ?', [fieldValue]);
|
||||
return this.modelSelectOne('SELECT * FROM `' + this.tableName() + '` WHERE `' + fieldName + '` = ?', [fieldValue]);
|
||||
}
|
||||
|
||||
static applyPatch(model, patch) {
|
||||
@ -200,9 +206,10 @@ class BaseModel {
|
||||
|
||||
static save(o, options = null) {
|
||||
options = this.modOptions(options);
|
||||
|
||||
options.isNew = options.isNew == 'auto' ? !o.id : options.isNew;
|
||||
|
||||
o = this.filter(o);
|
||||
|
||||
let queries = [];
|
||||
let saveQuery = this.saveQuery(o, options);
|
||||
let itemId = saveQuery.id;
|
||||
@ -242,7 +249,7 @@ class BaseModel {
|
||||
o = Object.assign({}, o);
|
||||
o.id = itemId;
|
||||
o = this.addModelMd(o);
|
||||
return o;
|
||||
return this.filter(o);
|
||||
}).catch((error) => {
|
||||
Log.error('Cannot save model', error);
|
||||
});
|
||||
@ -256,6 +263,18 @@ class BaseModel {
|
||||
return this.db().exec('DELETE FROM deleted_items WHERE item_id = ?', [itemId]);
|
||||
}
|
||||
|
||||
static filterArray(models) {
|
||||
let output = [];
|
||||
for (let i = 0; i < models.length; i++) {
|
||||
output.push(this.filter(models[i]));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
static filter(model) {
|
||||
return model;
|
||||
}
|
||||
|
||||
static delete(id, options = null) {
|
||||
options = this.modOptions(options);
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ class DatabaseDriverNode {
|
||||
});
|
||||
}
|
||||
|
||||
setDebugEnabled(v) {
|
||||
setDebugMode(v) {
|
||||
// ??
|
||||
}
|
||||
|
||||
|
||||
@ -13,8 +13,8 @@ class DatabaseDriverReactNative {
|
||||
});
|
||||
}
|
||||
|
||||
setDebugEnabled(v) {
|
||||
SQLite.DEBUG(v);
|
||||
setDebugMode(v) {
|
||||
//SQLite.DEBUG(v);
|
||||
}
|
||||
|
||||
selectOne(sql, params = null) {
|
||||
|
||||
@ -141,8 +141,8 @@ class Database {
|
||||
return this.logger_;
|
||||
}
|
||||
|
||||
setDebugEnabled(v) {
|
||||
this.driver_.setDebugEnabled(v);
|
||||
setDebugMode(v) {
|
||||
//this.driver_.setDebugMode(v);
|
||||
this.debugMode_ = v;
|
||||
}
|
||||
|
||||
@ -283,6 +283,7 @@ class Database {
|
||||
|
||||
logQuery(sql, params = null) {
|
||||
if (!this.debugMode()) return;
|
||||
|
||||
if (params !== null) {
|
||||
this.logger().debug('DB: ' + sql, JSON.stringify(params));
|
||||
} else {
|
||||
|
||||
@ -74,6 +74,16 @@ class Note extends BaseItem {
|
||||
});
|
||||
}
|
||||
|
||||
static filter(note) {
|
||||
if (!note) return note;
|
||||
|
||||
let output = Object.assign({}, note);
|
||||
if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8);
|
||||
if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8);
|
||||
if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4);
|
||||
return output;
|
||||
}
|
||||
|
||||
static all(parentId) {
|
||||
return this.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND parent_id = ?', [parentId]);
|
||||
}
|
||||
|
||||
@ -192,7 +192,7 @@ class AppComponent extends React.Component {
|
||||
componentDidMount() {
|
||||
let db = new Database(new DatabaseDriverReactNative());
|
||||
//db.setDebugEnabled(Registry.debugMode());
|
||||
db.setDebugEnabled(false);
|
||||
db.setDebugMode(false);
|
||||
|
||||
BaseModel.dispatch = this.props.dispatch;
|
||||
BaseModel.db_ = db;
|
||||
|
||||
@ -12,12 +12,17 @@ import moment from 'moment';
|
||||
class Synchronizer {
|
||||
|
||||
constructor(db, api) {
|
||||
this.state_ = 'idle';
|
||||
this.db_ = db;
|
||||
this.api_ = api;
|
||||
this.syncDirName_ = '.sync';
|
||||
this.logger_ = new Logger();
|
||||
}
|
||||
|
||||
state() {
|
||||
return this.state_;
|
||||
}
|
||||
|
||||
db() {
|
||||
return this.db_;
|
||||
}
|
||||
@ -34,6 +39,38 @@ class Synchronizer {
|
||||
return this.logger_;
|
||||
}
|
||||
|
||||
logSyncOperation(action, local, remote, reason) {
|
||||
let line = ['Sync'];
|
||||
line.push(action);
|
||||
line.push(reason);
|
||||
|
||||
if (local) {
|
||||
let s = [];
|
||||
s.push(local.id);
|
||||
if ('title' in local) s.push('"' + local.title + '"');
|
||||
line.push('(Local ' + s.join(', ') + ')');
|
||||
}
|
||||
|
||||
if (remote) {
|
||||
let s = [];
|
||||
s.push(remote.id);
|
||||
if ('title' in remote) s.push('"' + remote.title + '"');
|
||||
line.push('(Remote ' + s.join(', ') + ')');
|
||||
}
|
||||
|
||||
this.logger().debug(line.join(': '));
|
||||
}
|
||||
|
||||
async logSyncSummary(report) {
|
||||
for (let n in report) {
|
||||
this.logger().info(n + ': ' + (report[n] ? report[n] : '-'));
|
||||
}
|
||||
let folderCount = await Folder.count();
|
||||
let noteCount = await Note.count();
|
||||
this.logger().info('Total folders: ' + folderCount);
|
||||
this.logger().info('Total notes: ' + noteCount);
|
||||
}
|
||||
|
||||
async createWorkDir() {
|
||||
if (this.syncWorkDir_) return this.syncWorkDir_;
|
||||
let dir = await this.api().mkdir(this.syncDirName_);
|
||||
@ -41,12 +78,31 @@ class Synchronizer {
|
||||
}
|
||||
|
||||
async start() {
|
||||
if (this.state() != 'idle') {
|
||||
this.logger().warn('Synchronization is already in progress. State: ' + this.state());
|
||||
return;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// First, find all the items that have been changed since the
|
||||
// last sync and apply the changes to remote.
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
this.logger().info('Starting synchronization...');
|
||||
let synchronizationId = time.unixMs().toString();
|
||||
this.logger().info('Starting synchronization... [' + synchronizationId + ']');
|
||||
|
||||
this.state_ = 'started';
|
||||
|
||||
let report = {
|
||||
createLocal: 0,
|
||||
updateLocal: 0,
|
||||
deleteLocal: 0,
|
||||
createRemote: 0,
|
||||
updateRemote: 0,
|
||||
deleteRemote: 0,
|
||||
folderConflict: 0,
|
||||
noteConflict: 0,
|
||||
};
|
||||
|
||||
await this.createWorkDir();
|
||||
|
||||
@ -67,13 +123,16 @@ class Synchronizer {
|
||||
let content = ItemClass.serialize(local);
|
||||
let action = null;
|
||||
let updateSyncTimeOnly = true;
|
||||
let reason = '';
|
||||
|
||||
if (!remote) {
|
||||
if (!local.sync_time) {
|
||||
action = 'createRemote';
|
||||
reason = 'remote does not exist, and local is new and has never been synced';
|
||||
} else {
|
||||
// Note or folder was modified after having been deleted remotely
|
||||
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict';
|
||||
reason = 'remote has been deleted, but local has changes';
|
||||
}
|
||||
} else {
|
||||
if (remote.updated_time > local.sync_time) {
|
||||
@ -81,18 +140,20 @@ class Synchronizer {
|
||||
// remote has been modified after the sync time, it means both notes have been
|
||||
// modified and so there's a conflict.
|
||||
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict';
|
||||
reason = 'both remote and local have changes';
|
||||
} else {
|
||||
action = 'updateRemote';
|
||||
reason = 'local has changes';
|
||||
}
|
||||
}
|
||||
|
||||
this.logger().debug('Sync action (1): ' + action);
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
|
||||
if (action == 'createRemote' || action == 'updateRemote') {
|
||||
|
||||
// Make the operation atomic by doing the work on a copy of the file
|
||||
// and then copying it back to the original location.
|
||||
let tempPath = this.syncDirName_ + '/' + path;
|
||||
let tempPath = this.syncDirName_ + '/' + path + '_' + time.unixMs();
|
||||
|
||||
await this.api().put(tempPath, content);
|
||||
await this.api().setTimestamp(tempPath, local.updated_time);
|
||||
@ -131,6 +192,8 @@ class Synchronizer {
|
||||
|
||||
}
|
||||
|
||||
report[action]++;
|
||||
|
||||
donePaths.push(path);
|
||||
}
|
||||
|
||||
@ -145,9 +208,11 @@ class Synchronizer {
|
||||
for (let i = 0; i < deletedItems.length; i++) {
|
||||
let item = deletedItems[i];
|
||||
let path = BaseItem.systemPath(item.item_id)
|
||||
this.logger().debug('Sync action (2): deleteRemote');
|
||||
this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
|
||||
await this.api().delete(path);
|
||||
await BaseModel.remoteDeletedItem(item.item_id);
|
||||
|
||||
report['deleteRemote']++;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
@ -172,33 +237,37 @@ class Synchronizer {
|
||||
let local = await BaseItem.loadItemByPath(path);
|
||||
if (!local) {
|
||||
action = 'createLocal';
|
||||
reason = 'Local exists but remote does not';
|
||||
reason = 'remote exists but local does not';
|
||||
} else {
|
||||
if (remote.updated_time > local.updated_time) {
|
||||
action = 'updateLocal';
|
||||
reason = sprintf('Remote (%s) is more recent than local (%s)', time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time));
|
||||
reason = sprintf('remote is more recent than local'); // , time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time)
|
||||
}
|
||||
}
|
||||
|
||||
if (!action) continue;
|
||||
|
||||
this.logger().debug('Sync action (3): ' + action);
|
||||
this.logger().debug('Reason: ' + reason);
|
||||
|
||||
if (action == 'createLocal' || action == 'updateLocal') {
|
||||
let content = await this.api().get(path);
|
||||
if (!content) {
|
||||
this.logger().warn('Remote item has been deleted between now and the list() call? In that case it will handled during the next sync: ' + path);
|
||||
this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path);
|
||||
continue;
|
||||
}
|
||||
content = BaseItem.unserialize(content);
|
||||
let ItemClass = BaseItem.itemClass(content);
|
||||
|
||||
content.sync_time = time.unixMs();
|
||||
let newContent = Object.assign({}, content);
|
||||
newContent.sync_time = time.unixMs();
|
||||
let options = { autoTimestamp: false };
|
||||
if (action == 'createLocal') options.isNew = true;
|
||||
await ItemClass.save(content, options);
|
||||
await ItemClass.save(newContent, options);
|
||||
|
||||
this.logSyncOperation(action, local, content, reason);
|
||||
} else {
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
}
|
||||
|
||||
report[action]++;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
@ -208,14 +277,18 @@ class Synchronizer {
|
||||
|
||||
let noteIds = await Folder.syncedNoteIds();
|
||||
for (let i = 0; i < noteIds.length; i++) {
|
||||
if (remoteIds.indexOf(noteIds[i]) < 0) {
|
||||
this.logger().debug('Sync action (4): deleteLocal: ' + noteIds[i]);
|
||||
await Note.delete(noteIds[i], { trackDeleted: false });
|
||||
let noteId = noteIds[i];
|
||||
if (remoteIds.indexOf(noteId) < 0) {
|
||||
this.logSyncOperation('deleteLocal', { id: noteId }, null, 'remote has been deleted');
|
||||
await Note.delete(noteId, { trackDeleted: false });
|
||||
report['deleteLocal']++;
|
||||
}
|
||||
}
|
||||
|
||||
// Number of sync items (Created, updated, deleted Local/Remote)
|
||||
// Total number of items
|
||||
this.logger().info('Synchronization complete [' + synchronizationId + ']:');
|
||||
await this.logSyncSummary(report);
|
||||
|
||||
this.state_ = 'idle';
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user