self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

Fixed onedrive sync issue

Browse Source
This commit is contained in:
Laurent Cozic 2017-06-29 18:03:16 +00:00
parent 01fc71d732
commit 9060ed489c
11 changed files with 358 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CliClient/app/import-enex.js
View File

@ -50,19 +50,8 @@ function createNoteId(note) {
}
async function fuzzyMatch(note) {
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
if (!notes.length) return null;
if (notes.length === 1) return notes[0];
for (let i = 0; i < notes.length; i++) {
if (notes[i].title == note.title && note.title.trim() != '') return notes[i];
}
for (let i = 0; i < notes.length; i++) {
if (notes[i].body == note.body && note.body.trim() != '') return notes[i];
}
return null;
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]);
return notes.length !== 1 ? null : notes[0];
}
async function saveNoteResources(note) {

164
CliClient/app/main.js
View File

@ -42,21 +42,6 @@ let logger = new Logger();
let dbLogger = new Logger();
let syncLogger = new Logger();
// commands.push({
// usage: 'root',
// options: [
// ['--profile <filePath>', 'Sets the profile path directory.'],
// ],
// action: function(args, end) {
// if (args.profile) {
// initArgs.profileDir = args.profile;
// args.splice(0, 2);
// }
// end(args);
// },
// });
commands.push({
usage: 'version',
description: 'Displays version information',
@ -433,7 +418,7 @@ function commandByName(name) {
for (let i = 0; i < commands.length; i++) {
let c = commands[i];
let n = c.usage.split(' ');
n = n[0];
n = n[0].trim();
if (n == name) return c;
if (c.aliases && c.aliases.indexOf(name) >= 0) return c;
}
@ -453,6 +438,58 @@ function execCommand(name, args) {
});
}
// async function execCommand(args) {
// var parseArgs = require('minimist');
// let results = parseArgs(args);
// //var results = vorpal.parse(args, { use: 'minimist' });
// if (!results['_'].length) throw new Error(_('Invalid command: %s', args));
// console.info(results);
// let commandName = results['_'].splice(0, 1);
// let cmd = commandByName(commandName);
// if (!cmd) throw new Error(_('Unknown command: %s', args));
// let usage = cmd.usage.split(' ');
// let commandArgs = [];
// usage.splice(0, 1);
// for (let i = 0; i < usage.length; i++) {
// let u = usage[i].trim();
// if (u == '') continue;
// let required = false;
// if (u.length >= 3 && u[0] == '<' && u[u.length - 1] == '>') {
// required = true;
// u = u.substr(1, u.length - 2);
// }
// if (u.length >= 3 && u[0] == '[' && u[u.length - 1] == ']') {
// u = u.substr(1, u.length - 2);
// }
// if (required && !results['_'].length) throw new Error(_('Missing argument: %s', args));
// if (!results['_'].length) break;
// console.info(u);
// commandArgs[u] = results['_'].splice(0, 1);
// }
// console.info(commandArgs);
// // usage: 'import-enex <file> [notebook]',
// // description: _('Imports en Evernote notebook file (.enex file).'),
// // options: [
// // ['--fuzzy-matching', 'For debugging purposes. Do not use.'],
// // ],
// }
async function synchronizer(syncTarget) {
if (synchronizers_[syncTarget]) return synchronizers_[syncTarget];
@ -580,53 +617,49 @@ function cmdPromptConfirm(commandInstance, message) {
});
}
// Handles the initial arguments passed to main script and
// route them to the "root" command.
function handleStartArgs(argv) {
return new Promise((resolve, reject) => {
while (true) {
// Handles the initial flags passed to main script and
// returns the remaining args.
async function handleStartFlags(argv) {
argv = argv.slice(0);
argv.splice(0, 2); // First arguments are the node executable, and the node JS file
while (argv.length) {
let arg = argv[0];
let nextArg = argv.length >= 2 ? argv[1] : null;
if (arg == '--profile') {
if (!nextArg) {
throw new Error(_('Usage: --profile <dir-path>'));
}
initArgs.profileDir = nextArg;
argv.splice(0, 2);
if (argv[0] == '--profile') {
argv.splice(0, 1);
if (!argv.length) throw new Error(_('Profile path is missing'));
initArgs.profileDir = argv[0];
argv.splice(0, 1);
} else if (argv[0][0] === '-') {
throw new Error(_('Unknown flag: "%s"', argv[0]));
continue;
}
if (!argv.length || argv[0][0] != '-') {
resolve(argv);
if (arg.length && arg[0] == '-') {
throw new Error(_('Unknown flag: %s', arg));
} else {
break;
}
}
return;
// if (argv && argv.length >= 3 && argv[2][0] == '-') {
// const startParams = vorpal.parse(argv, { use: 'minimist' });
// const cmd = commandByName('root');
// cmd.action(startParams, (newArgs) => {
// console.info(newArgs);
// resolve();
// });
// } else {
// console.info(argv);
// resolve();
// }
return argv;
}
// if (argv && argv.length >= 3 && argv[2][0] == '-') {
// const startParams = vorpal.parse(argv, { use: 'minimist' });
// const cmd = commandByName('root');
// cmd.action(startParams, (newArgs) => {
// console.info(newArgs);
// resolve();
// });
// } else {
// console.info(argv);
// resolve();
// }
});
function escapeShellArg(arg) {
if (arg.indexOf('"') >= 0 && arg.indexOf("'") >= 0) throw new Error(_('Command line argument "%s" contains both quotes and double-quotes - aborting.', arg)); // Hopeless case
let quote = '"';
if (arg.indexOf('"') >= 0) quote = "'";
if (arg.indexOf(' ') >= 0 || arg.indexOf("\t") >= 0) return quote + arg + quote;
return arg;
}
function shellArgsToString(args) {
let output = [];
for (let i = 0; i < args.length; i++) {
output.push(escapeShellArg(args[i]));
}
return output.join(' ');
}
process.stdin.on('keypress', (_, key) => {
@ -645,7 +678,6 @@ const vorpal = require('vorpal')();
async function main() {
for (let commandIndex = 0; commandIndex < commands.length; commandIndex++) {
let c = commands[commandIndex];
if (c.usage == 'root') continue;
let o = vorpal.command(c.usage, c.description);
if (c.options) {
for (let i = 0; i < c.options.length; i++) {
@ -669,7 +701,8 @@ async function main() {
vorpal.history('net.cozic.joplin'); // Enables persistent history
await handleStartArgs(process.argv);
let argv = process.argv;
argv = await handleStartFlags(argv);
const profileDir = initArgs.profileDir ? initArgs.profileDir : os.homedir() + '/.config/' + Setting.value('appName');
const resourceDir = profileDir + '/resources';
@ -704,10 +737,19 @@ async function main() {
if (!activeFolder) activeFolder = await Folder.defaultFolder();
if (!activeFolder) activeFolder = await Folder.createDefaultFolder();
if (!activeFolder) throw new Error(_('No default notebook is defined and could not create a new one. The database might be corrupted, please delete it and try again.'));
Setting.setValue('activeFolderId', activeFolder.id);
if (activeFolder) await execCommand('cd', { 'notebook': activeFolder.title }); // Use execCommand() so that no history entry is created
await execCommand('cd', { 'notebook': activeFolder.title }); // Use execCommand() so that no history entry is created
vorpal.delimiter(promptString()).show();
// If we still have arguments, pass it to Vorpal and exit
if (argv.length) {
let cmd = shellArgsToString(argv);
vorpal.log(_('Executing: %s', cmd));
await vorpal.exec(cmd);
await vorpal.exec('exit');
return;
}
}
main().catch((error) => {

2
CliClient/package.json
View File

@ -1,6 +1,6 @@
{
"name": "joplin-cli",
"version": "0.8.11",
"version": "0.8.14",
"bin": {
"joplin": "./main.sh"
},

4
CliClient/run.sh
View File

@ -1,4 +1,6 @@
#!/bin/bash
set -e
CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Downloads/desktop/afaire.enex afaire
bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes
#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/afaire.enex afaire
#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/Laurent.enex laurent

3
CliClient/tests/synchronizer.js
View File

@ -20,6 +20,7 @@ async function allItems() {
async function localItemsSameAsRemote(locals, expect) {
try {
let files = await fileApi().list();
files = files.items;
expect(locals.length).toBe(files.length);
for (let i = 0; i < locals.length; i++) {
@ -116,7 +117,6 @@ describe('Synchronizer', function() {
await synchronizer().start();
let all = await allItems();
let files = await fileApi().list();
await localItemsSameAsRemote(all, expect);
@ -227,6 +227,7 @@ describe('Synchronizer', function() {
await synchronizer().start();
let files = await fileApi().list();
files = files.items;
expect(files.length).toBe(1);
expect(files[0].path).toBe(Folder.systemPath(folder1));

8
lib/file-api-driver-local.js
View File

@ -53,7 +53,7 @@ class FileApiDriverLocal {
});
}
list(path) {
list(path, options) {
return new Promise((resolve, reject) => {
fs.readdir(path, (error, items) => {
if (error) {
@ -75,7 +75,11 @@ class FileApiDriverLocal {
return promiseChain(chain).then((results) => {
if (!results) results = [];
resolve(results);
resolve({
items: results
hasMore: false,
context: null,
});
}).catch((error) => {
reject(error);
});

8
lib/file-api-driver-memory.js
View File

@ -41,7 +41,7 @@ class FileApiDriverMemory {
return Promise.resolve();
}
list(path) {
list(path, options) {
let output = [];
for (let i = 0; i < this.items_.length; i++) {
@ -57,7 +57,11 @@ class FileApiDriverMemory {
}
}
return Promise.resolve(output);
return Promise.resolve({
items: output,
hasMore: false,
context: null,
});
}
get(path) {

23
lib/file-api-driver-onedrive.js
View File

@ -45,7 +45,7 @@ class FileApiDriverOneDrive {
try {
item = await this.api_.execJson('GET', this.makePath_(path), this.itemFilter_());
} catch (error) {
if (error.error.code == 'itemNotFound') return null;
if (error.code == 'itemNotFound') return null;
throw error;
}
return item;
@ -67,9 +67,22 @@ class FileApiDriverOneDrive {
return this.makeItem_(item);
}
async list(path) {
let items = await this.api_.execJson('GET', this.makePath_(path) + ':/children', this.itemFilter_());
return this.makeItems_(items.value);
async list(path, options = null) {
let query = this.itemFilter_();
let url = this.makePath_(path) + ':/children';
if (options.context) {
query = null;
url = options.context;
}
let r = await this.api_.execJson('GET', url, query);
return {
hasMore: !!r['@odata.nextLink'],
items: this.makeItems_(r.value),
context: r["@odata.nextLink"],
}
}
async get(path) {
@ -77,7 +90,7 @@ class FileApiDriverOneDrive {
try {
content = await this.api_.execText('GET', this.makePath_(path) + ':/content');
} catch (error) {
if (error.error.code == 'itemNotFound') return null;
if (error.code == 'itemNotFound') return null;
throw error;
}
return content;

14
lib/file-api.js
View File

@ -27,17 +27,19 @@ class FileApi {
list(path = '', options = null) {
if (!options) options = {};
if (!('includeHidden' in options)) options.includeHidden = false;
if (!('context' in options)) options.context = null;
this.logger().debug('list');
return this.driver_.list(this.baseDir_).then((items) => {
this.logger().debug('list ' + this.baseDir_);
return this.driver_.list(this.baseDir_, options).then((result) => {
if (!options.includeHidden) {
let temp = [];
for (let i = 0; i < items.length; i++) {
if (!isHidden(items[i].path)) temp.push(items[i]);
for (let i = 0; i < result.items.length; i++) {
if (!isHidden(result.items[i].path)) temp.push(result.items[i]);
}
items = temp;
result.items = temp;
}
return items;
return result;
});
}

42
lib/onedrive-api.js
View File

@ -67,6 +67,20 @@ class OneDriveApi {
return 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize?' + stringify(query);
}
oneDriveErrorResponseToError(errorResponse) {
if (!errorResponse) return new Error('Undefined error');
if (errorResponse.error) {
let e = errorResponse.error;
let output = new Error(e.message);
if (e.code) output.code = e.code;
if (e.innerError) output.innerError = e.innerError;
return output;
} else {
return new Error(JSON.stringify(errorResponse));
}
}
async exec(method, path, query = null, data = null, options = null) {
method = method.toUpperCase();
@ -82,26 +96,42 @@ class OneDriveApi {
if (data) data = JSON.stringify(data);
}
let url = 'https://graph.microsoft.com/v1.0' + path;
let url = path;
if (query) url += '?' + stringify(query);
// In general, `path` contains a path relative to the base URL, but in some
// cases the full URL is provided (for example, when it's a URL that was
// retrieved from the API).
if (url.indexOf('https://') !== 0) url = 'https://graph.microsoft.com/v1.0' + path;
if (query) {
url += url.indexOf('?') < 0 ? '?' : '&';
url += stringify(query);
}
if (data) options.body = data;
// console.info(method + ' ' + url);
// console.info(data);
// Rare error (one Google hit) - maybe repeat the request when it happens?
// { error:
// { code: 'generalException',
// message: 'An error occurred in the data store.',
// innerError:
// { 'request-id': 'b4310552-c18a-45b1-bde1-68e2c2345eef',
// date: '2017-06-29T00:15:50' } } }
for (let i = 0; i < 5; i++) {
options.headers['Authorization'] = 'bearer ' + this.token();
let response = await fetch(url, options);
if (!response.ok) {
let error = await response.json();
let errorResponse = await response.json();
let error = this.oneDriveErrorResponseToError(errorResponse);
if (error && error.error && error.error.code == 'InvalidAuthenticationToken') {
if (error.code == 'InvalidAuthenticationToken') {
await this.refreshAccessToken();
continue;
} else {
error.request = method + ' ' + url + ' ' + JSON.stringify(query) + ' ' + JSON.stringify(data) + ' ' + JSON.stringify(options);
throw error;
}
}

16
lib/synchronizer.js
View File

@ -104,6 +104,7 @@ class Synchronizer {
noteConflict: 0,
};
try {
await this.createWorkDir();
let donePaths = [];
@ -224,8 +225,11 @@ class Synchronizer {
// or handled as conflicts, so no conflict is possible after this.
let remoteIds = [];
let remotes = await this.api().list();
let context = null;
while (true) {
let listResult = await this.api().list('', { context: context });
let remotes = listResult.items;
for (let i = 0; i < remotes.length; i++) {
let remote = remotes[i];
let path = remote.path;
@ -271,6 +275,10 @@ class Synchronizer {
report[action]++;
}
if (!listResult.hasMore) break;
context = listResult.context;
}
// ------------------------------------------------------------------------
// Search, among the local IDs, those that don't exist remotely, which
// means the item has been deleted.
@ -285,13 +293,15 @@ class Synchronizer {
report['deleteLocal']++;
}
}
} catch (error) {
this.logger().error(error);
throw error;
}
this.logger().info('Synchronization complete [' + synchronizationId + ']:');
await this.logSyncSummary(report);
this.state_ = 'idle';
return Promise.resolve();
}
}