self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

Allow cancelling sync and fixed resource issue

Browse Source
This commit is contained in:
Laurent Cozic 2017-07-09 16:47:05 +01:00
parent de16573742
commit 297edfa682
11 changed files with 210 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
CliClient/app/main.js
View File

@ -28,6 +28,7 @@ import { vorpalUtils } from 'vorpal-utils.js';
import { reg } from 'lib/registry.js';
import { FsDriverNode } from './fs-driver-node.js';
import { filename, basename } from 'lib/path-utils.js';
import { shim } from 'lib/shim.js';
import { _ } from 'lib/locale.js';
import os from 'os';
import fs from 'fs-extra';
@ -521,7 +522,7 @@ commands.push({
if (report.remotesToDelete) line.push(_('Remote items to delete: %d/%d.', report.deleteRemote, report.remotesToDelete));
if (report.localsToUdpate) line.push(_('Items to download: %d/%d.', report.createLocal + report.updateLocal, report.localsToUdpate));
if (report.localsToDelete) line.push(_('Local items to delete: %d/%d.', report.deleteLocal, report.localsToDelete));
vorpalUtils.redraw(line.join(' '));
if (line.length) vorpalUtils.redraw(line.join(' '));
},
onMessage: (msg) => {
vorpalUtils.redrawDone();
@ -549,6 +550,12 @@ commands.push({
this.log(_('Done.'));
end();
},
cancel: async function() {
vorpalUtils.redrawDone();
this.log(_('Cancelling...'));
let sync = await synchronizer(Setting.value('sync.target'));
sync.cancel();
},
});
commands.push({
@ -896,9 +903,74 @@ const vorpal = require('vorpal')();
vorpalUtils.initialize(vorpal);
async function main() {
shim.fetchBlob = async function(url, options) {
if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
if (!options.method) options.method = 'GET';
const urlParse = require('url').parse;
url = urlParse(url.trim());
const http = url.protocol.toLowerCase() == 'http:' ? require('follow-redirects').http : require('follow-redirects').https;
const headers = options.headers ? options.headers : {};
const method = options.method ? options.method : 'GET';
if (method != 'GET') throw new Error('Only GET is supported');
const filePath = options.path;
function makeResponse(response) {
const output = {
ok: response.statusCode < 400,
path: filePath,
text: () => { return response.statusMessage; },
json: () => { return ''; },
status: response.statusCode,
headers: response.headers,
}
console.info(output);
return output;
}
const requestOptions = {
protocol: url.protocol,
host: url.host,
port: url.port,
method: method,
path: url.path + (url.query ? '?' + url.query : ''),
headers: headers,
};
return new Promise((resolve, reject) => {
try {
// Note: relative paths aren't supported
const file = fs.createWriteStream(filePath);
const request = http.get(requestOptions, function(response) {
response.pipe(file);
file.on('finish', function() {
console.info('FINISH');
file.close(() => {
console.info('FINISH CLOSE');
resolve(makeResponse(response));
});
});
})
request.on('error', function(error) {
fs.unlink(filePath);
reject(error);
});
} catch(error) {
fs.unlink(filePath);
reject(error);
}
});
}
for (let commandIndex = 0; commandIndex < commands.length; commandIndex++) {
let c = commands[commandIndex];
let o = vorpal.command(c.usage, c.description);
if (c.options) {
for (let i = 0; i < c.options.length; i++) {
let options = c.options[i];
@ -906,16 +978,23 @@ async function main() {
if (options.length == 3) o.option(options[0], options[1], options[2]);
}
}
if (c.aliases) {
for (let i = 0; i < c.aliases.length; i++) {
o.alias(c.aliases[i]);
}
}
if (c.autocomplete) {
o.autocomplete({
data: c.autocomplete,
});
}
if (c.cancel) {
o.cancel(c.cancel);
}
o.action(c.action);
}
@ -936,14 +1015,15 @@ async function main() {
logger.addTarget('file', { path: profileDir + '/log.txt' });
logger.setLevel(logLevel);
reg.setLogger(logger);
dbLogger.addTarget('file', { path: profileDir + '/log-database.txt' });
dbLogger.setLevel(logLevel);
syncLogger.addTarget('file', { path: profileDir + '/log-sync.txt' });
syncLogger.setLevel(logLevel);
logger.info(sprintf('Starting %s %s...', packageJson.name, packageJson.version));
logger.info('Environment: ' + Setting.value('env'));
logger.info(sprintf('Starting %s %s (%s)...', packageJson.name, packageJson.version, Setting.value('env')));
logger.info('Profile directory: ' + profileDir);
// That's not good, but it's to avoid circular dependency issues

32
CliClient/app/main_launcher.js Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env node
// Because all the files in the "lib" directory are included as "lib/file.js" it
// means "lib" must be in NODE_PATH, however modifying the global NODE_PATH
// variable would be messy. So instead, the path is set temporarily just before running
// the app. To do this, this wrapper is needed.
// See https://gist.github.com/branneman/8048520
// Original wrapper code from https://gist.github.com/branneman/8775568
'use strict';
var spawn = require('child_process').spawn;
var args = ['main.js'];
var processArgs = process.argv.splice(2);
args = args.concat(processArgs);
var opt = {
cwd: __dirname,
env: (function() {
process.env.NODE_PATH = '.'; // Enables require() calls relative to the cwd :)
return process.env;
}()),
stdio: [process.stdin, process.stdout, process.stderr]
};
var app = spawn(process.execPath, args, opt);
// Pass on the exit code
app.on('close', (code) => {
process.exit(code);
});

13
CliClient/app/vorpal-utils.js
View File

@ -13,6 +13,15 @@ function initialize(vorpal) {
vorpal_ = vorpal;
}
function redrawEnabled() {
// Always disabled for now - doesn't play well with command.cancel()
// function (it makes the whole app quit instead of just the
// current command).
return false;
return redrawEnabled_;
}
function setRedrawEnabled(v) {
redrawEnabled_ = v;
}
@ -22,7 +31,7 @@ function setStackTraceEnabled(v) {
}
function redraw(s) {
if (!redrawEnabled_) {
if (!redrawEnabled()) {
const now = time.unixMs();
if (now - redrawLastUpdateTime_ > 1000) {
if (vorpal_.activeCommand) {
@ -45,7 +54,7 @@ function redraw(s) {
function redrawDone() {
if (!redrawStarted_) return;
if (!redrawEnabled_) {
if (!redrawEnabled()) {
if (redrawLastLog_) {
if (vorpal_.activeCommand) {
vorpal_.activeCommand.log(redrawLastLog_);

4
CliClient/install.sh
View File

@ -5,8 +5,8 @@ CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
npm version patch
bash $CLIENT_DIR/build.sh
cp "$CLIENT_DIR/package.json" build/
cp "$CLIENT_DIR/../lib/package.json" build/lib
cp "$CLIENT_DIR/app/main.sh" build/
cp "$CLIENT_DIR/../ReactNativeClient/lib/package.json" build/lib
#cp "$CLIENT_DIR/app/main.sh" build/
cd "$CLIENT_DIR/build"
sudo npm install -g --save
cd -

5
CliClient/package.json
View File

@ -7,13 +7,14 @@
"url": "https://github.com/laurent22/joplin"
},
"url": "git://github.com/laurent22/joplin.git",
"version": "0.8.27",
"version": "0.8.29",
"bin": {
"joplin": "./main.sh"
"joplin": "./main_launcher.js"
},
"dependencies": {
"app-module-path": "^2.2.0",
"babel-plugin-transform-runtime": "^6.23.0",
"follow-redirects": "^1.2.4",
"form-data": "^2.1.4",
"fs-extra": "^3.0.1",
"jssha": "^2.3.0",

2
CliClient/run.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
set -e
CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --redraw-disabled --env dev "$@"
bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev "$@"
#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/afaire.enex afaire
#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/Laurent.enex laurent

4
ReactNativeClient/android/app/build.gradle
View File

@ -90,8 +90,8 @@ android {
applicationId "net.cozic.joplin"
minSdkVersion 16
targetSdkVersion 22
versionCode 2
versionName "0.8.0"
versionCode 3
versionName "0.8.1"
ndk {
abiFilters "armeabi-v7a", "x86"
}

16
ReactNativeClient/lib/onedrive-api.js
View File

@ -1,6 +1,7 @@
import { shim } from 'lib/shim.js';
import { stringify } from 'query-string';
import { time } from 'lib/time-utils.js';
import { Logger } from 'lib/logger.js'
class OneDriveApi {
@ -16,6 +17,15 @@ class OneDriveApi {
this.listeners_ = {
'authRefreshed': [],
};
this.logger_ = new Logger();
}
setLogger(l) {
this.logger_ = l;
}
logger() {
return this.logger_;
}
isPublic() {
@ -161,17 +171,19 @@ class OneDriveApi {
let error = this.oneDriveErrorResponseToError(errorResponse);
if (error.code == 'InvalidAuthenticationToken' || error.code == 'unauthenticated') {
this.logger().info('Token expired: refreshing...');
await this.refreshAccessToken();
continue;
} else if (error && ((error.error && error.error.code == 'generalException') || (error.code == 'generalException'))) {
// Rare error (one Google hit) - I guess the request can be repeated
// { error:
// { code: 'generalException',
// message: 'An error occurred in the data store.',
// innerError:
// { 'request-id': 'b4310552-c18a-45b1-bde1-68e2c2345eef',
// date: '2017-06-29T00:15:50' } } }
this.logger().info('Got error below - retrying...');
this.logger().info(error);
await time.msleep(1000 * i);
continue;
} else if (error.code == 'EAGAIN') {
@ -181,6 +193,8 @@ class OneDriveApi {
// type: 'system',
// errno: 'EAGAIN',
// code: 'EAGAIN' }
this.logger().info('Got error below - retrying...');
this.logger().info(error);
await time.msleep(1000 * i);
continue;
} else {

5
ReactNativeClient/lib/registry.js
View File

@ -16,12 +16,17 @@ reg.logger = () => {
return reg.logger_;
}
reg.setLogger = (l) => {
reg.logger_ = l;
}
reg.oneDriveApi = () => {
if (reg.oneDriveApi_) return reg.oneDriveApi_;
const isPublic = Setting.value('appType') != 'cli';
reg.oneDriveApi_ = new OneDriveApi(parameters().oneDrive.id, parameters().oneDrive.secret, isPublic);
reg.oneDriveApi_.setLogger(reg.logger());
reg.oneDriveApi_.on('authRefreshed', (a) => {
reg.logger().info('Saving updated OneDrive auth.');

73
ReactNativeClient/lib/synchronizer.js
View File

@ -18,6 +18,7 @@ class Synchronizer {
this.resourceDirName_ = '.resource';
this.logger_ = new Logger();
this.appType_ = appType;
this.cancelling_ = false;
}
state() {
@ -86,6 +87,17 @@ class Synchronizer {
return false;
}
cancel() {
if (this.cancelling_) return;
this.logger().info('Cancelling synchronization...');
this.cancelling_ = true;
}
cancelling() {
return this.cancelling_;
}
async start(options = null) {
if (!options) options = {};
if (!options.onProgress) options.onProgress = function(o) {};
@ -96,6 +108,7 @@ class Synchronizer {
}
this.randomFailureChoice_ = Math.floor(Math.random() * 5);
this.cancelling_ = false;
// ------------------------------------------------------------------------
// First, find all the items that have been changed since the
@ -131,6 +144,8 @@ class Synchronizer {
let donePaths = [];
while (true) {
if (this.cancelling()) break;
let result = await BaseItem.itemsThatNeedSync();
let locals = result.items;
@ -138,6 +153,8 @@ class Synchronizer {
options.onProgress(report);
for (let i = 0; i < locals.length; i++) {
if (this.cancelling()) break;
let local = locals[i];
let ItemClass = BaseItem.itemClass(local);
let path = BaseItem.systemPath(local);
@ -249,6 +266,8 @@ class Synchronizer {
report.remotesToDelete = deletedItems.length;
options.onProgress(report);
for (let i = 0; i < deletedItems.length; i++) {
if (this.cancelling()) break;
let item = deletedItems[i];
let path = BaseItem.systemPath(item.item_id)
this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
@ -272,9 +291,13 @@ class Synchronizer {
let context = null;
while (true) {
if (this.cancelling()) break;
let listResult = await this.api().list('', { context: context });
let remotes = listResult.items;
for (let i = 0; i < remotes.length; i++) {
if (this.cancelling()) break;
let remote = remotes[i];
let path = remote.path;
@ -319,14 +342,17 @@ class Synchronizer {
if (newContent.type_ == BaseModel.TYPE_RESOURCE && action == 'createLocal') {
let localResourceContentPath = Resource.fullPath(newContent);
let remoteResourceContentPath = this.resourceDirName_ + '/' + newContent.id;
if (this.appType_ == 'cli') {
let remoteResourceContent = await this.api().get(remoteResourceContentPath, { encoding: 'binary' });
await Resource.setContent(newContent, remoteResourceContent);
} else if (this.appType_ == 'mobile') {
await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
} else {
throw new Error('Unknown appType: ' + this.appType_);
}
await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
// if (this.appType_ == 'cli') {
// let remoteResourceContent = await this.api().get(remoteResourceContentPath, { encoding: 'binary' });
// await Resource.setContent(newContent, remoteResourceContent);
// } else if (this.appType_ == 'mobile') {
// await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
// } else {
// throw new Error('Unknown appType: ' + this.appType_);
// }
}
await ItemClass.save(newContent, options);
@ -352,18 +378,22 @@ class Synchronizer {
if (this.randomFailure(options, 4)) return;
let items = await BaseItem.syncedItems();
for (let i = 0; i < items.length; i++) {
let item = items[i];
if (remoteIds.indexOf(item.id) < 0) {
report.localsToDelete++;
options.onProgress(report);
this.logSyncOperation('deleteLocal', { id: item.id }, null, 'remote has been deleted');
if (!this.cancelling()) {
let items = await BaseItem.syncedItems();
for (let i = 0; i < items.length; i++) {
if (this.cancelling()) break;
let ItemClass = BaseItem.itemClass(item);
await ItemClass.delete(item.id, { trackDeleted: false });
report['deleteLocal']++;
options.onProgress(report);
let item = items[i];
if (remoteIds.indexOf(item.id) < 0) {
report.localsToDelete++;
options.onProgress(report);
this.logSyncOperation('deleteLocal', { id: item.id }, null, 'remote has been deleted');
let ItemClass = BaseItem.itemClass(item);
await ItemClass.delete(item.id, { trackDeleted: false });
report['deleteLocal']++;
options.onProgress(report);
}
}
}
} catch (error) {
@ -371,6 +401,11 @@ class Synchronizer {
throw error;
}
if (this.cancelling()) {
this.logger().info('Synchronization was cancelled.');
this.cancelling_ = false;
}
this.logger().info('Synchronization complete [' + synchronizationId + ']:');
await this.logSyncSummary(report);

3
ReactNativeClient/start_server.bat
View File

@ -1,2 +1,3 @@
@echo off
npm start -- --reset-cache
npm start -- --reset-cache
rem "c:\Program Files (x86)\Yarn\bin\yarn.cmd" start -- --reset-cache