self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-10-31 00:07:48 +02:00
Code Issues Releases Activity

Handle delta api for filesystem

Browse Source
This commit is contained in:
Laurent Cozic
2017-07-18 23:14:20 +01:00
parent 0c30c1b70b
commit 7aa21174f6
16 changed files with 185 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
CliClient/app/app.js
View File

@@ -219,6 +219,8 @@ class Application {
let CommandClass = require('./' + path);
let cmd = new CommandClass();
if (!cmd.enabled()) return;
let vorpalCmd = this.vorpal().command(cmd.usage(), cmd.description());
vorpalCmd.__commandObject = cmd;
@@ -261,43 +263,44 @@ class Application {
if (cmd.hidden()) vorpalCmd.hidden();
});
this.vorpal().catch('[args...]', 'Catches undefined commands').action(function(args, end) {
args = args.args;
// this.vorpal().catch('[args...]', 'Catches undefined commands').action(function(args, end) {
// args = args.args;
function delayExec(command) {
setTimeout(() => {
app().vorpal().exec(command);
}, 100);
}
// function delayExec(command) {
// setTimeout(() => {
// app().vorpal().exec(command);
// }, 100);
// }
if (!args.length) {
end();
delayExec('help');
return;
}
// if (!args.length) {
// end();
// delayExec('help');
// return;
// }
let commandName = args.splice(0, 1);
// let commandName = args.splice(0, 1);
let aliases = Setting.value('aliases').trim();
aliases = aliases.length ? JSON.parse(aliases) : [];
// let aliases = Setting.value('aliases').trim();
// aliases = aliases.length ? JSON.parse(aliases) : [];
for (let i = 0; i < aliases.length; i++) {
const alias = aliases[i];
if (alias.name == commandName) {
let command = alias.command + ' ' + app().shellArgsToString(args);
end();
delayExec(command);
return;
}
}
// for (let i = 0; i < aliases.length; i++) {
// const alias = aliases[i];
// if (alias.name == commandName) {
// let command = alias.command + ' ' + app().shellArgsToString(args);
// end();
// delayExec(command);
// return;
// }
// }
this.log(_("Invalid command. Showing help:"));
end();
delayExec('help');
});
// this.log(_("Invalid command. Showing help:"));
// end();
// delayExec('help');
// });
}
async synchronizer(syncTarget) {
async synchronizer(syncTarget, options = null) {
if (!options) options = {};
if (this.synchronizers_[syncTarget]) return this.synchronizers_[syncTarget];
let fileApi = null;
@@ -305,6 +308,7 @@ class Application {
// TODO: create file api based on syncTarget
if (syncTarget == 'onedrive') {
const oneDriveApi = reg.oneDriveApi();
let driver = new FileApiDriverOneDrive(oneDriveApi);
let auth = Setting.value('sync.onedrive.auth');
@@ -320,18 +324,25 @@ class Application {
this.logger_.info('App dir: ' + appDir);
fileApi = new FileApi(appDir, driver);
fileApi.setLogger(this.logger_);
} else if (syncTarget == 'memory') {
fileApi = new FileApi('joplin', new FileApiDriverMemory());
fileApi.setLogger(this.logger_);
} else if (syncTarget == 'filesystem') {
let syncDir = Setting.value('sync.filesystem.path');
let syncDir = options['sync.filesystem.path'] ? options['sync.filesystem.path'] : Setting.value('sync.filesystem.path');
if (!syncDir) syncDir = Setting.value('profileDir') + '/sync';
this.vorpal().log(_('Synchronizing with directory "%s"', syncDir));
await fs.mkdirp(syncDir, 0o755);
fileApi = new FileApi(syncDir, new FileApiDriverLocal());
fileApi.setLogger(this.logger_);
} else {
throw new Error('Unknown backend: ' + syncTarget);
}
this.synchronizers_[syncTarget] = new Synchronizer(this.database_, fileApi, Setting.value('appType'));

4
CliClient/app/base-command.js
View File

@@ -28,6 +28,10 @@ class BaseCommand {
return false;
}
enabled() {
return true;
}
async cancel() {}
}

7
CliClient/app/build-translation.js
View File

@@ -47,7 +47,12 @@ function serializeTranslation(translation) {
for (let n in translations) {
if (!translations.hasOwnProperty(n)) continue;
if (n == '') continue;
output[n] = translations[n]['msgstr'][0];
const t = translations[n];
if (t.comments && t.comments.flag && t.comments.flag.indexOf('fuzzy') >= 0) {
output[n] = t['msgid'];
} else {
output[n] = t['msgstr'][0];
}
}
return JSON.stringify(output);
}

4
CliClient/app/command-alias.js
View File

@@ -23,6 +23,10 @@ class Command extends BaseCommand {
Setting.setValue('aliases', JSON.stringify(aliases));
}
enabled() {
return false; // Doesn't work properly at the moment
}
}
module.exports = Command;

6
CliClient/app/command-sync.js
View File

@@ -27,6 +27,7 @@ class Command extends BaseCommand {
options() {
return [
['--target <target>', _('Sync to provided target (defaults to sync.target config value)')],
['--filesystem-path <path>', _('For "filesystem" target only: Path to sync to.')],
['--random-failures', 'For debugging purposes. Do not use.'],
];
}
@@ -71,7 +72,10 @@ class Command extends BaseCommand {
this.syncTarget_ = Setting.value('sync.target');
if (args.options.target) this.syncTarget_ = args.options.target;
let sync = await app().synchronizer(this.syncTarget_);
let syncInitOptions = {};
if (args.options['filesystem-path']) syncInitOptions['sync.filesystem.path'] = args.options['filesystem-path'];
let sync = await app().synchronizer(this.syncTarget_, syncInitOptions);
let options = {
onProgress: (report) => {

26
CliClient/locales/en_GB.po
View File

@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 20:17+0100\n"
"POT-Creation-Date: 2017-07-18 23:12+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -57,16 +57,12 @@ msgstr ""
msgid "Exits the application."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:294
msgid "Invalid command. Showing help:"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:329
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:337
#, javascript-format
msgid "Synchronizing with directory \"%s\""
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:408
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:419
msgid "No notebook is defined. Create one with `mkbook <notebook>`."
msgstr ""
@@ -299,28 +295,32 @@ msgstr ""
msgid "Sync to provided target (defaults to sync.target config value)"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:66
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30
msgid "For \"filesystem\" target only: Path to sync to."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67
msgid "Synchronisation is already in progress."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:88
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#, javascript-format
msgid "Synchronization target: %s"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:90
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94
msgid "Cannot initialize synchronizer."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
msgid "Starting synchronization..."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:103
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107
msgid "Done."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:118
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..."
msgstr ""

31
CliClient/locales/fr_FR.po
View File

@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 20:13+0100\n"
"POT-Creation-Date: 2017-07-18 23:12+0100\n"
"PO-Revision-Date: 2017-07-18 13:27+0100\n"
"Last-Translator: \n"
"Language-Team: \n"
@@ -59,17 +59,12 @@ msgstr "Affiche l'aide pour la commande donnée."
msgid "Exits the application."
msgstr "Quitter le logiciel."
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:294
#, fuzzy
msgid "Invalid command. Showing help:"
msgstr "Commande invalie : \"%s\""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:329
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:337
#, javascript-format
msgid "Synchronizing with directory \"%s\""
msgstr "Synchronisation avec dossier \"%s\""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:408
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:419
msgid "No notebook is defined. Create one with `mkbook <notebook>`."
msgstr "Aucun carnet n'est défini. Créez-en un avec `mkbook <carnet>`."
@@ -325,28 +320,32 @@ msgstr ""
"Synchroniser avec la cible donnée (par défaut, la valeur de configuration "
"`sync.target`)."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:66
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30
msgid "For \"filesystem\" target only: Path to sync to."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67
msgid "Synchronisation is already in progress."
msgstr "Synchronisation est déjà en cours."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:88
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#, javascript-format
msgid "Synchronization target: %s"
msgstr "Cible de la synchronisation : %s"
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:90
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94
msgid "Cannot initialize synchronizer."
msgstr "Impossible d'initialiser le synchroniseur."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
msgid "Starting synchronization..."
msgstr "Commencement de la synchronisation..."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:103
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107
msgid "Done."
msgstr "Terminé."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:118
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..."
msgstr "Annulation..."
@@ -516,6 +515,10 @@ msgstr "Carnets"
msgid "%s: %d notes"
msgstr "%s : %d notes"
#, fuzzy
#~ msgid "Invalid command. Showing help:"
#~ msgstr "Commande invalie : \"%s\""
#~ msgid "cat <title>"
#~ msgstr "cat <titre>"

26
CliClient/locales/joplin.pot
View File

@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 20:17+0100\n"
"POT-Creation-Date: 2017-07-18 23:12+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -57,16 +57,12 @@ msgstr ""
msgid "Exits the application."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:294
msgid "Invalid command. Showing help:"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:329
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:337
#, javascript-format
msgid "Synchronizing with directory \"%s\""
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:408
#: /mnt/d/Web/www/joplin/CliClient/app/app.js:419
msgid "No notebook is defined. Create one with `mkbook <notebook>`."
msgstr ""
@@ -299,28 +295,32 @@ msgstr ""
msgid "Sync to provided target (defaults to sync.target config value)"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:66
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30
msgid "For \"filesystem\" target only: Path to sync to."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67
msgid "Synchronisation is already in progress."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:88
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#, javascript-format
msgid "Synchronization target: %s"
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:90
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94
msgid "Cannot initialize synchronizer."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
msgid "Starting synchronization..."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:103
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107
msgid "Done."
msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:118
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..."
msgstr ""

2
CliClient/package.json
View File

@@ -7,7 +7,7 @@
"url": "https://github.com/laurent22/joplin"
},
"url": "git://github.com/laurent22/joplin.git",
"version": "0.8.55",
"version": "0.8.58",
"bin": {
"joplin": "./main_launcher.js"
},

1
CliClient/tests/test-utils.js
View File

@@ -91,7 +91,6 @@ function setupDatabase(id = null) {
// Don't care if the file doesn't exist
}).then(() => {
databases_[id] = new JoplinDatabase(new DatabaseDriverNode());
//databases_[id].setLogger(logger);
return databases_[id].open({ name: filePath }).then(() => {
BaseModel.db_ = databases_[id];
return setupDatabase(id);

48
ReactNativeClient/lib/file-api-driver-local.js
View File

@@ -1,6 +1,7 @@
import fs from 'fs-extra';
import { promiseChain } from 'lib/promise-utils.js';
import moment from 'moment';
import { BaseItem } from 'lib/models/base-item.js';
import { time } from 'lib/time-utils.js';
class FileApiDriverLocal {
@@ -20,6 +21,10 @@ class FileApiDriverLocal {
return output;
}
supportsDelta() {
return false;
}
stat(path) {
return new Promise((resolve, reject) => {
fs.stat(path, (error, s) => {
@@ -68,6 +73,49 @@ class FileApiDriverLocal {
});
}
async delta(path, options) {
try {
let items = await fs.readdir(path);
let output = [];
for (let i = 0; i < items.length; i++) {
let stat = await this.stat(path + '/' + items[i]);
if (!stat) continue; // Has been deleted between the readdir() call and now
stat.path = items[i];
output.push(stat);
}
if (!Array.isArray(options.itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
let deletedItems = [];
for (let i = 0; i < options.itemIds.length; i++) {
const itemId = options.itemIds[i];
let found = false;
for (let j = 0; j < output.length; j++) {
const item = output[j];
if (BaseItem.pathToId(item.path) == itemId) {
found = true;
break;
}
}
if (!found) {
deletedItems.push({
path: BaseItem.systemPath(itemId),
isDeleted: true,
});
}
}
return {
hasMore: false,
context: null,
items: output,
};
} catch(error) {
throw this.fsErrorToJsError_(error);
}
}
async list(path, options) {
try {
let items = await fs.readdir(path);

4
ReactNativeClient/lib/file-api-driver-memory.js
View File

@@ -15,6 +15,10 @@ class FileApiDriverMemory {
this.deletedItems_ = [];
}
supportsDelta() {
return true;
}
itemIndexByPath(path) {
for (let i = 0; i < this.items_.length; i++) {
if (this.items_[i].path == path) return i;

4
ReactNativeClient/lib/file-api-driver-onedrive.js
View File

@@ -21,6 +21,10 @@ class FileApiDriverOneDrive {
return this.api_;
}
supportsDelta() {
return true;
}
itemFilter_() {
return {
select: 'name,file,folder,fileSystemInfo',

4
ReactNativeClient/lib/file-api.js
View File

@@ -13,6 +13,10 @@ class FileApi {
return this.driver_;
}
supportsDelta() {
return this.driver_.supportsDelta();
}
setLogger(l) {
this.logger_ = l;
}

3
ReactNativeClient/lib/models/base-item.js
View File

@@ -79,7 +79,8 @@ class BaseItem extends BaseModel {
}
static pathToId(path) {
let s = path.split('.');
let p = path.split('/');
let s = p[p.length - 1].split('.');
return s[0];
}

50
ReactNativeClient/lib/synchronizer.js
View File

@@ -102,6 +102,9 @@ class Synchronizer {
for (let n in report) {
if (!report.hasOwnProperty(n)) continue;
if (n == 'errors') continue;
if (n == 'starting') continue;
if (n == 'finished') continue;
if (n == 'state') continue;
this.logger().info(n + ': ' + (report[n] ? report[n] : '-'));
}
let folderCount = await Folder.count();
@@ -327,7 +330,15 @@ class Synchronizer {
while (true) {
if (this.cancelling()) break;
let listResult = await this.api().delta('', { context: context });
let allIds = null;
if (!this.api().supportsDelta()) {
allIds = await BaseItem.syncedItems(syncTargetId);
}
let listResult = await this.api().delta('', {
context: context,
itemIds: allIds,
});
let remotes = listResult.items;
for (let i = 0; i < remotes.length; i++) {
if (this.cancelling()) break;
@@ -335,8 +346,6 @@ class Synchronizer {
let remote = remotes[i];
if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder
//console.info(remote);
let path = remote.path;
let action = null;
let reason = '';
@@ -410,34 +419,13 @@ class Synchronizer {
outputContext.delta = newDeltaContext ? newDeltaContext : lastContext.delta;
// // ------------------------------------------------------------------------
// // Search, among the local IDs, those that don't exist remotely, which
// // means the item has been deleted.
// // ------------------------------------------------------------------------
// if (this.randomFailure(options, 4)) return;
// let localFoldersToDelete = [];
// if (!this.cancelling()) {
// let syncItems = await BaseItem.syncedItems(syncTargetId);
// for (let i = 0; i < syncItems.length; i++) {
// if (this.cancelling()) break;
// let syncItem = syncItems[i];
// if (remoteIds.indexOf(syncItem.item_id) < 0) {
// if (syncItem.item_type == Folder.modelType()) {
// localFoldersToDelete.push(syncItem);
// continue;
// }
// this.logSyncOperation('deleteLocal', { id: syncItem.item_id }, null, 'remote has been deleted');
// let ItemClass = BaseItem.itemClass(syncItem.item_type);
// await ItemClass.delete(syncItem.item_id, { trackDeleted: false });
// }
// }
// }
// ------------------------------------------------------------------------
// Delete the folders that have been collected in the loop above.
// Folders are always deleted last, and only if they are empty.
// If they are not empty it's considered a conflict since whatever deleted
// them should have deleted their content too. In that case, all its notes
// are marked as "is_conflict".
// ------------------------------------------------------------------------
if (!this.cancelling()) {
for (let i = 0; i < localFoldersToDelete.length; i++) {