self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-23 18:53:36 +02:00
Code Issues Releases Activity

All: Improved resource side loading

Browse Source
This commit is contained in:
Laurent Cozic 2018-10-08 19:11:53 +01:00
parent dbdd602f50
commit 2f62897fb6
13 changed files with 191 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CliClient/app/app.js
View File

@ -21,7 +21,6 @@ const { _, setLocale, defaultLocale, closestSupportedLocale } = require('lib/loc
const os = require('os');
const fs = require('fs-extra');
const { cliUtils } = require('./cli-utils.js');
const EventEmitter = require('events');
const Cache = require('lib/Cache');
class Application extends BaseApplication {

9
CliClient/app/command-sync.js
View File

@ -4,6 +4,7 @@ const { _ } = require('lib/locale.js');
const { OneDriveApiNodeUtils } = require('./onedrive-api-node-utils.js');
const Setting = require('lib/models/Setting.js');
const BaseItem = require('lib/models/BaseItem.js');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const { Synchronizer } = require('lib/synchronizer.js');
const { reg } = require('lib/registry.js');
const { cliUtils } = require('./cli-utils.js');
@ -191,6 +192,14 @@ class Command extends BaseCommand {
}
}
// When using the tool in command line mode, the ResourceFetcher service is
// not going to be running in the background, so the resources need to be
// explicitely downloaded below.
if (!app().hasGui()) {
await ResourceFetcher.instance().fetchAll();
await ResourceFetcher.instance().waitForAllFinished();
}
await app().refreshCurrentFolder();
} catch (error) {
cleanUp();

2
CliClient/tests/synchronizer.js
View File

@ -905,7 +905,7 @@ describe('Synchronizer', function() {
// Simulate a failed download
get: () => { return new Promise((resolve, reject) => { reject(new Error('did not work')) }); }
} });
fetcher.queueResource(resource1.id);
fetcher.queueDownload(resource1.id);
await fetcher.waitForAllFinished();
resource1 = await Resource.load(resource1.id);

19
ElectronClient/app/gui/NoteText.jsx
View File

@ -28,6 +28,7 @@ const urlUtils = require('lib/urlUtils');
const dialogs = require('./dialogs');
const markdownUtils = require('lib/markdownUtils');
const ExternalEditWatcher = require('lib/services/ExternalEditWatcher');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const { toSystemSlashes, safeFilename } = require('lib/path-utils');
const { clipboard } = require('electron');
@ -198,6 +199,16 @@ class NoteTextComponent extends React.Component {
this.reloadNote(this.props);
}
}
this.resourceFetcher_downloadComplete = async (resource) => {
if (!this.state.note || !this.state.note.body) return;
const resourceIds = await Note.linkedResourceIds(this.state.note.body);
if (resourceIds.indexOf(resource.id) >= 0) {
this.mdToHtml().clearCache();
this.lastSetHtml_ = '';
this.updateHtml(this.state.note.body);
}
}
}
// Note:
@ -287,6 +298,8 @@ class NoteTextComponent extends React.Component {
eventManager.on('alarmChange', this.onAlarmChange_);
eventManager.on('noteTypeToggle', this.onNoteTypeToggle_);
eventManager.on('todoToggle', this.onTodoToggle_);
ResourceFetcher.instance().on('downloadComplete', this.resourceFetcher_downloadComplete);
}
componentWillUnmount() {
@ -299,6 +312,8 @@ class NoteTextComponent extends React.Component {
eventManager.removeListener('noteTypeToggle', this.onNoteTypeToggle_);
eventManager.removeListener('todoToggle', this.onTodoToggle_);
ResourceFetcher.instance().off('downloadComplete', this.resourceFetcher_downloadComplete);
this.destroyExternalEditWatcher();
}
@ -552,6 +567,10 @@ class NoteTextComponent extends React.Component {
if (!item) throw new Error('No item with ID ' + itemId);
if (item.type_ === BaseModel.TYPE_RESOURCE) {
if (item.fetch_status !== Resource.FETCH_STATUS_DONE || !!item.encryption_blob_encrypted) {
bridge().showErrorMessageBox(_('This attachment is not downloaded or not decrypted yet.'));
return;
}
const filePath = Resource.fullPath(item);
bridge().openItem(filePath);
} else if (item.type_ === BaseModel.TYPE_NOTE) {

7
ElectronClient/app/gui/SideBar.jsx
View File

@ -470,7 +470,13 @@ class SideBarComponent extends React.Component {
);
}
let decryptionReportText = '';
if (this.props.decryptionWorker && this.props.decryptionWorker.state !== 'idle' && this.props.decryptionWorker.itemCount) {
decryptionReportText = _('Decrypting items: %d/%d', this.props.decryptionWorker.itemIndex + 1, this.props.decryptionWorker.itemCount);
}
let lines = Synchronizer.reportToLines(this.props.syncReport);
if (decryptionReportText) lines.push(decryptionReportText);
const syncReportText = [];
for (let i = 0; i < lines.length; i++) {
syncReportText.push(
@ -510,6 +516,7 @@ const mapStateToProps = state => {
locale: state.settings.locale,
theme: state.settings.theme,
collapsedFolderIds: state.collapsedFolderIds,
decryptionWorker: state.decryptionWorker,
};
};

11
ReactNativeClient/lib/BaseApplication.js
View File

@ -33,6 +33,7 @@ const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
const EncryptionService = require('lib/services/EncryptionService');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const DecryptionWorker = require('lib/services/DecryptionWorker');
const BaseService = require('lib/services/BaseService');
@ -356,6 +357,10 @@ class BaseApplication {
DecryptionWorker.instance().scheduleStart();
}
if (this.hasGui() && action.type === 'SYNC_CREATED_RESOURCE') {
ResourceFetcher.instance().queueDownload(action.id);
}
return result;
}
@ -454,7 +459,7 @@ class BaseApplication {
initArgs = Object.assign(initArgs, extraFlags);
this.logger_.addTarget('file', { path: profileDir + '/log.txt' });
//this.logger_.addTarget('console');
// if (Setting.value('env') === 'dev') this.logger_.addTarget('console');
this.logger_.setLevel(initArgs.logLevel);
reg.setLogger(this.logger_);
@ -510,6 +515,10 @@ class BaseApplication {
DecryptionWorker.instance().setEncryptionService(EncryptionService.instance());
await EncryptionService.instance().loadMasterKeysFromSettings();
ResourceFetcher.instance().setFileApi(() => { return reg.syncTarget().fileApi() });
ResourceFetcher.instance().setLogger(this.logger_);
ResourceFetcher.instance().start();
let currentFolderId = Setting.value('activeFolderId');
let currentFolder = null;
if (currentFolderId) currentFolder = await Folder.load(currentFolderId);

18
ReactNativeClient/lib/MdToHtml.js
View File

@ -2,7 +2,6 @@ const MarkdownIt = require('markdown-it');
const Entities = require('html-entities').AllHtmlEntities;
const htmlentities = (new Entities()).encode;
const Resource = require('lib/models/Resource.js');
const ModelCache = require('lib/ModelCache');
const ObjectUtils = require('lib/ObjectUtils');
const { shim } = require('lib/shim.js');
const { _ } = require('lib/locale');
@ -17,7 +16,6 @@ class MdToHtml {
this.loadedResources_ = {};
this.cachedContent_ = null;
this.cachedContentKey_ = null;
this.modelCache_ = new ModelCache();
// Must include last "/"
this.resourceBaseUrl_ = ('resourceBaseUrl' in options) ? options.resourceBaseUrl : null;
@ -30,12 +28,18 @@ class MdToHtml {
const r = resources[n];
k.push(r.id);
}
k.push(md5(escape(body))); // https://github.com/pvorb/node-md5/issues/41
k.push(md5(JSON.stringify(style)));
k.push(md5(JSON.stringify(options)));
return k.join('_');
}
clearCache() {
this.cachedContent_ = null;
this.cachedContentKey_ = null;
}
renderAttrs_(attrs) {
if (!attrs) return '';
@ -74,8 +78,6 @@ class MdToHtml {
}
async loadResource(id, options) {
// console.info('Loading resource: ' + id);
// Initially set to to an empty object to make
// it clear that it is being loaded. Otherwise
// it sometimes results in multiple calls to
@ -83,7 +85,6 @@ class MdToHtml {
this.loadedResources_[id] = {};
const resource = await Resource.load(id);
//const resource = await this.modelCache_.load(Resource, id);
if (!resource) {
// Can happen for example if an image is attached to a note, but the resource hasn't
@ -92,6 +93,12 @@ class MdToHtml {
return;
}
if (resource.fetch_status !== Resource.FETCH_STATUS_DONE) {
delete this.loadedResources_[id];
console.warn('Resource not yet fetched: ' + id);
return;
}
this.loadedResources_[id] = resource;
if (options.onResourceLoaded) options.onResourceLoaded();
@ -209,7 +216,6 @@ class MdToHtml {
}
}
renderTokens_(markdownIt, tokens, options) {
let output = [];
let previousToken = null;

2
ReactNativeClient/lib/joplin-database.js
View File

@ -3,6 +3,7 @@ const { promiseChain } = require('lib/promise-utils.js');
const { time } = require('lib/time-utils.js');
const { Database } = require('lib/database.js');
const { sprintf } = require('sprintf-js');
const Resource = require('lib/models/Resource');
const structureSql = `
CREATE TABLE folders (
@ -398,6 +399,7 @@ class JoplinDatabase extends Database {
if (targetVersion == 13) {
queries.push('ALTER TABLE resources ADD COLUMN fetch_status INT NOT NULL DEFAULT "0"');
queries.push('ALTER TABLE resources ADD COLUMN fetch_error TEXT NOT NULL DEFAULT ""');
queries.push({ sql: 'UPDATE resources SET fetch_status = ?', params: [Resource.FETCH_STATUS_DONE] });
}
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });

4
ReactNativeClient/lib/models/BaseItem.js
View File

@ -291,8 +291,8 @@ class BaseItem extends BaseModel {
let shownKeys = ItemClass.fieldNames();
shownKeys.push('type_');
if (ItemClass.serializeForSyncExcludedKeys) {
const keys = ItemClass.serializeForSyncExcludedKeys();
if (ItemClass.syncExcludedKeys) {
const keys = ItemClass.syncExcludedKeys();
for (let i = 0; i < keys.length; i++) {
const idx = shownKeys.indexOf(keys[i]);
shownKeys.splice(idx, 1);

27
ReactNativeClient/lib/models/Resource.js
View File

@ -30,6 +30,27 @@ class Resource extends BaseItem {
return imageMimeTypes.indexOf(type.toLowerCase()) >= 0;
}
static resetStartedFetchStatus() {
return this.db().exec('UPDATE resources SET fetch_status = ? WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE, Resource.FETCH_STATUS_STARTED]);
}
static needToBeFetched(limit = null) {
let sql = 'SELECT * FROM resources WHERE fetch_status = ? ORDER BY updated_time DESC';
if (limit !== null) sql += ' LIMIT ' + limit;
return this.modelSelectAll(sql, [Resource.FETCH_STATUS_IDLE]);
}
static async saveFetchStatus(resourceId, status, error = null) {
const o = {
id: resourceId,
fetch_status: status,
}
if (error !== null) o.fetch_error = error;
return Resource.save(o, { autoTimestamp: false });
}
static fsDriver() {
if (!Resource.fsDriver_) Resource.fsDriver_ = new FsDriverDummy();
return Resource.fsDriver_;
@ -42,7 +63,7 @@ class Resource extends BaseItem {
return resource.id + extension;
}
static serializeForSyncExcludedKeys() {
static syncExcludedKeys() {
return ['fetch_status', 'fetch_error'];
}
@ -59,6 +80,10 @@ class Resource extends BaseItem {
return Setting.value('resourceDir') + '/' + this.filename(resource, encryptedBlob);
}
static isReady(resource) {
return resource && resource.fetch_status === Resource.FETCH_STATUS_DONE && !resource.encryption_blob_encrypted;
}
// For resources, we need to decrypt the item (metadata) and the resource binary blob.
static async decrypt(item) {
// The item might already be decrypted but not the blob (for instance if it crashes while

13
ReactNativeClient/lib/services/DecryptionWorker.js
View File

@ -1,4 +1,5 @@
const BaseItem = require('lib/models/BaseItem');
const Resource = require('lib/models/Resource');
const { Logger } = require('lib/logger.js');
class DecryptionWorker {
@ -43,7 +44,7 @@ class DecryptionWorker {
this.scheduleId_ = setTimeout(() => {
this.scheduleId_ = null;
this.start({
materKeyNotLoadedHandler: 'dispatch',
masterKeyNotLoadedHandler: 'dispatch',
});
}, 1000);
}
@ -56,7 +57,7 @@ class DecryptionWorker {
async start(options = null) {
if (options === null) options = {};
if (!('materKeyNotLoadedHandler' in options)) options.materKeyNotLoadedHandler = 'throw';
if (!('masterKeyNotLoadedHandler' in options)) options.masterKeyNotLoadedHandler = 'throw';
if (this.state_ !== 'idle') {
this.logger().info('DecryptionWorker: cannot start because state is "' + this.state_ + '"');
@ -83,6 +84,8 @@ class DecryptionWorker {
const ItemClass = BaseItem.itemClass(item);
if (('fetch_status' in item) && item.fetch_status !== Resource.FETCH_STATUS_DONE) continue;
this.dispatchReport({
itemIndex: i,
itemCount: items.length,
@ -95,7 +98,7 @@ class DecryptionWorker {
} catch (error) {
excludedIds.push(item.id);
if (error.code === 'masterKeyNotLoaded' && options.materKeyNotLoadedHandler === 'dispatch') {
if (error.code === 'masterKeyNotLoaded' && options.masterKeyNotLoadedHandler === 'dispatch') {
if (notLoadedMasterKeyDisptaches.indexOf(error.masterKeyId) < 0) {
this.dispatch({
type: 'MASTERKEY_ADD_NOT_LOADED',
@ -106,11 +109,11 @@ class DecryptionWorker {
continue;
}
if (error.code === 'masterKeyNotLoaded' && options.materKeyNotLoadedHandler === 'throw') {
if (error.code === 'masterKeyNotLoaded' && options.masterKeyNotLoadedHandler === 'throw') {
throw error;
}
this.logger().warn('DecryptionWorker: error for: ' + item.id + ' (' + ItemClass.tableName() + ')', error);
this.logger().warn('DecryptionWorker: error for: ' + item.id + ' (' + ItemClass.tableName() + ')', error, item);
}
}

93
ReactNativeClient/lib/services/ResourceFetcher.js
View File

@ -2,21 +2,35 @@ const Resource = require('lib/models/Resource');
const BaseService = require('lib/services/BaseService');
const BaseSyncTarget = require('lib/BaseSyncTarget');
const { Logger } = require('lib/logger.js');
const EventEmitter = require('events');
class ResourceFetcher extends BaseService {
constructor(fileApi) {
constructor(fileApi = null) {
super();
if (typeof fileApi !== 'function') throw new Error('fileApi must be a function that returns the API');
this.setFileApi(fileApi);
this.logger_ = new Logger();
this.fileApi_ = fileApi;
this.queue_ = [];
this.fetchingItems_ = {};
this.resourceDirName_ = BaseSyncTarget.resourceDirName();
this.queueMutex_ = new Mutex();
this.maxDownloads_ = 3;
this.addingResources_ = false;
this.eventEmitter_ = new EventEmitter();
}
static instance() {
if (this.instance_) return this.instance_;
this.instance_ = new ResourceFetcher();
return this.instance_;
}
on(eventName, callback) {
return this.eventEmitter_.on(eventName, callback);
}
off(eventName, callback) {
return this.eventEmitter_.removeListener(eventName, callback);
}
setLogger(logger) {
@ -27,7 +41,12 @@ class ResourceFetcher extends BaseService {
return this.logger_;
}
fileApi() {
setFileApi(v) {
if (v !== null && typeof v !== 'function') throw new Error('fileApi must be a function that returns the API. Type is ' + (typeof v));
this.fileApi_ = v;
}
async fileApi() {
return this.fileApi_();
}
@ -43,7 +62,7 @@ class ResourceFetcher extends BaseService {
if (priority === null) priority = 'normal';
const index = this.queuedItemIndex_(resourceId);
if (index >= 0) return;
if (index >= 0) return false;
const item = { id: resourceId };
@ -53,7 +72,8 @@ class ResourceFetcher extends BaseService {
this.queue_.push(item);
}
this.scheduleQueueProcess_();
this.scheduleQueueProcess();
return true;
}
async startDownload_(resourceId) {
@ -67,25 +87,39 @@ class ResourceFetcher extends BaseService {
const localResourceContentPath = Resource.fullPath(resource);
const remoteResourceContentPath = this.resourceDirName_ + "/" + resource.id;
await Resource.save({ id: resource.id, fetch_status: Resource.FETCH_STATUS_STARTED });
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_STARTED);
this.fileApi().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => {
const fileApi = await this.fileApi();
this.logger().debug('ResourceFetcher: Downloading resource: ' + resource.id);
const completeDownload = () => {
delete this.fetchingItems_[resource.id];
await Resource.save({ id: resource.id, fetch_status: Resource.FETCH_STATUS_DONE });
this.scheduleQueueProcess_();
this.scheduleQueueProcess();
this.eventEmitter_.emit('downloadComplete', { id: resource.id });
}
fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => {
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_DONE);
this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id);
completeDownload();
}).catch(async (error) => {
delete this.fetchingItems_[resource.id];
await Resource.save({ id: resource.id, fetch_status: Resource.FETCH_STATUS_ERROR, fetch_error: error.message });
this.scheduleQueueProcess_();
this.logger().error('ResourceFetcher: Could not download resource: ' + resource.id, error);
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_ERROR, error.message);
completeDownload();
});
}
processQueue_() {
while (Object.getOwnPropertyNames(this.fetchingItems_).length < this.maxDownloads_) {
if (!this.queue_.length) return;
if (!this.queue_.length) break;
const item = this.queue_.splice(0, 1)[0];
this.startDownload_(item.id);
}
if (!this.queue_.length) {
this.autoAddResources(10);
}
}
async waitForAllFinished() {
@ -99,7 +133,27 @@ class ResourceFetcher extends BaseService {
});
}
scheduleQueueProcess_() {
async autoAddResources(limit) {
if (this.addingResources_) return;
this.addingResources_ = true;
let count = 0;
const resources = await Resource.needToBeFetched(limit);
for (let i = 0; i < resources.length; i++) {
const added = this.queueDownload(resources[i].id);
if (added) count++;
}
this.logger().info('ResourceFetcher: Auto-added resources: ' + count);
this.addingResources_ = false;
}
async start() {
await Resource.resetStartedFetchStatus();
this.autoAddResources(10);
}
scheduleQueueProcess() {
if (this.scheduleQueueProcessIID_) {
clearTimeout(this.scheduleQueueProcessIID_);
this.scheduleQueueProcessIID_ = null;
@ -111,6 +165,11 @@ class ResourceFetcher extends BaseService {
}, 100);
}
async fetchAll() {
await Resource.resetStartedFetchStatus();
this.autoAddResources(null);
}
}
module.exports = ResourceFetcher;

34
ReactNativeClient/lib/synchronizer.js
View File

@ -555,24 +555,28 @@ class Synchronizer {
if (action == "createLocal") options.isNew = true;
if (action == "updateLocal") options.oldItem = local;
if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") {
let localResourceContentPath = Resource.fullPath(content);
let remoteResourceContentPath = this.resourceDirName_ + "/" + content.id;
try {
await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" });
} catch (error) {
if (error.code === 'rejectedByTarget') {
this.progressReport_.errors.push(error);
this.logger().warn('Rejected by target: ' + path + ': ' + error.message);
continue;
} else {
throw error;
}
}
}
// if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") {
// let localResourceContentPath = Resource.fullPath(content);
// let remoteResourceContentPath = this.resourceDirName_ + "/" + content.id;
// try {
// await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" });
// } catch (error) {
// if (error.code === 'rejectedByTarget') {
// this.progressReport_.errors.push(error);
// this.logger().warn('Rejected by target: ' + path + ': ' + error.message);
// continue;
// } else {
// throw error;
// }
// }
// }
await ItemClass.save(content, options);
if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") {
this.dispatch({ type: "SYNC_CREATED_RESOURCE", id: content.id });
}
if (!hasAutoEnabledEncryption && content.type_ === BaseModel.TYPE_MASTER_KEY && !masterKeysBefore) {
hasAutoEnabledEncryption = true;
this.logger().info("One master key was downloaded and none was previously available: automatically enabling encryption");