self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-06-21 23:17:42 +02:00
Code Issues Releases Activity

All: Fix integration test harness issues. (#2723)

Browse Source
This commit is contained in:
mic704b
2020-03-16 13:30:54 +11:00
committed by GitHub
parent 0863f0d564
commit 8cd26c9380
13 changed files with 424 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
ReactNativeClient/lib/BaseApplication.js
View File

@ -34,6 +34,7 @@ const EncryptionService = require('lib/services/EncryptionService');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const SearchEngineUtils = require('lib/services/SearchEngineUtils');
const RevisionService = require('lib/services/RevisionService');
const ResourceService = require('lib/services/RevisionService');
const DecryptionWorker = require('lib/services/DecryptionWorker');
const BaseService = require('lib/services/BaseService');
const SearchEngine = require('lib/services/SearchEngine');
@ -63,12 +64,24 @@ class BaseApplication {
await SearchEngine.instance().destroy();
await DecryptionWorker.instance().destroy();
await FoldersScreenUtils.cancelTimers();
await BaseItem.revisionService_.cancelTimers();
await ResourceService.instance().cancelTimers();
await reg.cancelTimers();
this.eventEmitter_.removeAllListeners();
BaseModel.db_ = null;
KvStore.instance_ = null;
BaseModel.setDb(null);
reg.setDb(null);
BaseItem.revisionService_ = null;
RevisionService.instance_ = null;
ResourceService.instance_ = null;
ResourceService.isRunningInBackground = false;
ResourceFetcher.instance_ = null;
EncryptionService.instance_ = null;
DecryptionWorker.instance_ = null;
this.logger_.info('Base application terminated...');
this.logger_ = null;
this.dbLogger_ = null;
this.eventEmitter_ = null;
@ -320,16 +333,7 @@ class BaseApplication {
}
async decryptionWorker_resourceMetadataButNotBlobDecrypted() {
this.scheduleAutoAddResources();
}
scheduleAutoAddResources() {
if (this.scheduleAutoAddResourcesIID_) return;
this.scheduleAutoAddResourcesIID_ = setTimeout(() => {
this.scheduleAutoAddResourcesIID_ = null;
ResourceFetcher.instance().autoAddResources();
}, 1000);
ResourceFetcher.instance().scheduleAutoAddResources();
}
reducerActionToString(action) {
@ -645,7 +649,7 @@ class BaseApplication {
// if (Setting.value('env') === 'dev') await this.database_.clearForTesting();
reg.setDb(this.database_);
BaseModel.db_ = this.database_;
BaseModel.setDb(this.database_);
await Setting.load();

4
ReactNativeClient/lib/BaseModel.js
View File

@ -12,6 +12,10 @@ class BaseModel {
throw new Error('Must be overriden');
}
static setDb(db) {
this.db_ = db;
}
static addModelMd(model) {
if (!model) return model;

279
ReactNativeClient/lib/registry.js
View File

@ -56,140 +56,152 @@ reg.syncTarget = (syncTargetId = null) => {
// sure it gets synced. So we wait for the current sync operation to
// finish (if one is running), then we trigger a sync just after.
reg.waitForSyncFinishedThenSync = async () => {
const synchronizer = await reg.syncTarget().synchronizer();
await synchronizer.waitForSyncToFinish();
await reg.scheduleSync(0);
};
reg.scheduleSync_ = async (delay = null, syncOptions = null) => {
if (delay === null) delay = 1000 * 10;
if (syncOptions === null) syncOptions = {};
let promiseResolve = null;
const promise = new Promise((resolve) => {
promiseResolve = resolve;
});
if (reg.scheduleSyncId_) {
clearTimeout(reg.scheduleSyncId_);
reg.scheduleSyncId_ = null;
reg.waitForReSyncCalls_.push(true);
try {
const synchronizer = await reg.syncTarget().synchronizer();
await synchronizer.waitForSyncToFinish();
await reg.scheduleSync(0);
} finally {
reg.waitForReSyncCalls_.pop();
}
reg.logger().info('Scheduling sync operation...');
if (Setting.value('env') === 'dev' && delay !== 0) {
reg.logger().info('Schedule sync DISABLED!!!');
return;
}
const timeoutCallback = async () => {
reg.scheduleSyncId_ = null;
reg.logger().info('Preparing scheduled sync');
const syncTargetId = Setting.value('sync.target');
if (!(await reg.syncTarget(syncTargetId).isAuthenticated())) {
reg.logger().info('Synchroniser is missing credentials - manual sync required to authenticate.');
promiseResolve();
return;
}
try {
const sync = await reg.syncTarget(syncTargetId).synchronizer();
const contextKey = `sync.${syncTargetId}.context`;
let context = Setting.value(contextKey);
try {
context = context ? JSON.parse(context) : {};
} catch (error) {
// Clearing the context is inefficient since it means all items are going to be re-downloaded
// however it won't result in duplicate items since the synchroniser is going to compare each
// item to the current state.
reg.logger().warn(`Could not parse JSON sync context ${contextKey}:`, context);
reg.logger().info('Clearing context and starting from scratch');
context = null;
}
try {
reg.logger().info('Starting scheduled sync');
const options = Object.assign({}, syncOptions, { context: context });
if (!options.saveContextHandler) {
options.saveContextHandler = newContext => {
Setting.setValue(contextKey, JSON.stringify(newContext));
};
}
const newContext = await sync.start(options);
Setting.setValue(contextKey, JSON.stringify(newContext));
} catch (error) {
if (error.code == 'alreadyStarted') {
reg.logger().info(error.message);
} else {
promiseResolve();
throw error;
}
}
} catch (error) {
reg.logger().info('Could not run background sync:');
reg.logger().info(error);
// Special case to display OneDrive Business error. This is the full error that's received when trying to use a OneDrive Business account:
//
// {"error":"invalid_client","error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for
// the application: 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation
// ID: acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}: TOKEN: null Error: {"error":"invalid_client",
// "error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for the application:
// 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation ID
// acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}
if (error && error.message && error.message.indexOf('"invalid_client"') >= 0) {
reg.showErrorMessageBox(_('Could not synchronize with OneDrive.\n\nThis error often happens when using OneDrive for Business, which unfortunately cannot be supported.\n\nPlease consider using a regular OneDrive account.'));
}
}
reg.setupRecurrentSync();
promiseResolve();
};
if (delay === 0) {
timeoutCallback();
} else {
reg.scheduleSyncId_ = setTimeout(timeoutCallback, delay);
}
return promise;
};
reg.scheduleSync = async (delay = null, syncOptions = null) => {
reg.syncCalls_.push(true);
reg.schedSyncCalls_.push(true);
try {
await reg.scheduleSync_(delay, syncOptions);
if (delay === null) delay = 1000 * 10;
if (syncOptions === null) syncOptions = {};
let promiseResolve = null;
const promise = new Promise((resolve) => {
promiseResolve = resolve;
});
if (reg.scheduleSyncId_) {
clearTimeout(reg.scheduleSyncId_);
reg.scheduleSyncId_ = null;
}
reg.logger().info('Scheduling sync operation...', delay);
if (Setting.value('env') === 'dev' && delay !== 0) {
reg.logger().info('Schedule sync DISABLED!!!');
return;
}
const timeoutCallback = async () => {
reg.timerCallbackCalls_.push(true);
try {
reg.scheduleSyncId_ = null;
reg.logger().info('Preparing scheduled sync');
const syncTargetId = Setting.value('sync.target');
if (!(await reg.syncTarget(syncTargetId).isAuthenticated())) {
reg.logger().info('Synchroniser is missing credentials - manual sync required to authenticate.');
promiseResolve();
return;
}
try {
const sync = await reg.syncTarget(syncTargetId).synchronizer();
const contextKey = `sync.${syncTargetId}.context`;
let context = Setting.value(contextKey);
try {
context = context ? JSON.parse(context) : {};
} catch (error) {
// Clearing the context is inefficient since it means all items are going to be re-downloaded
// however it won't result in duplicate items since the synchroniser is going to compare each
// item to the current state.
reg.logger().warn(`Could not parse JSON sync context ${contextKey}:`, context);
reg.logger().info('Clearing context and starting from scratch');
context = null;
}
try {
reg.logger().info('Starting scheduled sync');
const options = Object.assign({}, syncOptions, { context: context });
if (!options.saveContextHandler) {
options.saveContextHandler = newContext => {
Setting.setValue(contextKey, JSON.stringify(newContext));
};
}
const newContext = await sync.start(options);
Setting.setValue(contextKey, JSON.stringify(newContext));
} catch (error) {
if (error.code == 'alreadyStarted') {
reg.logger().info(error.message);
} else {
promiseResolve();
throw error;
}
}
} catch (error) {
reg.logger().info('Could not run background sync:');
reg.logger().info(error);
// Special case to display OneDrive Business error. This is the full error that's received when trying to use a OneDrive Business account:
//
// {"error":"invalid_client","error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for
// the application: 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation
// ID: acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}: TOKEN: null Error: {"error":"invalid_client",
// "error_description":"AADSTS50011: The reply address 'http://localhost:1917' does not match the reply addresses configured for the application:
// 'cbabb902-d276-4ea4-aa88-062a5889d6dc'. More details: not specified\r\nTrace ID: 6e63dac6-8b37-47e2-bd1b-4768f8713400\r\nCorrelation ID
// acfd6503-8d97-4349-ae2e-e7a19dd7b6bc\r\nTimestamp: 2017-12-01 13:35:55Z","error_codes":[50011],"timestamp":"2017-12-01 13:35:55Z","trace_id":
// "6e63dac6-8b37-47e2-bd1b-4768f8713400","correlation_id":"acfd6503-8d97-4349-ae2e-e7a19dd7b6bc"}
if (error && error.message && error.message.indexOf('"invalid_client"') >= 0) {
reg.showErrorMessageBox(_('Could not synchronize with OneDrive.\n\nThis error often happens when using OneDrive for Business, which unfortunately cannot be supported.\n\nPlease consider using a regular OneDrive account.'));
}
}
reg.setupRecurrentSync();
promiseResolve();
} finally {
reg.timerCallbackCalls_.pop();
}
};
if (delay === 0) {
timeoutCallback();
} else {
reg.scheduleSyncId_ = setTimeout(timeoutCallback, delay);
}
return promise;
} finally {
reg.syncCalls_.pop();
reg.schedSyncCalls_.pop();
}
};
reg.setupRecurrentSync = () => {
if (reg.recurrentSyncId_) {
shim.clearInterval(reg.recurrentSyncId_);
reg.recurrentSyncId_ = null;
}
reg.setupRecurrentCalls_.push(true);
if (!Setting.value('sync.interval')) {
reg.logger().debug('Recurrent sync is disabled');
} else {
reg.logger().debug(`Setting up recurrent sync with interval ${Setting.value('sync.interval')}`);
if (Setting.value('env') === 'dev') {
reg.logger().info('Recurrent sync operation DISABLED!!!');
return;
try {
if (reg.recurrentSyncId_) {
shim.clearInterval(reg.recurrentSyncId_);
reg.recurrentSyncId_ = null;
}
reg.recurrentSyncId_ = shim.setInterval(() => {
reg.logger().info('Running background sync on timer...');
reg.scheduleSync(0);
}, 1000 * Setting.value('sync.interval'));
if (!Setting.value('sync.interval')) {
reg.logger().debug('Recurrent sync is disabled');
} else {
reg.logger().debug(`Setting up recurrent sync with interval ${Setting.value('sync.interval')}`);
if (Setting.value('env') === 'dev') {
reg.logger().info('Recurrent sync operation DISABLED!!!');
return;
}
reg.recurrentSyncId_ = shim.setInterval(() => {
reg.logger().info('Running background sync on timer...');
reg.scheduleSync(0);
}, 1000 * Setting.value('sync.interval'));
}
} finally {
reg.setupRecurrentCalls_.pop();
}
};
@ -201,15 +213,26 @@ reg.db = () => {
return reg.db_;
};
reg.cancelTimers = async () => {
reg.cancelTimers_ = () => {
if (this.recurrentSyncId_) {
clearTimeout(this.recurrentSyncId_);
shim.clearInterval(reg.recurrentSyncId_);
this.recurrentSyncId_ = null;
}
if (reg.scheduleSyncId_) {
clearTimeout(reg.scheduleSyncId_);
reg.scheduleSyncId_ = null;
}
};
reg.cancelTimers = async () => {
reg.logger().info('Cancelling sync timers');
reg.cancelTimers_();
return new Promise((resolve) => {
const iid = setInterval(() => {
if (!reg.syncCalls_.length) {
clearInterval(iid);
setInterval(() => {
// ensure processing complete
if (!reg.setupRecurrentCalls_.length && !reg.schedSyncCalls_.length && !reg.timerCallbackCalls_.length && !reg.waitForReSyncCalls_.length) {
reg.cancelTimers_();
resolve();
}
}, 100);
@ -217,5 +240,9 @@ reg.cancelTimers = async () => {
};
reg.syncCalls_ = [];
reg.schedSyncCalls_ = [];
reg.waitForReSyncCalls_ = [];
reg.setupRecurrentCalls_ = [];
reg.timerCallbackCalls_ = [];
module.exports = { reg };

60
ReactNativeClient/lib/services/ResourceFetcher.js
View File

@ -21,6 +21,7 @@ class ResourceFetcher extends BaseService {
this.maxDownloads_ = 3;
this.addingResources_ = false;
this.eventEmitter_ = new EventEmitter();
this.autoAddResourcesCalls_ = [];
}
static instance() {
@ -197,7 +198,12 @@ class ResourceFetcher extends BaseService {
async waitForAllFinished() {
return new Promise((resolve) => {
const iid = setInterval(() => {
if (!this.updateReportIID_ && !this.scheduleQueueProcessIID_ && !this.addingResources_ && !this.queue_.length && !Object.getOwnPropertyNames(this.fetchingItems_).length) {
if (!this.updateReportIID_ &&
!this.scheduleQueueProcessIID_ &&
!this.queue_.length &&
!this.autoAddResourcesCalls_.length &&
!Object.getOwnPropertyNames(this.fetchingItems_).length) {
clearInterval(iid);
resolve();
}
@ -206,25 +212,31 @@ class ResourceFetcher extends BaseService {
}
async autoAddResources(limit = null) {
if (limit === null) limit = 10;
this.autoAddResourcesCalls_.push(true);
try {
if (limit === null) limit = 10;
if (this.addingResources_) return;
this.addingResources_ = true;
if (this.addingResources_) return;
this.addingResources_ = true;
this.logger().info(`ResourceFetcher: Auto-add resources: Mode: ${Setting.value('sync.resourceDownloadMode')}`);
this.logger().info(`ResourceFetcher: Auto-add resources: Mode: ${Setting.value('sync.resourceDownloadMode')}`);
let count = 0;
const resources = await Resource.needToBeFetched(Setting.value('sync.resourceDownloadMode'), limit);
for (let i = 0; i < resources.length; i++) {
const added = this.queueDownload_(resources[i].id);
if (added) count++;
let count = 0;
const resources = await Resource.needToBeFetched(Setting.value('sync.resourceDownloadMode'), limit);
for (let i = 0; i < resources.length; i++) {
const added = this.queueDownload_(resources[i].id);
if (added) count++;
}
this.logger().info(`ResourceFetcher: Auto-added resources: ${count}`);
const errorCount = await Resource.downloadStatusCounts(Resource.FETCH_STATUS_ERROR);
if (errorCount) this.dispatch({ type: 'SYNC_HAS_DISABLED_SYNC_ITEMS' });
} finally {
this.addingResources_ = false;
this.autoAddResourcesCalls_.pop();
}
this.logger().info(`ResourceFetcher: Auto-added resources: ${count}`);
this.addingResources_ = false;
const errorCount = await Resource.downloadStatusCounts(Resource.FETCH_STATUS_ERROR);
if (errorCount) this.dispatch({ type: 'SYNC_HAS_DISABLED_SYNC_ITEMS' });
}
async start() {
@ -244,6 +256,15 @@ class ResourceFetcher extends BaseService {
}, 100);
}
scheduleAutoAddResources() {
if (this.scheduleAutoAddResourcesIID_) return;
this.scheduleAutoAddResourcesIID_ = setTimeout(() => {
this.scheduleAutoAddResourcesIID_ = null;
ResourceFetcher.instance().autoAddResources();
}, 1000);
}
async fetchAll() {
await Resource.resetStartedFetchStatus();
this.autoAddResources(null);
@ -255,10 +276,13 @@ class ResourceFetcher extends BaseService {
clearTimeout(this.scheduleQueueProcessIID_);
this.scheduleQueueProcessIID_ = null;
}
if (this.scheduleAutoAddResourcesIID_) {
clearTimeout(this.scheduleAutoAddResourcesIID_);
this.scheduleAutoAddResourcesIID_ = null;
}
await this.waitForAllFinished();
this.eventEmitter_ = null;
ResourceFetcher.instance_ = null;
return await this.waitForAllFinished();
}
}

49
ReactNativeClient/lib/services/ResourceService.js
View File

@ -11,6 +11,20 @@ const ItemChangeUtils = require('lib/services/ItemChangeUtils');
const { sprintf } = require('sprintf-js');
class ResourceService extends BaseService {
constructor() {
super();
this.maintenanceCalls_ = [];
this.maintenanceTimer1_ = null;
this.maintenanceTimer2_ = null;
}
static instance() {
if (this.instance_) return this.instance_;
this.instance_ = new ResourceService();
return this.instance_;
}
async indexNoteResources() {
this.logger().info('ResourceService::indexNoteResources: Start');
@ -131,24 +145,49 @@ class ResourceService extends BaseService {
}
async maintenance() {
await this.indexNoteResources();
await this.deleteOrphanResources();
this.maintenanceCalls_.push(true);
try {
await this.indexNoteResources();
await this.deleteOrphanResources();
} finally {
this.maintenanceCalls_.pop();
}
}
static runInBackground() {
if (this.isRunningInBackground_) return;
this.isRunningInBackground_ = true;
const service = new ResourceService();
const service = this.instance();
setTimeout(() => {
service.maintenanceTimer1_ = setTimeout(() => {
service.maintenance();
}, 1000 * 30);
shim.setInterval(() => {
service.maintenanceTimer2_ = shim.setInterval(() => {
service.maintenance();
}, 1000 * 60 * 60 * 4);
}
async cancelTimers() {
if (this.maintenanceTimer1_) {
clearTimeout(this.maintenanceTimer1);
this.maintenanceTimer1_ = null;
}
if (this.maintenanceTimer2_) {
shim.clearInterval(this.maintenanceTimer2);
this.maintenanceTimer2_ = null;
}
return new Promise((resolve) => {
const iid = setInterval(() => {
if (!this.maintenanceCalls_.length) {
clearInterval(iid);
resolve();
}
}, 100);
});
}
}
module.exports = ResourceService;

61
ReactNativeClient/lib/services/RevisionService.js
View File

@ -19,6 +19,10 @@ class RevisionService extends BaseService {
// the original note is saved. The goal is to have at least one revision in case the note
// is deleted or modified as a result of a bug or user mistake.
this.isOldNotesCache_ = {};
this.maintenanceCalls_ = [];
this.maintenanceTimer1_ = null;
this.maintenanceTimer2_ = null;
}
static instance() {
@ -235,22 +239,27 @@ class RevisionService extends BaseService {
}
async maintenance() {
const startTime = Date.now();
this.logger().info('RevisionService::maintenance: Starting...');
this.maintenanceCalls_.push(true);
try {
const startTime = Date.now();
this.logger().info('RevisionService::maintenance: Starting...');
if (!Setting.value('revisionService.enabled')) {
this.logger().info('RevisionService::maintenance: Service is disabled');
// We do as if we had processed all the latest changes so that they can be cleaned up
// later on by ItemChangeUtils.deleteProcessedChanges().
Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId());
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
} else {
this.logger().info('RevisionService::maintenance: Service is enabled');
await this.collectRevisions();
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
if (!Setting.value('revisionService.enabled')) {
this.logger().info('RevisionService::maintenance: Service is disabled');
// We do as if we had processed all the latest changes so that they can be cleaned up
// later on by ItemChangeUtils.deleteProcessedChanges().
Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId());
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
} else {
this.logger().info('RevisionService::maintenance: Service is enabled');
await this.collectRevisions();
await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
this.logger().info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`);
}
} finally {
this.maintenanceCalls_.pop();
}
this.logger().info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`);
}
runInBackground(collectRevisionInterval = null) {
@ -261,14 +270,34 @@ class RevisionService extends BaseService {
this.logger().info(`RevisionService::runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`);
setTimeout(() => {
this.maintenanceTimer1_ = setTimeout(() => {
this.maintenance();
}, 1000 * 4);
shim.setInterval(() => {
this.maintenanceTImer2_ = shim.setInterval(() => {
this.maintenance();
}, collectRevisionInterval);
}
async cancelTimers() {
if (this.maintenanceTimer1_) {
clearTimeout(this.maintenanceTimer1);
this.maintenanceTimer1_ = null;
}
if (this.maintenanceTimer2_) {
shim.clearInterval(this.maintenanceTimer2);
this.maintenanceTimer2_ = null;
}
return new Promise((resolve) => {
const iid = setInterval(() => {
if (!this.maintenanceCalls_.length) {
clearInterval(iid);
resolve();
}
}, 100);
});
}
}
module.exports = RevisionService;

1
ReactNativeClient/lib/services/SearchEngine.js
View File

@ -424,6 +424,7 @@ class SearchEngine {
const iid = setInterval(() => {
if (!this.syncCalls_.length) {
clearInterval(iid);
this.instance_ = null;
resolve();
}
}, 100);