self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-02-16 19:47:40 +02:00
Code Issues Releases Activity

All: Moved resource app-specific state to different table

Browse Source
This commit is contained in:
Laurent Cozic 2018-11-13 00:45:08 +00:00
parent 3a9948e528
commit 06091933e1
13 changed files with 871 additions and 733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
CliClient/tests/models_Resource.js
View File

@ -25,7 +25,35 @@ describe('models_Resource', function() {
let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
let resource1 = (await Resource.all())[0];
expect(resource1.fetch_status).toBe(Resource.FETCH_STATUS_DONE);
let ls = await Resource.localState(resource1);
expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE);
}));
it('should have a default local state', asyncTest(async () => {
let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
let resource1 = (await Resource.all())[0];
let ls = await Resource.localState(resource1);
expect(!ls.id).toBe(true);
expect(ls.resource_id).toBe(resource1.id);
expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE);
}));
it('should save and delete local state', asyncTest(async () => {
let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
let resource1 = (await Resource.all())[0];
await Resource.setLocalState(resource1, { fetch_status: Resource.FETCH_STATUS_IDLE });
let ls = await Resource.localState(resource1);
expect(!!ls.id).toBe(true);
expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_IDLE);
await Resource.delete(resource1.id);
ls = await Resource.localState(resource1);
expect(!ls.id).toBe(true);
}));
});

1383
CliClient/tests/synchronizer.js
View File

File diff suppressed because it is too large Load Diff

3
ElectronClient/app/gui/NoteText.jsx
View File

@ -611,7 +611,8 @@ class NoteTextComponent extends React.Component {
if (!item) throw new Error('No item with ID ' + itemId);
if (item.type_ === BaseModel.TYPE_RESOURCE) {
if (item.fetch_status !== Resource.FETCH_STATUS_DONE || !!item.encryption_blob_encrypted) {
const localState = await Resource.localState(item);
if (localState.fetch_status !== Resource.FETCH_STATUS_DONE || !!item.encryption_blob_encrypted) {
bridge().showErrorMessageBox(_('This attachment is not downloaded or not decrypted yet.'));
return;
}

5
ReactNativeClient/lib/BaseModel.js
View File

@ -482,7 +482,9 @@ class BaseModel {
static batchDelete(ids, options = null) {
if (!ids.length) return;
options = this.modOptions(options);
return this.db().exec('DELETE FROM ' + this.tableName() + ' WHERE id IN ("' + ids.join('","') + '")');
const idFieldName = options.idFieldName ? options.idFieldName : 'id';
const sql = 'DELETE FROM ' + this.tableName() + ' WHERE ' + idFieldName + ' IN ("' + ids.join('","') + '")';
return this.db().exec(sql);
}
static db() {
@ -508,6 +510,7 @@ BaseModel.typeEnum_ = [
['TYPE_MASTER_KEY', 9],
['TYPE_ITEM_CHANGE', 10],
['TYPE_NOTE_RESOURCE', 11],
['TYPE_RESOURCE_LOCAL_STATE', 12],
];
for (let i = 0; i < BaseModel.typeEnum_.length; i++) {

4
ReactNativeClient/lib/MdToHtml.js
View File

@ -94,7 +94,9 @@ class MdToHtml {
return;
}
if (resource.fetch_status !== Resource.FETCH_STATUS_DONE) {
const localState = await Resource.localState(resource);
if (localState.fetch_status !== Resource.FETCH_STATUS_DONE) {
delete this.loadedResources_[id];
console.info('Resource not yet fetched: ' + id);
return;

2
ReactNativeClient/lib/components/screens/note.js
View File

@ -139,7 +139,7 @@ class NoteScreenComponent extends BaseScreenComponent {
});
}, 5);
} else if (item.type_ === BaseModel.TYPE_RESOURCE) {
if (!Resource.isReady(item)) throw new Error(_('This attachment is not downloaded or not decrypted yet.'));
if (!(await Resource.isReady(item))) throw new Error(_('This attachment is not downloaded or not decrypted yet.'));
const resourcePath = Resource.fullPath(item);
await FileViewer.open(resourcePath);
} else {

45
ReactNativeClient/lib/joplin-database.js
View File

@ -161,6 +161,16 @@ class JoplinDatabase extends Database {
return output;
}
createDefaultRow(tableName) {
const row = {};
const fields = this.tableFields('resource_local_states');
for (let i = 0; i < fields.length; i++) {
const f = fields[i];
row[f.name] = Database.formatValue(f.type, f.default);
}
return row;
}
fieldDescription(tableName, fieldName) {
const sp = sprintf;
@ -250,7 +260,7 @@ class JoplinDatabase extends Database {
// default value and thus might cause problems. In that case, the default value
// must be set in the synchronizer too.
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
@ -402,6 +412,39 @@ class JoplinDatabase extends Database {
queries.push({ sql: 'UPDATE resources SET fetch_status = ?', params: [Resource.FETCH_STATUS_DONE] });
}
if (targetVersion == 14) {
const resourceLocalStates = `
CREATE TABLE resource_local_states (
id INTEGER PRIMARY KEY,
resource_id TEXT NOT NULL,
fetch_status INT NOT NULL DEFAULT "2",
fetch_error TEXT NOT NULL DEFAULT ""
);
`;
queries.push(this.sqlStringToLines(resourceLocalStates)[0]);
queries.push('INSERT INTO resource_local_states SELECT null, id, fetch_status, fetch_error FROM resources');
queries.push('CREATE INDEX resource_local_states_resource_id ON resource_local_states (resource_id)');
queries.push('CREATE INDEX resource_local_states_resource_fetch_status ON resource_local_states (fetch_status)');
queries = queries.concat(this.alterColumnQueries('resources', {
id: 'TEXT PRIMARY KEY',
title: 'TEXT NOT NULL DEFAULT ""',
mime: 'TEXT NOT NULL',
filename: 'TEXT NOT NULL DEFAULT ""',
created_time: 'INT NOT NULL',
updated_time: 'INT NOT NULL',
user_created_time: 'INT NOT NULL DEFAULT 0',
user_updated_time: 'INT NOT NULL DEFAULT 0',
file_extension: 'TEXT NOT NULL DEFAULT ""',
encryption_cipher_text: 'TEXT NOT NULL DEFAULT ""',
encryption_applied: 'INT NOT NULL DEFAULT 0',
encryption_blob_encrypted: 'INT NOT NULL DEFAULT 0',
}));
}
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
await this.transactionExecBatch(queries);

14
ReactNativeClient/lib/models/BaseItem.js
View File

@ -291,13 +291,13 @@ class BaseItem extends BaseModel {
let shownKeys = ItemClass.fieldNames();
shownKeys.push('type_');
if (ItemClass.syncExcludedKeys) {
const keys = ItemClass.syncExcludedKeys();
for (let i = 0; i < keys.length; i++) {
const idx = shownKeys.indexOf(keys[i]);
shownKeys.splice(idx, 1);
}
}
// if (ItemClass.syncExcludedKeys) {
// const keys = ItemClass.syncExcludedKeys();
// for (let i = 0; i < keys.length; i++) {
// const idx = shownKeys.indexOf(keys[i]);
// shownKeys.splice(idx, 1);
// }
// }
const serialized = await ItemClass.serialize(item, shownKeys);

38
ReactNativeClient/lib/models/Resource.js
View File

@ -1,6 +1,7 @@
const BaseModel = require('lib/BaseModel.js');
const BaseItem = require('lib/models/BaseItem.js');
const NoteResource = require('lib/models/NoteResource.js');
const ResourceLocalState = require('lib/models/ResourceLocalState.js');
const Setting = require('lib/models/Setting.js');
const ArrayUtils = require('lib/ArrayUtils.js');
const pathUtils = require('lib/path-utils.js');
@ -30,27 +31,12 @@ class Resource extends BaseItem {
return imageMimeTypes.indexOf(type.toLowerCase()) >= 0;
}
static resetStartedFetchStatus() {
return this.db().exec('UPDATE resources SET fetch_status = ? WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE, Resource.FETCH_STATUS_STARTED]);
}
static needToBeFetched(limit = null) {
let sql = 'SELECT * FROM resources WHERE fetch_status = ? ORDER BY updated_time DESC';
let sql = 'SELECT * FROM resources WHERE id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = ?) ORDER BY updated_time DESC';
if (limit !== null) sql += ' LIMIT ' + limit;
return this.modelSelectAll(sql, [Resource.FETCH_STATUS_IDLE]);
}
static async saveFetchStatus(resourceId, status, error = null) {
const o = {
id: resourceId,
fetch_status: status,
}
if (error !== null) o.fetch_error = error;
return Resource.save(o, { autoTimestamp: false });
}
static fsDriver() {
if (!Resource.fsDriver_) Resource.fsDriver_ = new FsDriverDummy();
return Resource.fsDriver_;
@ -63,10 +49,6 @@ class Resource extends BaseItem {
return resource.id + extension;
}
static syncExcludedKeys() {
return ['fetch_status', 'fetch_error'];
}
static friendlyFilename(resource) {
let output = safeFilename(resource.title); // Make sure not to allow spaces or any special characters as it's not supported in HTTP headers
if (!output) output = resource.id;
@ -80,8 +62,9 @@ class Resource extends BaseItem {
return Setting.value('resourceDir') + '/' + this.filename(resource, encryptedBlob);
}
static isReady(resource) {
return resource && resource.fetch_status === Resource.FETCH_STATUS_DONE && !resource.encryption_blob_encrypted;
static async isReady(resource) {
const ls = await this.localState(resource);
return resource && ls.fetch_status === Resource.FETCH_STATUS_DONE && !resource.encryption_blob_encrypted;
}
// For resources, we need to decrypt the item (metadata) and the resource binary blob.
@ -201,6 +184,15 @@ class Resource extends BaseItem {
return url.substr(2);
}
static localState(resourceOrId) {
return ResourceLocalState.byResourceId(typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId);
}
static async setLocalState(resourceOrId, state) {
const id = typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId;
await ResourceLocalState.save(Object.assign({}, state, { resource_id: id }));
}
static async batchDelete(ids, options = null) {
// For resources, there's not really batch deleting since there's the file data to delete
// too, so each is processed one by one with the item being deleted last (since the db
@ -215,6 +207,8 @@ class Resource extends BaseItem {
await super.batchDelete([id], options);
await NoteResource.deleteByResource(id); // Clean up note/resource relationships
}
await ResourceLocalState.batchDelete(ids);
}
}

51
ReactNativeClient/lib/models/ResourceLocalState.js Normal file
View File

@ -0,0 +1,51 @@
const BaseModel = require('lib/BaseModel.js');
const Resource = require('lib/models/Resource.js');
const { Database } = require('lib/database.js');
class ResourceLocalState extends BaseModel {
static tableName() {
return 'resource_local_states';
}
static modelType() {
return BaseModel.TYPE_RESOURCE_LOCAL_STATE;
}
static async byResourceId(resourceId) {
if (!resourceId) throw new Error('Resource ID not provided'); // Sanity check
const result = await this.modelSelectOne('SELECT * FROM resource_local_states WHERE resource_id = ?', [resourceId]);
if (!result) {
const defaultRow = this.db().createDefaultRow(this.tableName());
delete defaultRow.id;
defaultRow.resource_id = resourceId;
return defaultRow;
}
return result;
}
static resetStartedFetchStatus() {
return this.db().exec('UPDATE resource_local_states SET fetch_status = ? WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE, Resource.FETCH_STATUS_STARTED]);
}
static async save(o) {
const queries = [
{ sql: 'DELETE FROM resource_local_states WHERE resource_id = ?', params: [o.resource_id] },
Database.insertQuery(this.tableName(), o),
];
return this.db().transactionExecBatch(queries);
}
static batchDelete(ids, options = null) {
options = options ? Object.assign({}, options) : {};
options.idFieldName = 'resource_id';
return super.batchDelete(ids, options);
}
}
module.exports = ResourceLocalState;

8
ReactNativeClient/lib/services/DecryptionWorker.js
View File

@ -84,7 +84,13 @@ class DecryptionWorker {
const ItemClass = BaseItem.itemClass(item);
if (('fetch_status' in item) && item.fetch_status !== Resource.FETCH_STATUS_DONE) continue;
if (item.type_ === Resource.modelType()) {
const ls = await Resource.localState(item);
if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) {
excludedIds.push(item.id);
continue;
}
}
this.dispatchReport({
itemIndex: i,

14
ReactNativeClient/lib/services/ResourceFetcher.js
View File

@ -1,4 +1,5 @@
const Resource = require('lib/models/Resource');
const ResourceLocalState = require('lib/models/ResourceLocalState');
const BaseService = require('lib/services/BaseService');
const BaseSyncTarget = require('lib/BaseSyncTarget');
const { Logger } = require('lib/logger.js');
@ -87,10 +88,11 @@ class ResourceFetcher extends BaseService {
}
const resource = await Resource.load(resourceId);
const localState = await Resource.localState(resource);
// Shouldn't happen, but just to be safe don't re-download the
// resource if it's already been downloaded.
if (resource.fetch_status === Resource.FETCH_STATUS_DONE) {
if (localState.fetch_status === Resource.FETCH_STATUS_DONE) {
completeDownload(false);
return;
}
@ -100,19 +102,19 @@ class ResourceFetcher extends BaseService {
const localResourceContentPath = Resource.fullPath(resource);
const remoteResourceContentPath = this.resourceDirName_ + "/" + resource.id;
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_STARTED);
await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_STARTED });
const fileApi = await this.fileApi();
this.logger().debug('ResourceFetcher: Downloading resource: ' + resource.id);
fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => {
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_DONE);
await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_DONE });
this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id);
completeDownload();
}).catch(async (error) => {
this.logger().error('ResourceFetcher: Could not download resource: ' + resource.id, error);
await Resource.saveFetchStatus(resource.id, Resource.FETCH_STATUS_ERROR, error.message);
await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_ERROR, fetch_error: error.message });
completeDownload();
});
}
@ -156,7 +158,7 @@ class ResourceFetcher extends BaseService {
}
async start() {
await Resource.resetStartedFetchStatus();
await ResourceLocalState.resetStartedFetchStatus();
this.autoAddResources(10);
}
@ -173,7 +175,7 @@ class ResourceFetcher extends BaseService {
}
async fetchAll() {
await Resource.resetStartedFetchStatus();
await ResourceLocalState.resetStartedFetchStatus();
this.autoAddResources(null);
}

7
ReactNativeClient/lib/synchronizer.js
View File

@ -2,6 +2,7 @@ const BaseItem = require('lib/models/BaseItem.js');
const Folder = require('lib/models/Folder.js');
const Note = require('lib/models/Note.js');
const Resource = require('lib/models/Resource.js');
const ResourceLocalState = require('lib/models/ResourceLocalState.js');
const MasterKey = require('lib/models/MasterKey.js');
const BaseModel = require('lib/BaseModel.js');
const DecryptionWorker = require('lib/services/DecryptionWorker');
@ -574,7 +575,11 @@ class Synchronizer {
// }
// }
if (creatingNewResource) content.fetch_status = Resource.FETCH_STATUS_IDLE;
// if (creatingNewResource) content.fetch_status = Resource.FETCH_STATUS_IDLE;
if (creatingNewResource) {
await ResourceLocalState.save({ resource_id: content.id, fetch_status: Resource.FETCH_STATUS_IDLE });
}
await ItemClass.save(content, options);