self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-11-29 22:48:10 +02:00
Code Issues Releases Activity

All: Finished service to clean up resources

Browse Source
This commit is contained in:
Laurent Cozic
2018-03-15 18:08:46 +00:00
parent 945018b698
commit c1bb51c12b
10 changed files with 205 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ReactNativeClient/lib/BaseApplication.js
View File

@@ -31,6 +31,7 @@ const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
const EncryptionService = require('lib/services/EncryptionService');
const DecryptionWorker = require('lib/services/DecryptionWorker');
const BaseService = require('lib/services/BaseService');
SyncTargetRegistry.addClass(SyncTargetFilesystem);
SyncTargetRegistry.addClass(SyncTargetOneDrive);
@@ -426,6 +427,7 @@ class BaseApplication {
setLocale(Setting.value('locale'));
}
BaseService.logger_ = this.logger_;
EncryptionService.instance().setLogger(this.logger_);
BaseItem.encryptionService_ = EncryptionService.instance();
DecryptionWorker.instance().setLogger(this.logger_);

6
ReactNativeClient/lib/joplin-database.js
View File

@@ -301,7 +301,7 @@ class JoplinDatabase extends Database {
if (targetVersion == 10) {
const itemChangesTable = `
CREATE TABLE item_changes (
id INTEGER PRIMARY KEY,
id INTEGER PRIMARY KEY AUTOINCREMENT,
item_type INT NOT NULL,
item_id TEXT NOT NULL,
type INT NOT NULL,
@@ -313,7 +313,9 @@ class JoplinDatabase extends Database {
CREATE TABLE note_resources (
id INTEGER PRIMARY KEY,
note_id TEXT NOT NULL,
resource_id TEXT NOT NULL
resource_id TEXT NOT NULL,
is_associated INT NOT NULL,
last_seen_time INT NOT NULL
);
`;

19
ReactNativeClient/lib/models/ItemChange.js
View File

@@ -12,6 +12,10 @@ class ItemChange extends BaseModel {
}
static async add(itemType, itemId, type) {
ItemChange.saveCalls_.push(true);
// Using a mutex so that records can be added to the database in the
// background, without making the UI wait.
const release = await ItemChange.addChangeMutex_.acquire();
try {
@@ -21,12 +25,27 @@ class ItemChange extends BaseModel {
]);
} finally {
release();
ItemChange.saveCalls_.pop();
}
}
// Because item changes are recorded in the background, this function
// can be used for synchronous code, in particular when unit testing.
static async waitForAllSaved() {
return new Promise((resolve, reject) => {
const iid = setInterval(() => {
if (!ItemChange.saveCalls_.length) {
clearInterval(iid);
resolve();
}
}, 100);
});
}
}
ItemChange.addChangeMutex_ = new Mutex();
ItemChange.saveCalls_ = [];
ItemChange.TYPE_CREATE = 1;
ItemChange.TYPE_UPDATE = 2;

36
ReactNativeClient/lib/models/NoteResource.js
View File

@@ -10,21 +10,39 @@ class NoteResource extends BaseModel {
return BaseModel.TYPE_NOTE_RESOURCE;
}
static async associate(noteId, resourceIds) {
let queries = [];
queries.push({ sql: 'DELETE FROM note_resources WHERE note_id = ?', params: [noteId] });
static async setAssociatedResources(noteId, resourceIds) {
const existingRows = await this.modelSelectAll('SELECT * FROM note_resources WHERE note_id = ?', [noteId]);
for (let i = 0; i < resourceIds.length; i++) {
queries.push({ sql: 'INSERT INTO note_resources (note_id, resource_id) VALUES (?, ?)', params: [noteId, resourceIds[i]] });
const notProcessedResourceIds = resourceIds.slice();
const queries = [];
for (let i = 0; i < existingRows.length; i++) {
const row = existingRows[i];
const resourceIndex = resourceIds.indexOf(row.resource_id);
if (resourceIndex >= 0) {
queries.push({ sql: 'UPDATE note_resources SET last_seen_time = ?, is_associated = 1 WHERE id = ?', params: [Date.now(), row.id] });
notProcessedResourceIds.splice(notProcessedResourceIds.indexOf(row.resource_id), 1);
} else {
queries.push({ sql: 'UPDATE note_resources SET is_associated = 0 WHERE id = ?', params: [row.id] });
}
}
await this.db().transactionExecBatch(queries);
for (let i = 0; i < notProcessedResourceIds.length; i++) {
queries.push({ sql: 'INSERT INTO note_resources (note_id, resource_id, is_associated, last_seen_time) VALUES (?, ?, ?, ?)', params: [noteId, notProcessedResourceIds[i], 1, Date.now()] });
}
await this.db().transactionExecBatch(queries);
}
static async remove(noteId) {
let queries = [];
queries.push({ sql: 'DELETE FROM note_resources WHERE note_id = ?', params: [noteId] });
await this.db().transactionExecBatch(queries);
await this.db().exec({ sql: 'UPDATE note_resources SET is_associated = 0 WHERE note_id = ?', params: [noteId] });
}
static async orphanResources(expiryDelay = null) {
if (expiryDelay === null) expiryDelay = 1000 * 60 * 60 * 24;
const cutOffTime = Date.now() - expiryDelay;
const output = await this.modelSelectAll('SELECT DISTINCT resource_id FROM note_resources WHERE is_associated = 0 AND last_seen_time < ?', [cutOffTime]);
return output.map(r => r.resource_id);
}
}

12
ReactNativeClient/lib/services/BaseService.js Normal file
View File

@@ -0,0 +1,12 @@
class BaseService {
logger() {
if (!BaseService.logger_) throw new Error('BaseService.logger_ not set!!');
return BaseService.logger_;
}
}
BaseService.logger_ = null;
module.exports = BaseService;

62
ReactNativeClient/lib/services/ResourceService.js
View File

@@ -1,52 +1,80 @@
const ItemChange = require('lib/models/ItemChange');
const NoteResource = require('lib/models/NoteResource');
const Note = require('lib/models/Note');
const Resource = require('lib/models/Resource');
const BaseModel = require('lib/BaseModel');
const BaseService = require('lib/services/BaseService');
class ResourceService {
class ResourceService extends BaseService {
async indexNoteResources() {
this.logger().info('ResourceService::indexNoteResources: Start');
let lastId = 0;
let lastCreatedTime = 0
const processedChangeIds = [];
await ItemChange.waitForAllSaved();
while (true) {
const changes = await ItemChange.modelSelectAll(`
SELECT id, item_id, type, created_time
SELECT id, item_id, type
FROM item_changes
WHERE item_type = ?
AND id > ?
AND created_time >= ?
ORDER BY id, created_time ASC
ORDER BY id ASC
LIMIT 10
`, [BaseModel.TYPE_NOTE, lastId, lastCreatedTime]);
`, [BaseModel.TYPE_NOTE, lastId]);
if (!changes.length) break;
const noteIds = changes.map(a => a.item_id);
const changesByNoteId = {};
for (let i = 0; i < changes.length; i++) {
changesByNoteId[changes[i].item_id] = changes[i];
}
const notes = await Note.modelSelectAll('SELECT id, title, body FROM notes WHERE id IN ("' + noteIds.join('","') + '")');
for (let i = 0; i < notes.length; i++) {
const note = notes[i];
const change = changesByNoteId[note.id];
const noteById = (noteId) => {
for (let i = 0; i < notes.length; i++) {
if (notes[i].id === noteId) return notes[i];
}
throw new Error('Invalid note ID: ' + noteId);
}
for (let i = 0; i < changes.length; i++) {
const change = changes[i];
if (change.type === ItemChange.TYPE_CREATE || change.type === ItemChange.TYPE_UPDATE) {
const note = noteById(change.item_id);
const resourceIds = Note.linkedResourceIds(note.body);
await NoteResource.associate(note.id, resourceIds);
await NoteResource.setAssociatedResources(note.id, resourceIds);
} else if (change.type === ItemChange.TYPE_DELETE) {
await NoteResource.remove(note.id);
await NoteResource.remove(change.item_id);
} else {
throw new Error('Invalid change type: ' + change.type);
}
lastId = change.id;
lastCreatedTime = change.created_time;
processedChangeIds.push(change.id);
}
}
if (lastId) {
await ItemChange.db().exec('DELETE FROM item_changes WHERE id <= ?', [lastId]);
}
this.logger().info('ResourceService::indexNoteResources: Completed');
}
async deleteOrphanResources(expiryDelay = null) {
const resourceIds = await NoteResource.orphanResources(expiryDelay);
this.logger().info('ResourceService::deleteOrphanResources:', resourceIds);
for (let i = 0; i < resourceIds.length; i++) {
await Resource.delete(resourceIds[i]);
}
}
async maintenance() {
await this.indexNoteResources();
await this.deleteOrphanResources();
}
}