joplin/packages/lib/services/ResourceService.ts

import NoteResource from '../models/NoteResource';
import BaseModel from '../BaseModel';
import BaseService from './BaseService';
import Setting from '../models/Setting';
import shim from '../shim';
const ItemChange = require('../models/ItemChange');
const Note = require('../models/Note');
const Resource = require('../models/Resource');
const SearchEngine = require('./searchengine/SearchEngine');
const ItemChangeUtils = require('./ItemChangeUtils');
const { sprintf } = require('sprintf-js');

export default class ResourceService extends BaseService {

	private static isRunningInBackground_:boolean = false;

	private maintenanceCalls_:boolean[] = [];
	private maintenanceTimer1_:any = null
	private maintenanceTimer2_:any = null

	public async indexNoteResources() {
		this.logger().info('ResourceService::indexNoteResources: Start');

		await ItemChange.waitForAllSaved();

		let foundNoteWithEncryption = false;

		while (true) {
			const changes = await ItemChange.modelSelectAll(`
				SELECT id, item_id, type
				FROM item_changes
				WHERE item_type = ?
				AND id > ?
				ORDER BY id ASC
				LIMIT 10
			`,
			[BaseModel.TYPE_NOTE, Setting.value('resourceService.lastProcessedChangeId')]
			);

			if (!changes.length) break;

			const noteIds = changes.map((a:any) => a.item_id);
			const notes = await Note.modelSelectAll(`SELECT id, title, body, encryption_applied FROM notes WHERE id IN ("${noteIds.join('","')}")`);

			const noteById = (noteId:string) => {
				for (let i = 0; i < notes.length; i++) {
					if (notes[i].id === noteId) return notes[i];
				}
				// The note may have been deleted since the change was recorded. For example in this case:
				// - Note created (Some Change object is recorded)
				// - Note is deleted
				// - ResourceService indexer runs.
				// In that case, there will be a change for the note, but the note will be gone.
				return null;
			};

			for (let i = 0; i < changes.length; i++) {
				const change = changes[i];

				if (change.type === ItemChange.TYPE_CREATE || change.type === ItemChange.TYPE_UPDATE) {
					const note = noteById(change.item_id);

					if (note) {
						if (note.encryption_applied) {
							// If we hit an encrypted note, abort processing for now.
							// Note will eventually get decrypted and processing can resume then.
							// This is a limitation of the change tracking system - we cannot skip a change
							// and keep processing the rest since we only keep track of "lastProcessedChangeId".
							foundNoteWithEncryption = true;
							break;
						}

						await this.setAssociatedResources(note.id, note.body);
					} else {
						this.logger().warn(`ResourceService::indexNoteResources: A change was recorded for a note that has been deleted: ${change.item_id}`);
					}
				} else if (change.type === ItemChange.TYPE_DELETE) {
					await NoteResource.remove(change.item_id);
				} else {
					throw new Error(`Invalid change type: ${change.type}`);
				}

				Setting.setValue('resourceService.lastProcessedChangeId', change.id);
			}

			if (foundNoteWithEncryption) break;
		}

		await Setting.saveAll();

		await NoteResource.addOrphanedResources();

		await ItemChangeUtils.deleteProcessedChanges();

		this.logger().info('ResourceService::indexNoteResources: Completed');
	}

	public async setAssociatedResources(noteId:string, noteBody:string) {
		const resourceIds = await Note.linkedResourceIds(noteBody);
		await NoteResource.setAssociatedResources(noteId, resourceIds);
	}

	public async deleteOrphanResources(expiryDelay:number = null) {
		if (expiryDelay === null) expiryDelay = Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000;
		const resourceIds = await NoteResource.orphanResources(expiryDelay);
		this.logger().info('ResourceService::deleteOrphanResources:', resourceIds);
		for (let i = 0; i < resourceIds.length; i++) {
			const resourceId = resourceIds[i];
			const results = await SearchEngine.instance().search(resourceId);
			if (results.length) {
				const note = await Note.load(results[0].id);
				if (note) {
					this.logger().info(sprintf('ResourceService::deleteOrphanResources: Skipping deletion of resource %s because it is still referenced in note %s. Re-indexing note content to fix the issue.', resourceId, note.id));
					await this.setAssociatedResources(note.id, note.body);
				}
			} else {
				await Resource.delete(resourceId);
			}
		}
	}

	private static async autoSetFileSize(resourceId:string, filePath:string, waitTillExists:boolean = true) {
		const itDoes = await shim.fsDriver().waitTillExists(filePath, waitTillExists ? 10000 : 0);
		if (!itDoes) {
			// this.logger().warn('Trying to set file size on non-existent resource:', resourceId, filePath);
			return;
		}
		const fileStat = await shim.fsDriver().stat(filePath);
		await Resource.setFileSizeOnly(resourceId, fileStat.size);
	}

	public static async autoSetFileSizes() {
		const resources = await Resource.needFileSizeSet();

		for (const r of resources) {
			await this.autoSetFileSize(r.id, Resource.fullPath(r), false);
		}
	}

	public async maintenance() {
		this.maintenanceCalls_.push(true);
		try {
			await this.indexNoteResources();
			await this.deleteOrphanResources();
		} finally {
			this.maintenanceCalls_.pop();
		}
	}

	public static runInBackground() {
		if (this.isRunningInBackground_) return;

		this.isRunningInBackground_ = true;
		const service = this.instance();

		service.maintenanceTimer1_ = shim.setTimeout(() => {
			service.maintenance();
		}, 1000 * 30);

		service.maintenanceTimer2_ = shim.setInterval(() => {
			service.maintenance();
		}, 1000 * 60 * 60 * 4);
	}

	public async cancelTimers() {
		if (this.maintenanceTimer1_) {
			shim.clearTimeout(this.maintenanceTimer1_);
			this.maintenanceTimer1_ = null;
		}
		if (this.maintenanceTimer2_) {
			shim.clearInterval(this.maintenanceTimer2_);
			this.maintenanceTimer2_ = null;
		}

		return new Promise((resolve) => {
			const iid = shim.setInterval(() => {
				if (!this.maintenanceCalls_.length) {
					shim.clearInterval(iid);
					resolve();
				}
			}, 100);
		});
	}

	private static instance_:ResourceService = null;

	public static instance() {
		if (this.instance_) return this.instance_;
		this.instance_ = new ResourceService();
		return this.instance_;
	}

}