joplin/packages/lib/models/Resource.ts

import BaseModel from '../BaseModel';
import BaseItem from './BaseItem';
import ItemChange from './ItemChange';
import NoteResource from './NoteResource';
import Setting from './Setting';
import markdownUtils from '../markdownUtils';
import { _ } from '../locale';
import { ResourceEntity, ResourceLocalStateEntity, ResourceOcrStatus, SqlQuery } from '../services/database/types';
import ResourceLocalState from './ResourceLocalState';
import * as pathUtils from '../path-utils';
import { safeFilename } from '../path-utils';
const { mime } = require('../mime-utils.js');
const { FsDriverDummy } = require('../fs-driver-dummy.js');
import JoplinError from '../JoplinError';
import itemCanBeEncrypted from './utils/itemCanBeEncrypted';
import { getEncryptionEnabled } from '../services/synchronizer/syncInfoUtils';
import ShareService from '../services/share/ShareService';
import { LoadOptions } from './utils/types';
import { SaveOptions } from './utils/types';
import { MarkupLanguage } from '@joplin/renderer';
import { htmlentities } from '@joplin/utils/html';
import { RecognizeResultLine } from '../services/ocr/utils/types';
import eventManager, { EventName } from '../eventManager';
import { unique } from '../array';
import isSqliteSyntaxError from '../services/database/isSqliteSyntaxError';
import { internalUrl, isResourceUrl, isSupportedImageMimeType, resourceFilename, resourceFullPath, resourcePathToId, resourceRelativePath, resourceUrlToId } from './utils/resourceUtils';

export default class Resource extends BaseItem {

	public static IMAGE_MAX_DIMENSION = 1920;

	public static FETCH_STATUS_IDLE = 0;
	public static FETCH_STATUS_STARTED = 1;
	public static FETCH_STATUS_DONE = 2;
	public static FETCH_STATUS_ERROR = 3;

	public static shareService_: ShareService = null;

	public static fsDriver_: any;

	public static tableName() {
		return 'resources';
	}

	public static modelType() {
		return BaseModel.TYPE_RESOURCE;
	}

	public static encryptionService() {
		if (!this.encryptionService_) throw new Error('Resource.encryptionService_ is not set!!');
		return this.encryptionService_;
	}

	protected static shareService() {
		if (!this.shareService_) throw new Error('Resource.shareService_ is not set!!');
		return this.shareService_;
	}

	public static isSupportedImageMimeType(type: string) {
		return isSupportedImageMimeType(type);
	}

	public static fetchStatuses(resourceIds: string[]): Promise<any[]> {
		if (!resourceIds.length) return Promise.resolve([]);
		return this.db().selectAll(`SELECT resource_id, fetch_status FROM resource_local_states WHERE resource_id IN ("${resourceIds.join('","')}")`);
	}

	public static sharedResourceIds(): Promise<string[]> {
		return this.db().selectAllFields('SELECT id FROM resources WHERE is_shared = 1', [], 'id');
	}

	public static errorFetchStatuses() {
		return this.db().selectAll(`
			SELECT title AS resource_title, resource_id, fetch_error
			FROM resource_local_states
			LEFT JOIN resources ON resources.id = resource_local_states.resource_id
			WHERE fetch_status = ?
		`, [Resource.FETCH_STATUS_ERROR]);
	}

	public static needToBeFetched(resourceDownloadMode: string = null, limit: number = null) {
		const sql = ['SELECT * FROM resources WHERE encryption_applied = 0 AND id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = ?)'];
		if (resourceDownloadMode !== 'always') {
			sql.push('AND resources.id IN (SELECT resource_id FROM resources_to_download)');
		}
		sql.push('ORDER BY updated_time DESC');
		if (limit !== null) sql.push(`LIMIT ${limit}`);
		return this.modelSelectAll(sql.join(' '), [Resource.FETCH_STATUS_IDLE]);
	}

	public static async resetStartedFetchStatus() {
		return await this.db().exec('UPDATE resource_local_states SET fetch_status = ? WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE, Resource.FETCH_STATUS_STARTED]);
	}

	public static async resetFetchErrorStatus(resourceId: string) {
		await this.db().exec('UPDATE resource_local_states SET fetch_status = ?, fetch_error = "" WHERE resource_id = ?', [Resource.FETCH_STATUS_IDLE, resourceId]);
		await this.resetOcrStatus(resourceId);
	}

	public static fsDriver() {
		if (!Resource.fsDriver_) Resource.fsDriver_ = new FsDriverDummy();
		return Resource.fsDriver_;
	}

	// DEPRECATED IN FAVOUR OF friendlySafeFilename()
	public static friendlyFilename(resource: ResourceEntity) {
		let output = safeFilename(resource.title); // Make sure not to allow spaces or any special characters as it's not supported in HTTP headers
		if (!output) output = resource.id;
		let extension = resource.file_extension;
		if (!extension) extension = resource.mime ? mime.toFileExtension(resource.mime) : '';
		extension = extension ? `.${extension}` : '';
		return output + extension;
	}

	public static baseDirectoryPath() {
		return Setting.value('resourceDir');
	}

	public static baseRelativeDirectoryPath() {
		return Setting.value('resourceDirName');
	}

	public static filename(resource: ResourceEntity, encryptedBlob = false) {
		return resourceFilename(resource, encryptedBlob);
	}

	public static friendlySafeFilename(resource: ResourceEntity) {
		let ext = resource.file_extension;
		if (!ext) ext = resource.mime ? mime.toFileExtension(resource.mime) : '';
		const safeExt = ext ? pathUtils.safeFileExtension(ext).toLowerCase() : '';
		let title = resource.title ? resource.title : resource.id;
		if (safeExt && pathUtils.fileExtension(title).toLowerCase() === safeExt) title = pathUtils.filename(title);
		return pathUtils.friendlySafeFilename(title) + (safeExt ? `.${safeExt}` : '');
	}

	public static relativePath(resource: ResourceEntity, encryptedBlob = false) {
		return resourceRelativePath(resource, this.baseRelativeDirectoryPath(), encryptedBlob);
	}

	public static fullPath(resource: ResourceEntity, encryptedBlob = false) {
		return resourceFullPath(resource, this.baseDirectoryPath(), encryptedBlob);
	}

	public static async isReady(resource: ResourceEntity) {
		const r = await this.readyStatus(resource);
		return r === 'ok';
	}

	public static async readyStatus(resource: ResourceEntity) {
		const ls = await this.localState(resource);
		if (!resource) return 'notFound';
		if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) return 'notDownloaded';
		if (resource.encryption_blob_encrypted) return 'encrypted';
		return 'ok';
	}

	public static async requireIsReady(resource: ResourceEntity) {
		const readyStatus = await Resource.readyStatus(resource);
		if (readyStatus !== 'ok') throw new Error(`Resource is not ready. Status: ${readyStatus}`);
	}

	// For resources, we need to decrypt the item (metadata) and the resource binary blob.
	public static async decrypt(item: ResourceEntity) {
		// The item might already be decrypted but not the blob (for instance if it crashes while
		// decrypting the blob or was otherwise interrupted).
		const decryptedItem = item.encryption_cipher_text ? await super.decrypt(item) : { ...item };
		if (!decryptedItem.encryption_blob_encrypted) return decryptedItem;

		const localState = await this.localState(item);
		if (localState.fetch_status !== Resource.FETCH_STATUS_DONE) {
			// Not an error - it means the blob has not been downloaded yet.
			// It will be decrypted later on, once downloaded.
			return decryptedItem;
		}

		const plainTextPath = this.fullPath(decryptedItem);
		const encryptedPath = this.fullPath(decryptedItem, true);
		const noExtPath = `${pathUtils.dirname(encryptedPath)}/${pathUtils.filename(encryptedPath)}`;

		// When the resource blob is downloaded by the synchroniser, it's initially a file with no
		// extension (since it's encrypted, so we don't know its extension). So here rename it
		// to a file with a ".crypted" extension so that it's better identified, and then decrypt it.
		// Potentially plainTextPath is also a path with no extension if it's an unknown mime type.
		if (await this.fsDriver().exists(noExtPath)) {
			await this.fsDriver().move(noExtPath, encryptedPath);
		}

		try {
			await this.encryptionService().decryptFile(encryptedPath, plainTextPath);
		} catch (error) {
			if (error.code === 'invalidIdentifier') {
				// As the identifier is invalid it most likely means that this is not encrypted data
				// at all. It can happen for example when there's a crash between the moment the data
				// is decrypted and the resource item is updated.
				this.logger().warn(`Found a resource that was most likely already decrypted but was marked as encrypted. Marked it as decrypted: ${item.id}`);
				this.fsDriver().move(encryptedPath, plainTextPath);
			} else {
				throw error;
			}
		}

		decryptedItem.encryption_blob_encrypted = 0;
		return super.save(decryptedItem, { autoTimestamp: false });
	}

	// Prepare the resource by encrypting it if needed.
	// The call returns the path to the physical file AND a representation of the resource object
	// as it should be uploaded to the sync target. Note that this may be different from what is stored
	// in the database. In particular, the flag encryption_blob_encrypted might be 1 on the sync target
	// if the resource is encrypted, but will be 0 locally because the device has the decrypted resource.
	public static async fullPathForSyncUpload(resource: ResourceEntity) {
		const plainTextPath = this.fullPath(resource);

		const share = resource.share_id ? await this.shareService().shareById(resource.share_id) : null;

		if (!getEncryptionEnabled() || !itemCanBeEncrypted(resource as any, share)) {
			// Normally not possible since itemsThatNeedSync should only return decrypted items
			if (resource.encryption_blob_encrypted) throw new Error('Trying to access encrypted resource but encryption is currently disabled');
			return { path: plainTextPath, resource: resource };
		}

		const encryptedPath = this.fullPath(resource, true);
		if (resource.encryption_blob_encrypted) return { path: encryptedPath, resource: resource };

		try {
			await this.encryptionService().encryptFile(plainTextPath, encryptedPath, {
				masterKeyId: share && share.master_key_id ? share.master_key_id : '',
			});
		} catch (error) {
			if (error.code === 'ENOENT') {
				throw new JoplinError(
					`Trying to encrypt resource but only metadata is present: ${error.toString()}`, 'fileNotFound',
				);
			}
			throw error;
		}

		const resourceCopy = { ...resource };
		resourceCopy.encryption_blob_encrypted = 1;
		return { path: encryptedPath, resource: resourceCopy };
	}

	public static markupTag(resource: any, markupLanguage: MarkupLanguage = MarkupLanguage.Markdown) {
		let tagAlt = resource.alt ? resource.alt : resource.title;
		if (!tagAlt) tagAlt = '';
		const lines = [];
		if (Resource.isSupportedImageMimeType(resource.mime)) {
			if (markupLanguage === MarkupLanguage.Markdown) {
				lines.push('![');
				lines.push(markdownUtils.escapeTitleText(tagAlt));
				lines.push(`](:/${resource.id})`);
			} else {
				const altHtml = tagAlt ? `alt="${htmlentities(tagAlt)}"` : '';
				lines.push(`<img src=":/${resource.id}" ${altHtml}/>`);
			}
		} else {
			if (markupLanguage === MarkupLanguage.Markdown) {
				lines.push('[');
				lines.push(markdownUtils.escapeTitleText(tagAlt));
				lines.push(`](:/${resource.id})`);
			} else {
				const altHtml = tagAlt ? `alt="${htmlentities(tagAlt)}"` : '';
				lines.push(`<a href=":/${resource.id}" ${altHtml}>${htmlentities(tagAlt ? tagAlt : resource.id)}</a>`);
			}
		}
		return lines.join('');
	}

	public static internalUrl(resource: ResourceEntity) {
		return internalUrl(resource);
	}

	public static pathToId(path: string) {
		return resourcePathToId(path);
	}

	public static async content(resource: ResourceEntity) {
		return this.fsDriver().readFile(this.fullPath(resource), 'Buffer');
	}

	public static isResourceUrl(url: string) {
		return isResourceUrl(url);
	}

	public static urlToId(url: string) {
		return resourceUrlToId(url);
	}

	public static async localState(resourceOrId: any): Promise<ResourceLocalStateEntity> {
		return ResourceLocalState.byResourceId(typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId);
	}

	public static setLocalStateQueries(resourceOrId: any, state: ResourceLocalStateEntity) {
		const id = typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId;
		return ResourceLocalState.saveQueries({ ...state, resource_id: id });
	}

	public static async setLocalState(resourceOrId: any, state: ResourceLocalStateEntity) {
		const id = typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId;
		await ResourceLocalState.save({ ...state, resource_id: id });
	}

	public static async needFileSizeSet() {
		return this.modelSelectAll('SELECT * FROM resources WHERE `size` < 0 AND encryption_blob_encrypted = 0');
	}

	// Only set the `size` field and nothing else, not even the update_time
	// This is because it's only necessary to do it once after migration 20
	// and each client does it so there's no need to sync the resource.
	public static async setFileSizeOnly(resourceId: string, fileSize: number) {
		return this.db().exec('UPDATE resources set `size` = ? WHERE id = ?', [fileSize, resourceId]);
	}

	public static async batchDelete(ids: string[], options: any = null) {
		// For resources, there's not really batch deletion since there's the
		// file data to delete too, so each is processed one by one with the
		// file data being deleted last since the metadata deletion call may
		// throw (for example if trying to delete a read-only item).
		for (let i = 0; i < ids.length; i++) {
			const id = ids[i];
			const resource = await Resource.load(id);
			if (!resource) continue;

			const path = Resource.fullPath(resource);
			await super.batchDelete([id], options);
			await this.fsDriver().remove(path);
			await NoteResource.deleteByResource(id); // Clean up note/resource relationships
			await this.db().exec('DELETE FROM items_normalized WHERE item_id = ?', [id]);
		}

		await ResourceLocalState.batchDelete(ids);
	}

	public static async markForDownload(resourceId: string) {
		// Insert the row only if it's not already there
		const t = Date.now();
		await this.db().exec('INSERT INTO resources_to_download (resource_id, updated_time, created_time) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM resources_to_download WHERE resource_id = ?)', [resourceId, t, t, resourceId]);
	}

	public static async downloadedButEncryptedBlobCount(excludedIds: string[] = null) {
		let excludedSql = '';
		if (excludedIds && excludedIds.length) {
			excludedSql = `AND resource_id NOT IN ("${excludedIds.join('","')}")`;
		}

		const r = await this.db().selectOne(`
			SELECT count(*) as total
			FROM resource_local_states
			WHERE fetch_status = ?
			AND resource_id IN (SELECT id FROM resources WHERE encryption_blob_encrypted = 1)
			${excludedSql}
		`, [Resource.FETCH_STATUS_DONE]);

		return r ? r.total : 0;
	}

	public static async downloadStatusCounts(status: number) {
		const r = await this.db().selectOne(`
			SELECT count(*) as total
			FROM resource_local_states
			WHERE fetch_status = ?
		`, [status]);

		return r ? r.total : 0;
	}

	public static async createdLocallyCount() {
		const r = await this.db().selectOne(`
			SELECT count(*) as total
			FROM resources
			WHERE id NOT IN
			(SELECT resource_id FROM resource_local_states)
		`);

		return r ? r.total : 0;
	}

	public static fetchStatusToLabel(status: number) {
		if (status === Resource.FETCH_STATUS_IDLE) return _('Not downloaded');
		if (status === Resource.FETCH_STATUS_STARTED) return _('Downloading');
		if (status === Resource.FETCH_STATUS_DONE) return _('Downloaded');
		if (status === Resource.FETCH_STATUS_ERROR) return _('Error');
		throw new Error(`Invalid status: ${status}`);
	}

	public static async updateResourceBlobContent(resourceId: string, newBlobFilePath: string) {
		const resource = await Resource.load(resourceId);
		await this.requireIsReady(resource);

		const fileStat = await this.fsDriver().stat(newBlobFilePath);

		// We first save the resource metadata because this can throw, for
		// example if modifying a resource that is read-only

		const now = Date.now();

		const result = await Resource.save({
			id: resource.id,
			size: fileStat.size,
			updated_time: now,
			blob_updated_time: now,
		}, {
			autoTimestamp: false,
		});

		// If the above call has succeeded, we save the data blob

		await this.fsDriver().copy(newBlobFilePath, Resource.fullPath(resource));

		return result;
	}

	public static async resourceBlobContent(resourceId: string, encoding = 'Buffer') {
		const resource = await Resource.load(resourceId);
		await this.requireIsReady(resource);
		return await this.fsDriver().readFile(Resource.fullPath(resource), encoding);
	}

	public static async duplicateResource(resourceId: string): Promise<ResourceEntity> {
		const resource = await Resource.load(resourceId);
		const localState = await Resource.localState(resource);

		let newResource: ResourceEntity = { ...resource };
		delete newResource.id;
		delete newResource.is_shared;
		delete newResource.share_id;
		newResource = await Resource.save(newResource);

		const newLocalState = { ...localState };
		newLocalState.resource_id = newResource.id;
		delete newLocalState.id;

		await Resource.setLocalState(newResource, newLocalState);

		const sourcePath = Resource.fullPath(resource);
		if (await this.fsDriver().exists(sourcePath)) {
			await this.fsDriver().copy(sourcePath, Resource.fullPath(newResource));
		}

		return newResource;
	}

	public static async resourceConflictFolderId(): Promise<string> {
		const folder = await this.resourceConflictFolder();
		return folder.id;
	}

	private static async resourceConflictFolder(): Promise<any> {
		const conflictFolderTitle = _('Conflicts (attachments)');
		const Folder = this.getClass('Folder');

		const folder = await Folder.loadByTitle(conflictFolderTitle);
		if (!folder || folder.parent_id) {
			return Folder.save({ title: conflictFolderTitle });
		}

		return folder;
	}

	public static mustHandleConflict(local: ResourceEntity, remote: ResourceEntity) {
		// That shouldn't happen so throw an exception
		if (local.id !== remote.id) throw new Error('Cannot handle conflict for two different resources');

		// If the content has changed, we need to handle the conflict
		if (local.blob_updated_time !== remote.blob_updated_time) return true;

		// If nothing has been changed, or if only the metadata has been
		// changed, we just keep the remote version. Most of the resource
		// metadata is not user-editable so there won't be any data loss. Such a
		// conflict might happen for example if a resource is OCRed by two
		// different clients.
		return false;
	}

	public static async createConflictResourceNote(resource: ResourceEntity) {
		const Note = this.getClass('Note');
		const conflictResource = await Resource.duplicateResource(resource.id);

		await Note.save({
			title: _('Attachment conflict: "%s"', resource.title),
			body: _('There was a [conflict](%s) on the attachment below.\n\n%s', 'https://joplinapp.org/help/apps/conflict', Resource.markupTag(conflictResource)),
			parent_id: await this.resourceConflictFolderId(),
		}, { changeSource: ItemChange.SOURCE_SYNC });
	}

	private static baseNeedOcrQuery(selectSql: string, supportedMimeTypes: string[]): SqlQuery {
		return {
			sql: `
				SELECT ${selectSql}
				FROM resources
				WHERE
					ocr_status = ? AND
					encryption_applied = 0 AND
					mime IN ("${supportedMimeTypes.join('","')}")
			`,
			params: [
				ResourceOcrStatus.Todo,
			],
		};
	}

	public static async needOcrCount(supportedMimeTypes: string[]): Promise<number> {
		const query = this.baseNeedOcrQuery('count(*) as total', supportedMimeTypes);
		const r = await this.db().selectOne(query.sql, query.params);
		return r ? r['total'] : 0;
	}

	public static async needOcr(supportedMimeTypes: string[], skippedResourceIds: string[], limit: number, options: LoadOptions): Promise<ResourceEntity[]> {
		const query = this.baseNeedOcrQuery(this.selectFields(options), supportedMimeTypes);
		const skippedResourcesSql = skippedResourceIds.length ? `AND resources.id NOT IN  ("${skippedResourceIds.join('","')}")` : '';

		return await this.db().selectAll(`
			${query.sql}
			${skippedResourcesSql}			
			ORDER BY updated_time DESC
			LIMIT ${limit}
		`, query.params);
	}

	private static async resetOcrStatus(resourceId: string) {
		await Resource.save({
			id: resourceId,
			ocr_error: '',
			ocr_text: '',
			ocr_status: ResourceOcrStatus.Todo,
		});
	}

	public static serializeOcrDetails(details: RecognizeResultLine[]) {
		if (!details || !details.length) return '';
		return JSON.stringify(details);
	}

	public static unserializeOcrDetails(s: string): RecognizeResultLine[] | null {
		if (!s) return null;
		try {
			const r = JSON.parse(s);
			if (!r) return null;
			if (!Array.isArray(r)) throw new Error('OCR details are not valid (not an array');
			return r;
		} catch (error) {
			error.message = `Could not unserialized OCR data: ${error.message}`;
			throw error;
		}
	}

	public static async resourceOcrTextsByIds(ids: string[]): Promise<ResourceEntity[]> {
		if (!ids.length) return [];
		ids = unique(ids);
		return this.modelSelectAll(`SELECT id, ocr_text FROM resources WHERE id IN ("${ids.join('","')}")`);
	}

	public static async allForNormalization(updatedTime: number, id: string, limit = 100, options: LoadOptions = null) {
		const makeQuery = (useRowValue: boolean): SqlQuery => {
			const whereSql = useRowValue ? '(updated_time, id) > (?, ?)' : 'updated_time > ?';

			const params: any[] = [updatedTime];
			if (useRowValue) {
				params.push(id);
			}
			params.push(ResourceOcrStatus.Done);
			params.push(limit);

			return {
				sql: `
					SELECT ${this.selectFields(options)} FROM resources
					WHERE ${whereSql}
					AND ocr_text != ""
					AND ocr_status = ?
					ORDER BY updated_time ASC, id ASC
					LIMIT ?
				`,
				params,
			};
		};

		// We use a row value in this query, and that's not supported on certain
		// Android devices (API level <= 24). So if the query fails, we fallback
		// to a non-row value query. Although it may be inaccurate in some cases
		// it wouldn't be a critical issue (some OCRed resources may not be part
		// of the search engine results) and it means we can keep supporting old
		// Android devices.
		try {
			const r = await this.modelSelectAll(makeQuery(true));
			return r;
		} catch (error) {
			if (isSqliteSyntaxError(error)) {
				const r = await this.modelSelectAll(makeQuery(false));
				return r;
			} else {
				throw error;
			}
		}
	}

	public static async save(o: ResourceEntity, options: SaveOptions = null): Promise<ResourceEntity> {
		const resource = { ...o };

		const isNew = this.isNew(o, options);

		if (isNew) {
			const now = Date.now();
			options = { ...options, autoTimestamp: false };
			if (!resource.created_time) resource.created_time = now;
			if (!resource.updated_time) resource.updated_time = now;
			if (!resource.blob_updated_time) resource.blob_updated_time = now;
		}

		const output = await super.save(resource, options);
		if (isNew) eventManager.emit(EventName.ResourceCreate);
		return output;
	}

}