1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
joplin/packages/lib/models/Resource.ts

619 lines
23 KiB
TypeScript

import BaseModel from '../BaseModel';
import BaseItem from './BaseItem';
import ItemChange from './ItemChange';
import NoteResource from './NoteResource';
import Setting from './Setting';
import markdownUtils from '../markdownUtils';
import { _ } from '../locale';
import { ResourceEntity, ResourceLocalStateEntity, ResourceOcrStatus, SqlQuery } from '../services/database/types';
import ResourceLocalState from './ResourceLocalState';
const pathUtils = require('../path-utils');
const { mime } = require('../mime-utils.js');
const { filename, safeFilename } = require('../path-utils');
const { FsDriverDummy } = require('../fs-driver-dummy.js');
import JoplinError from '../JoplinError';
import itemCanBeEncrypted from './utils/itemCanBeEncrypted';
import { getEncryptionEnabled } from '../services/synchronizer/syncInfoUtils';
import ShareService from '../services/share/ShareService';
import { LoadOptions } from './utils/types';
import { SaveOptions } from './utils/types';
import { MarkupLanguage } from '@joplin/renderer';
import { htmlentities } from '@joplin/utils/html';
import { RecognizeResultLine } from '../services/ocr/utils/types';
import eventManager, { EventName } from '../eventManager';
import { unique } from '../array';
import isSqliteSyntaxError from '../services/database/isSqliteSyntaxError';
export default class Resource extends BaseItem {
public static IMAGE_MAX_DIMENSION = 1920;
public static FETCH_STATUS_IDLE = 0;
public static FETCH_STATUS_STARTED = 1;
public static FETCH_STATUS_DONE = 2;
public static FETCH_STATUS_ERROR = 3;
public static shareService_: ShareService = null;
public static fsDriver_: any;
public static tableName() {
return 'resources';
}
public static modelType() {
return BaseModel.TYPE_RESOURCE;
}
public static encryptionService() {
if (!this.encryptionService_) throw new Error('Resource.encryptionService_ is not set!!');
return this.encryptionService_;
}
protected static shareService() {
if (!this.shareService_) throw new Error('Resource.shareService_ is not set!!');
return this.shareService_;
}
public static isSupportedImageMimeType(type: string) {
const imageMimeTypes = ['image/jpg', 'image/jpeg', 'image/png', 'image/gif', 'image/svg+xml', 'image/webp', 'image/avif'];
return imageMimeTypes.indexOf(type.toLowerCase()) >= 0;
}
public static fetchStatuses(resourceIds: string[]): Promise<any[]> {
if (!resourceIds.length) return Promise.resolve([]);
return this.db().selectAll(`SELECT resource_id, fetch_status FROM resource_local_states WHERE resource_id IN ("${resourceIds.join('","')}")`);
}
public static sharedResourceIds(): Promise<string[]> {
return this.db().selectAllFields('SELECT id FROM resources WHERE is_shared = 1', [], 'id');
}
public static errorFetchStatuses() {
return this.db().selectAll(`
SELECT title AS resource_title, resource_id, fetch_error
FROM resource_local_states
LEFT JOIN resources ON resources.id = resource_local_states.resource_id
WHERE fetch_status = ?
`, [Resource.FETCH_STATUS_ERROR]);
}
public static needToBeFetched(resourceDownloadMode: string = null, limit: number = null) {
const sql = ['SELECT * FROM resources WHERE encryption_applied = 0 AND id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = ?)'];
if (resourceDownloadMode !== 'always') {
sql.push('AND resources.id IN (SELECT resource_id FROM resources_to_download)');
}
sql.push('ORDER BY updated_time DESC');
if (limit !== null) sql.push(`LIMIT ${limit}`);
return this.modelSelectAll(sql.join(' '), [Resource.FETCH_STATUS_IDLE]);
}
public static async resetStartedFetchStatus() {
return await this.db().exec('UPDATE resource_local_states SET fetch_status = ? WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE, Resource.FETCH_STATUS_STARTED]);
}
public static async resetFetchErrorStatus(resourceId: string) {
await this.db().exec('UPDATE resource_local_states SET fetch_status = ?, fetch_error = "" WHERE resource_id = ?', [Resource.FETCH_STATUS_IDLE, resourceId]);
await this.resetOcrStatus(resourceId);
}
public static fsDriver() {
if (!Resource.fsDriver_) Resource.fsDriver_ = new FsDriverDummy();
return Resource.fsDriver_;
}
// DEPRECATED IN FAVOUR OF friendlySafeFilename()
public static friendlyFilename(resource: ResourceEntity) {
let output = safeFilename(resource.title); // Make sure not to allow spaces or any special characters as it's not supported in HTTP headers
if (!output) output = resource.id;
let extension = resource.file_extension;
if (!extension) extension = resource.mime ? mime.toFileExtension(resource.mime) : '';
extension = extension ? `.${extension}` : '';
return output + extension;
}
public static baseDirectoryPath() {
return Setting.value('resourceDir');
}
public static baseRelativeDirectoryPath() {
return Setting.value('resourceDirName');
}
public static filename(resource: ResourceEntity, encryptedBlob = false) {
let extension = encryptedBlob ? 'crypted' : resource.file_extension;
if (!extension) extension = resource.mime ? mime.toFileExtension(resource.mime) : '';
extension = extension ? `.${extension}` : '';
return resource.id + extension;
}
public static friendlySafeFilename(resource: ResourceEntity) {
let ext = resource.file_extension;
if (!ext) ext = resource.mime ? mime.toFileExtension(resource.mime) : '';
const safeExt = ext ? pathUtils.safeFileExtension(ext).toLowerCase() : '';
let title = resource.title ? resource.title : resource.id;
if (safeExt && pathUtils.fileExtension(title).toLowerCase() === safeExt) title = pathUtils.filename(title);
return pathUtils.friendlySafeFilename(title) + (safeExt ? `.${safeExt}` : '');
}
public static relativePath(resource: ResourceEntity, encryptedBlob = false) {
return `${Setting.value('resourceDirName')}/${this.filename(resource, encryptedBlob)}`;
}
public static fullPath(resource: ResourceEntity, encryptedBlob = false) {
return `${Setting.value('resourceDir')}/${this.filename(resource, encryptedBlob)}`;
}
public static async isReady(resource: ResourceEntity) {
const r = await this.readyStatus(resource);
return r === 'ok';
}
public static async readyStatus(resource: ResourceEntity) {
const ls = await this.localState(resource);
if (!resource) return 'notFound';
if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) return 'notDownloaded';
if (resource.encryption_blob_encrypted) return 'encrypted';
return 'ok';
}
public static async requireIsReady(resource: ResourceEntity) {
const readyStatus = await Resource.readyStatus(resource);
if (readyStatus !== 'ok') throw new Error(`Resource is not ready. Status: ${readyStatus}`);
}
// For resources, we need to decrypt the item (metadata) and the resource binary blob.
public static async decrypt(item: ResourceEntity) {
// The item might already be decrypted but not the blob (for instance if it crashes while
// decrypting the blob or was otherwise interrupted).
const decryptedItem = item.encryption_cipher_text ? await super.decrypt(item) : { ...item };
if (!decryptedItem.encryption_blob_encrypted) return decryptedItem;
const localState = await this.localState(item);
if (localState.fetch_status !== Resource.FETCH_STATUS_DONE) {
// Not an error - it means the blob has not been downloaded yet.
// It will be decrypted later on, once downloaded.
return decryptedItem;
}
const plainTextPath = this.fullPath(decryptedItem);
const encryptedPath = this.fullPath(decryptedItem, true);
const noExtPath = `${pathUtils.dirname(encryptedPath)}/${pathUtils.filename(encryptedPath)}`;
// When the resource blob is downloaded by the synchroniser, it's initially a file with no
// extension (since it's encrypted, so we don't know its extension). So here rename it
// to a file with a ".crypted" extension so that it's better identified, and then decrypt it.
// Potentially plainTextPath is also a path with no extension if it's an unknown mime type.
if (await this.fsDriver().exists(noExtPath)) {
await this.fsDriver().move(noExtPath, encryptedPath);
}
try {
await this.encryptionService().decryptFile(encryptedPath, plainTextPath);
} catch (error) {
if (error.code === 'invalidIdentifier') {
// As the identifier is invalid it most likely means that this is not encrypted data
// at all. It can happen for example when there's a crash between the moment the data
// is decrypted and the resource item is updated.
this.logger().warn(`Found a resource that was most likely already decrypted but was marked as encrypted. Marked it as decrypted: ${item.id}`);
this.fsDriver().move(encryptedPath, plainTextPath);
} else {
throw error;
}
}
decryptedItem.encryption_blob_encrypted = 0;
return super.save(decryptedItem, { autoTimestamp: false });
}
// Prepare the resource by encrypting it if needed.
// The call returns the path to the physical file AND a representation of the resource object
// as it should be uploaded to the sync target. Note that this may be different from what is stored
// in the database. In particular, the flag encryption_blob_encrypted might be 1 on the sync target
// if the resource is encrypted, but will be 0 locally because the device has the decrypted resource.
public static async fullPathForSyncUpload(resource: ResourceEntity) {
const plainTextPath = this.fullPath(resource);
const share = resource.share_id ? await this.shareService().shareById(resource.share_id) : null;
if (!getEncryptionEnabled() || !itemCanBeEncrypted(resource as any, share)) {
// Normally not possible since itemsThatNeedSync should only return decrypted items
if (resource.encryption_blob_encrypted) throw new Error('Trying to access encrypted resource but encryption is currently disabled');
return { path: plainTextPath, resource: resource };
}
const encryptedPath = this.fullPath(resource, true);
if (resource.encryption_blob_encrypted) return { path: encryptedPath, resource: resource };
try {
await this.encryptionService().encryptFile(plainTextPath, encryptedPath, {
masterKeyId: share && share.master_key_id ? share.master_key_id : '',
});
} catch (error) {
if (error.code === 'ENOENT') {
throw new JoplinError(
`Trying to encrypt resource but only metadata is present: ${error.toString()}`, 'fileNotFound',
);
}
throw error;
}
const resourceCopy = { ...resource };
resourceCopy.encryption_blob_encrypted = 1;
return { path: encryptedPath, resource: resourceCopy };
}
public static markupTag(resource: any, markupLanguage: MarkupLanguage = MarkupLanguage.Markdown) {
let tagAlt = resource.alt ? resource.alt : resource.title;
if (!tagAlt) tagAlt = '';
const lines = [];
if (Resource.isSupportedImageMimeType(resource.mime)) {
if (markupLanguage === MarkupLanguage.Markdown) {
lines.push('![');
lines.push(markdownUtils.escapeTitleText(tagAlt));
lines.push(`](:/${resource.id})`);
} else {
const altHtml = tagAlt ? `alt="${htmlentities(tagAlt)}"` : '';
lines.push(`<img src=":/${resource.id}" ${altHtml}/>`);
}
} else {
if (markupLanguage === MarkupLanguage.Markdown) {
lines.push('[');
lines.push(markdownUtils.escapeTitleText(tagAlt));
lines.push(`](:/${resource.id})`);
} else {
const altHtml = tagAlt ? `alt="${htmlentities(tagAlt)}"` : '';
lines.push(`<a href=":/${resource.id}" ${altHtml}>${htmlentities(tagAlt ? tagAlt : resource.id)}</a>`);
}
}
return lines.join('');
}
public static internalUrl(resource: ResourceEntity) {
return `:/${resource.id}`;
}
public static pathToId(path: string) {
return filename(path);
}
public static async content(resource: ResourceEntity) {
return this.fsDriver().readFile(this.fullPath(resource), 'Buffer');
}
public static isResourceUrl(url: string) {
return url && url.length === 34 && url[0] === ':' && url[1] === '/';
}
public static urlToId(url: string) {
if (!this.isResourceUrl(url)) throw new Error(`Not a valid resource URL: ${url}`);
return url.substr(2);
}
public static async localState(resourceOrId: any): Promise<ResourceLocalStateEntity> {
return ResourceLocalState.byResourceId(typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId);
}
public static setLocalStateQueries(resourceOrId: any, state: ResourceLocalStateEntity) {
const id = typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId;
return ResourceLocalState.saveQueries({ ...state, resource_id: id });
}
public static async setLocalState(resourceOrId: any, state: ResourceLocalStateEntity) {
const id = typeof resourceOrId === 'object' ? resourceOrId.id : resourceOrId;
await ResourceLocalState.save({ ...state, resource_id: id });
}
public static async needFileSizeSet() {
return this.modelSelectAll('SELECT * FROM resources WHERE `size` < 0 AND encryption_blob_encrypted = 0');
}
// Only set the `size` field and nothing else, not even the update_time
// This is because it's only necessary to do it once after migration 20
// and each client does it so there's no need to sync the resource.
public static async setFileSizeOnly(resourceId: string, fileSize: number) {
return this.db().exec('UPDATE resources set `size` = ? WHERE id = ?', [fileSize, resourceId]);
}
public static async batchDelete(ids: string[], options: any = null) {
// For resources, there's not really batch deletion since there's the
// file data to delete too, so each is processed one by one with the
// file data being deleted last since the metadata deletion call may
// throw (for example if trying to delete a read-only item).
for (let i = 0; i < ids.length; i++) {
const id = ids[i];
const resource = await Resource.load(id);
if (!resource) continue;
const path = Resource.fullPath(resource);
await super.batchDelete([id], options);
await this.fsDriver().remove(path);
await NoteResource.deleteByResource(id); // Clean up note/resource relationships
await this.db().exec('DELETE FROM items_normalized WHERE item_id = ?', [id]);
}
await ResourceLocalState.batchDelete(ids);
}
public static async markForDownload(resourceId: string) {
// Insert the row only if it's not already there
const t = Date.now();
await this.db().exec('INSERT INTO resources_to_download (resource_id, updated_time, created_time) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM resources_to_download WHERE resource_id = ?)', [resourceId, t, t, resourceId]);
}
public static async downloadedButEncryptedBlobCount(excludedIds: string[] = null) {
let excludedSql = '';
if (excludedIds && excludedIds.length) {
excludedSql = `AND resource_id NOT IN ("${excludedIds.join('","')}")`;
}
const r = await this.db().selectOne(`
SELECT count(*) as total
FROM resource_local_states
WHERE fetch_status = ?
AND resource_id IN (SELECT id FROM resources WHERE encryption_blob_encrypted = 1)
${excludedSql}
`, [Resource.FETCH_STATUS_DONE]);
return r ? r.total : 0;
}
public static async downloadStatusCounts(status: number) {
const r = await this.db().selectOne(`
SELECT count(*) as total
FROM resource_local_states
WHERE fetch_status = ?
`, [status]);
return r ? r.total : 0;
}
public static async createdLocallyCount() {
const r = await this.db().selectOne(`
SELECT count(*) as total
FROM resources
WHERE id NOT IN
(SELECT resource_id FROM resource_local_states)
`);
return r ? r.total : 0;
}
public static fetchStatusToLabel(status: number) {
if (status === Resource.FETCH_STATUS_IDLE) return _('Not downloaded');
if (status === Resource.FETCH_STATUS_STARTED) return _('Downloading');
if (status === Resource.FETCH_STATUS_DONE) return _('Downloaded');
if (status === Resource.FETCH_STATUS_ERROR) return _('Error');
throw new Error(`Invalid status: ${status}`);
}
public static async updateResourceBlobContent(resourceId: string, newBlobFilePath: string) {
const resource = await Resource.load(resourceId);
await this.requireIsReady(resource);
const fileStat = await this.fsDriver().stat(newBlobFilePath);
// We first save the resource metadata because this can throw, for
// example if modifying a resource that is read-only
const now = Date.now();
const result = await Resource.save({
id: resource.id,
size: fileStat.size,
updated_time: now,
blob_updated_time: now,
}, {
autoTimestamp: false,
});
// If the above call has succeeded, we save the data blob
await this.fsDriver().copy(newBlobFilePath, Resource.fullPath(resource));
return result;
}
public static async resourceBlobContent(resourceId: string, encoding = 'Buffer') {
const resource = await Resource.load(resourceId);
await this.requireIsReady(resource);
return await this.fsDriver().readFile(Resource.fullPath(resource), encoding);
}
public static async duplicateResource(resourceId: string): Promise<ResourceEntity> {
const resource = await Resource.load(resourceId);
const localState = await Resource.localState(resource);
let newResource: ResourceEntity = { ...resource };
delete newResource.id;
delete newResource.is_shared;
delete newResource.share_id;
newResource = await Resource.save(newResource);
const newLocalState = { ...localState };
newLocalState.resource_id = newResource.id;
delete newLocalState.id;
await Resource.setLocalState(newResource, newLocalState);
const sourcePath = Resource.fullPath(resource);
if (await this.fsDriver().exists(sourcePath)) {
await this.fsDriver().copy(sourcePath, Resource.fullPath(newResource));
}
return newResource;
}
public static async resourceConflictFolderId(): Promise<string> {
const folder = await this.resourceConflictFolder();
return folder.id;
}
private static async resourceConflictFolder(): Promise<any> {
const conflictFolderTitle = _('Conflicts (attachments)');
const Folder = this.getClass('Folder');
const folder = await Folder.loadByTitle(conflictFolderTitle);
if (!folder || folder.parent_id) {
return Folder.save({ title: conflictFolderTitle });
}
return folder;
}
public static mustHandleConflict(local: ResourceEntity, remote: ResourceEntity) {
// That shouldn't happen so throw an exception
if (local.id !== remote.id) throw new Error('Cannot handle conflict for two different resources');
// If the content has changed, we need to handle the conflict
if (local.blob_updated_time !== remote.blob_updated_time) return true;
// If nothing has been changed, or if only the metadata has been
// changed, we just keep the remote version. Most of the resource
// metadata is not user-editable so there won't be any data loss. Such a
// conflict might happen for example if a resource is OCRed by two
// different clients.
return false;
}
public static async createConflictResourceNote(resource: ResourceEntity) {
const Note = this.getClass('Note');
const conflictResource = await Resource.duplicateResource(resource.id);
await Note.save({
title: _('Attachment conflict: "%s"', resource.title),
body: _('There was a [conflict](%s) on the attachment below.\n\n%s', 'https://joplinapp.org/help/apps/conflict', Resource.markupTag(conflictResource)),
parent_id: await this.resourceConflictFolderId(),
}, { changeSource: ItemChange.SOURCE_SYNC });
}
private static baseNeedOcrQuery(selectSql: string, supportedMimeTypes: string[]): SqlQuery {
return {
sql: `
SELECT ${selectSql}
FROM resources
WHERE
ocr_status = ? AND
encryption_applied = 0 AND
mime IN ("${supportedMimeTypes.join('","')}")
`,
params: [
ResourceOcrStatus.Todo,
],
};
}
public static async needOcrCount(supportedMimeTypes: string[]): Promise<number> {
const query = this.baseNeedOcrQuery('count(*) as total', supportedMimeTypes);
const r = await this.db().selectOne(query.sql, query.params);
return r ? r['total'] : 0;
}
public static async needOcr(supportedMimeTypes: string[], skippedResourceIds: string[], limit: number, options: LoadOptions): Promise<ResourceEntity[]> {
const query = this.baseNeedOcrQuery(this.selectFields(options), supportedMimeTypes);
const skippedResourcesSql = skippedResourceIds.length ? `AND resources.id NOT IN ("${skippedResourceIds.join('","')}")` : '';
return await this.db().selectAll(`
${query.sql}
${skippedResourcesSql}
ORDER BY updated_time DESC
LIMIT ${limit}
`, query.params);
}
private static async resetOcrStatus(resourceId: string) {
await Resource.save({
id: resourceId,
ocr_error: '',
ocr_text: '',
ocr_status: ResourceOcrStatus.Todo,
});
}
public static serializeOcrDetails(details: RecognizeResultLine[]) {
if (!details || !details.length) return '';
return JSON.stringify(details);
}
public static unserializeOcrDetails(s: string): RecognizeResultLine[] | null {
if (!s) return null;
try {
const r = JSON.parse(s);
if (!r) return null;
if (!Array.isArray(r)) throw new Error('OCR details are not valid (not an array');
return r;
} catch (error) {
error.message = `Could not unserialized OCR data: ${error.message}`;
throw error;
}
}
public static async resourceOcrTextsByIds(ids: string[]): Promise<ResourceEntity[]> {
if (!ids.length) return [];
ids = unique(ids);
return this.modelSelectAll(`SELECT id, ocr_text FROM resources WHERE id IN ("${ids.join('","')}")`);
}
public static async allForNormalization(updatedTime: number, id: string, limit = 100, options: LoadOptions = null) {
const makeQuery = (useRowValue: boolean): SqlQuery => {
const whereSql = useRowValue ? '(updated_time, id) > (?, ?)' : 'updated_time > ?';
const params: any[] = [updatedTime];
if (useRowValue) {
params.push(id);
}
params.push(ResourceOcrStatus.Done);
params.push(limit);
return {
sql: `
SELECT ${this.selectFields(options)} FROM resources
WHERE ${whereSql}
AND ocr_text != ""
AND ocr_status = ?
ORDER BY updated_time ASC, id ASC
LIMIT ?
`,
params,
};
};
// We use a row value in this query, and that's not supported on certain
// Android devices (API level <= 24). So if the query fails, we fallback
// to a non-row value query. Although it may be inaccurate in some cases
// it wouldn't be a critical issue (some OCRed resources may not be part
// of the search engine results) and it means we can keep supporting old
// Android devices.
try {
const r = await this.modelSelectAll(makeQuery(true));
return r;
} catch (error) {
if (isSqliteSyntaxError(error)) {
const r = await this.modelSelectAll(makeQuery(false));
return r;
} else {
throw error;
}
}
}
public static async save(o: ResourceEntity, options: SaveOptions = null): Promise<ResourceEntity> {
const resource = { ...o };
const isNew = this.isNew(o, options);
if (isNew) {
const now = Date.now();
options = { ...options, autoTimestamp: false };
if (!resource.created_time) resource.created_time = now;
if (!resource.updated_time) resource.updated_time = now;
if (!resource.blob_updated_time) resource.blob_updated_time = now;
}
const output = await super.save(resource, options);
if (isNew) eventManager.emit(EventName.ResourceCreate);
return output;
}
}