1
0
mirror of https://github.com/immich-app/immich.git synced 2024-11-24 08:52:28 +02:00

feat(server): use embedded preview from raw images (#8773)

* extract embedded

* update api

* add tests

* move temp file logic outside of media repo

* formatting

* revert `toSorted`

* disable by default

* clarify setting description

* wording

* wording

* update docs

* check extracted image dimensions

* test that it unlinks

* formatting

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Mert 2024-04-19 11:50:13 -04:00 committed by GitHub
parent 74c921148b
commit 431ffebddd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 259 additions and 45 deletions

View File

@ -120,7 +120,8 @@ The default configuration looks like this:
"previewFormat": "jpeg",
"previewSize": 1440,
"quality": 80,
"colorspace": "p3"
"colorspace": "p3",
"extractEmbedded": false
},
"newVersionCheck": {
"enabled": true

Binary file not shown.

View File

@ -10531,6 +10531,9 @@
"colorspace": {
"$ref": "#/components/schemas/Colorspace"
},
"extractEmbedded": {
"type": "boolean"
},
"previewFormat": {
"$ref": "#/components/schemas/ImageFormat"
},
@ -10549,6 +10552,7 @@
},
"required": [
"colorspace",
"extractEmbedded",
"previewFormat",
"previewSize",
"quality",

View File

@ -864,6 +864,7 @@ export type SystemConfigFFmpegDto = {
};
export type SystemConfigImageDto = {
colorspace: Colorspace;
extractEmbedded: boolean;
previewFormat: ImageFormat;
previewSize: number;
quality: number;

View File

@ -1,3 +1,4 @@
import { randomUUID } from 'node:crypto';
import { dirname, join, resolve } from 'node:path';
import { APP_MEDIA_LOCATION } from 'src/constants';
import { SystemConfigCore } from 'src/cores/system-config.core';
@ -308,4 +309,8 @@ export class StorageCore {
static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
return join(this.getNestedFolder(folder, ownerId, filename), filename);
}
static getTempPathInDir(dir: string): string {
return join(dir, `${randomUUID()}.tmp`);
}
}

View File

@ -120,6 +120,7 @@ export const defaults = Object.freeze<SystemConfig>({
previewSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
extractEmbedded: false,
},
newVersionCheck: {
enabled: true,

View File

@ -417,6 +417,9 @@ class SystemConfigImageDto {
@IsEnum(Colorspace)
@ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
colorspace!: Colorspace;
@ValidateBoolean()
extractEmbedded!: boolean;
}
class SystemConfigTrashDto {

View File

@ -114,6 +114,7 @@ export const SystemConfigKey = {
IMAGE_PREVIEW_SIZE: 'image.previewSize',
IMAGE_QUALITY: 'image.quality',
IMAGE_COLORSPACE: 'image.colorspace',
IMAGE_EXTRACT_EMBEDDED: 'image.extractEmbedded',
TRASH_ENABLED: 'trash.enabled',
TRASH_DAYS: 'trash.days',
@ -284,6 +285,7 @@ export interface SystemConfig {
previewSize: number;
quality: number;
colorspace: Colorspace;
extractEmbedded: boolean;
};
newVersionCheck: {
enabled: boolean;

View File

@ -34,6 +34,11 @@ export interface VideoFormat {
bitrate: number;
}
export interface ImageDimensions {
width: number;
height: number;
}
export interface VideoInfo {
format: VideoFormat;
videoStreams: VideoStreamInfo[];
@ -70,9 +75,11 @@ export interface VideoCodecHWConfig extends VideoCodecSWConfig {
export interface IMediaRepository {
// image
extract(input: string, output: string): Promise<boolean>;
resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;
crop(input: string, options: CropOptions): Promise<Buffer>;
generateThumbhash(imagePath: string): Promise<Buffer>;
getImageDimensions(input: string): Promise<ImageDimensions>;
// video
probe(input: string): Promise<VideoInfo>;

View File

@ -1,4 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { exiftool } from 'exiftool-vendored';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import fs from 'node:fs/promises';
import { Writable } from 'node:stream';
@ -9,6 +10,7 @@ import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
CropOptions,
IMediaRepository,
ImageDimensions,
ResizeOptions,
TranscodeOptions,
VideoInfo,
@ -26,6 +28,23 @@ export class MediaRepository implements IMediaRepository {
constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
this.logger.setContext(MediaRepository.name);
}
async extract(input: string, output: string): Promise<boolean> {
try {
await exiftool.extractJpgFromRaw(input, output);
} catch (error: any) {
this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
try {
await exiftool.extractPreview(input, output);
} catch (error: any) {
this.logger.debug('Could not extract preview from image', error.message);
return false;
}
}
return true;
}
crop(input: string | Buffer, options: CropOptions): Promise<Buffer> {
return sharp(input, { failOn: 'none' })
.pipelineColorspace('rgb16')
@ -133,6 +152,11 @@ export class MediaRepository implements IMediaRepository {
return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
}
async getImageDimensions(input: string): Promise<ImageDimensions> {
const { width = 0, height = 0 } = await sharp(input).metadata();
return { width, height };
}
private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
return ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)
@ -140,9 +164,4 @@ export class MediaRepository implements IMediaRepository {
.output(output)
.on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
}
private chainPath(existing: string, path: string) {
const separator = existing.endsWith(':') ? '' : ':';
return `${existing}${separator}${path}`;
}
}

View File

@ -393,14 +393,12 @@ describe(MediaService.name, () => {
});
it('should generate a P3 thumbnail for a wide gamut image', async () => {
assetMock.getByIds.mockResolvedValue([
{ ...assetStub.image, exifInfo: { profileDescription: 'Adobe RGB', bitsPerSample: 14 } as ExifEntity },
]);
assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
await sut.handleGenerateThumbnail({ id: assetStub.image.id });
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
expect(mediaMock.resize).toHaveBeenCalledWith(
'/original/path.jpg',
assetStub.imageDng.originalPath,
'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
{
format: ImageFormat.WEBP,
@ -415,7 +413,96 @@ describe(MediaService.name, () => {
});
});
describe('handleGenerateThumbhashThumbnail', () => {
it('should extract embedded image if enabled and available', async () => {
mediaMock.extract.mockResolvedValue(true);
mediaMock.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
await sut.handleGenerateThumbnail({ id: assetStub.image.id });
const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
expect(mediaMock.resize.mock.calls).toEqual([
[
extractedPath,
'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
{
format: ImageFormat.WEBP,
size: 250,
quality: 80,
colorspace: Colorspace.P3,
},
],
]);
expect(extractedPath?.endsWith('.tmp')).toBe(true);
expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
});
it('should resize original image if embedded image is too small', async () => {
mediaMock.extract.mockResolvedValue(true);
mediaMock.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 });
configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
await sut.handleGenerateThumbnail({ id: assetStub.image.id });
expect(mediaMock.resize.mock.calls).toEqual([
[
assetStub.imageDng.originalPath,
'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
{
format: ImageFormat.WEBP,
size: 250,
quality: 80,
colorspace: Colorspace.P3,
},
],
]);
const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
expect(extractedPath?.endsWith('.tmp')).toBe(true);
expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
});
it('should resize original image if embedded image not found', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
await sut.handleGenerateThumbnail({ id: assetStub.image.id });
expect(mediaMock.resize).toHaveBeenCalledWith(
assetStub.imageDng.originalPath,
'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
{
format: ImageFormat.WEBP,
size: 250,
quality: 80,
colorspace: Colorspace.P3,
},
);
expect(mediaMock.getImageDimensions).not.toHaveBeenCalled();
});
it('should resize original image if embedded image extraction is not enabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: false }]);
assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
await sut.handleGenerateThumbnail({ id: assetStub.image.id });
expect(mediaMock.extract).not.toHaveBeenCalled();
expect(mediaMock.resize).toHaveBeenCalledWith(
assetStub.imageDng.originalPath,
'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
{
format: ImageFormat.WEBP,
size: 250,
quality: 80,
colorspace: Colorspace.P3,
},
);
expect(mediaMock.getImageDimensions).not.toHaveBeenCalled();
});
describe('handleGenerateThumbhash', () => {
it('should skip thumbhash generation if asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateThumbhash({ id: assetStub.image.id });

View File

@ -1,4 +1,5 @@
import { Inject, Injectable, UnsupportedMediaTypeException } from '@nestjs/common';
import { dirname } from 'node:path';
import { GeneratedImageType, StorageCore, StorageFolder } from 'src/cores/storage.core';
import { SystemConfigCore } from 'src/cores/system-config.core';
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
@ -42,6 +43,7 @@ import {
VAAPIConfig,
VP9Config,
} from 'src/utils/media';
import { mimeTypes } from 'src/utils/mime-types';
import { usePagination } from 'src/utils/pagination';
@Injectable()
@ -195,9 +197,21 @@ export class MediaService {
switch (asset.type) {
case AssetType.IMAGE: {
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
const imageOptions = { format, size, colorspace, quality: image.quality };
await this.mediaRepository.resize(asset.originalPath, path, imageOptions);
const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
const extractedPath = StorageCore.getTempPathInDir(dirname(path));
const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath));
try {
const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.previewSize));
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
const imageOptions = { format, size, colorspace, quality: image.quality };
await this.mediaRepository.resize(useExtracted ? extractedPath : asset.originalPath, path, imageOptions);
} finally {
if (didExtract) {
await this.storageRepository.unlink(extractedPath);
}
}
break;
}
@ -527,7 +541,7 @@ export class MediaService {
}
}
parseBitrateToBps(bitrateString: string) {
private parseBitrateToBps(bitrateString: string) {
const bitrateValue = Number.parseInt(bitrateString);
if (Number.isNaN(bitrateValue)) {
@ -542,4 +556,11 @@ export class MediaService {
return bitrateValue;
}
}
private async shouldUseExtractedImage(extractedPath: string, targetSize: number) {
const { width, height } = await this.mediaRepository.getImageDimensions(extractedPath);
const extractedSize = Math.min(width, height);
return extractedSize >= targetSize;
}
}

View File

@ -129,6 +129,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
previewSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
extractEmbedded: false,
},
newVersionCheck: {
enabled: true,

View File

@ -106,12 +106,6 @@ describe('mimeTypes', () => {
expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
});
it('should be a sorted list', () => {
const keys = Object.keys(mimeTypes.profile);
// TODO: use toSorted in NodeJS 20.
expect(keys).toEqual([...keys].sort());
});
for (const [extension, v] of Object.entries(mimeTypes.profile)) {
it(`should lookup ${extension}`, () => {
expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
@ -128,12 +122,6 @@ describe('mimeTypes', () => {
expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
});
it('should be a sorted list', () => {
const keys = Object.keys(mimeTypes.image);
// TODO: use toSorted in NodeJS 20.
expect(keys).toEqual([...keys].sort());
});
it('should contain only image mime types', () => {
const values = Object.values(mimeTypes.image).flat();
expect(values).toEqual(values.filter((mimeType) => mimeType.startsWith('image/')));
@ -157,7 +145,6 @@ describe('mimeTypes', () => {
it('should be a sorted list', () => {
const keys = Object.keys(mimeTypes.video);
// TODO: use toSorted in NodeJS 20.
expect(keys).toEqual([...keys].sort());
});
@ -184,7 +171,6 @@ describe('mimeTypes', () => {
it('should be a sorted list', () => {
const keys = Object.keys(mimeTypes.sidecar);
// TODO: use toSorted in NodeJS 20.
expect(keys).toEqual([...keys].sort());
});
@ -198,4 +184,20 @@ describe('mimeTypes', () => {
});
}
});
describe('raw', () => {
it('should contain only lowercase mime types', () => {
const keys = Object.keys(mimeTypes.raw);
expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
const values = Object.values(mimeTypes.raw).flat();
expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
});
for (const [extension, v] of Object.entries(mimeTypes.video)) {
it(`should lookup ${extension}`, () => {
expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
});
}
});
});

View File

@ -1,12 +1,10 @@
import { extname } from 'node:path';
import { AssetType } from 'src/entities/asset.entity';
const image: Record<string, string[]> = {
const raw: Record<string, string[]> = {
'.3fr': ['image/3fr', 'image/x-hasselblad-3fr'],
'.ari': ['image/ari', 'image/x-arriflex-ari'],
'.arw': ['image/arw', 'image/x-sony-arw'],
'.avif': ['image/avif'],
'.bmp': ['image/bmp'],
'.cap': ['image/cap', 'image/x-phaseone-cap'],
'.cin': ['image/cin', 'image/x-phantom-cin'],
'.cr2': ['image/cr2', 'image/x-canon-cr2'],
@ -16,16 +14,7 @@ const image: Record<string, string[]> = {
'.dng': ['image/dng', 'image/x-adobe-dng'],
'.erf': ['image/erf', 'image/x-epson-erf'],
'.fff': ['image/fff', 'image/x-hasselblad-fff'],
'.gif': ['image/gif'],
'.heic': ['image/heic'],
'.heif': ['image/heif'],
'.hif': ['image/hif'],
'.iiq': ['image/iiq', 'image/x-phaseone-iiq'],
'.insp': ['image/jpeg'],
'.jpe': ['image/jpeg'],
'.jpeg': ['image/jpeg'],
'.jpg': ['image/jpeg'],
'.jxl': ['image/jxl'],
'.k25': ['image/k25', 'image/x-kodak-k25'],
'.kdc': ['image/kdc', 'image/x-kodak-kdc'],
'.mrw': ['image/mrw', 'image/x-minolta-mrw'],
@ -33,7 +22,6 @@ const image: Record<string, string[]> = {
'.orf': ['image/orf', 'image/x-olympus-orf'],
'.ori': ['image/ori', 'image/x-olympus-ori'],
'.pef': ['image/pef', 'image/x-pentax-pef'],
'.png': ['image/png'],
'.psd': ['image/psd', 'image/vnd.adobe.photoshop'],
'.raf': ['image/raf', 'image/x-fuji-raf'],
'.raw': ['image/raw', 'image/x-panasonic-raw'],
@ -42,11 +30,27 @@ const image: Record<string, string[]> = {
'.sr2': ['image/sr2', 'image/x-sony-sr2'],
'.srf': ['image/srf', 'image/x-sony-srf'],
'.srw': ['image/srw', 'image/x-samsung-srw'],
'.x3f': ['image/x3f', 'image/x-sigma-x3f'],
};
const image: Record<string, string[]> = {
...raw,
'.avif': ['image/avif'],
'.bmp': ['image/bmp'],
'.gif': ['image/gif'],
'.heic': ['image/heic'],
'.heif': ['image/heif'],
'.hif': ['image/hif'],
'.insp': ['image/jpeg'],
'.jpe': ['image/jpeg'],
'.jpeg': ['image/jpeg'],
'.jpg': ['image/jpeg'],
'.jxl': ['image/jxl'],
'.png': ['image/png'],
'.svg': ['image/svg'],
'.tif': ['image/tiff'],
'.tiff': ['image/tiff'],
'.webp': ['image/webp'],
'.x3f': ['image/x3f', 'image/x-sigma-x3f'],
};
const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']);
@ -77,22 +81,25 @@ const sidecar: Record<string, string[]> = {
'.xmp': ['application/xml', 'text/xml'],
};
const types = { ...image, ...video, ...sidecar };
const isType = (filename: string, r: Record<string, string[]>) => extname(filename).toLowerCase() in r;
const lookup = (filename: string) =>
({ ...image, ...video, ...sidecar })[extname(filename).toLowerCase()]?.[0] ?? 'application/octet-stream';
const lookup = (filename: string) => types[extname(filename).toLowerCase()]?.[0] ?? 'application/octet-stream';
export const mimeTypes = {
image,
profile,
sidecar,
video,
raw,
isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
isImage: (filename: string) => isType(filename, image),
isProfile: (filename: string) => isType(filename, profile),
isSidecar: (filename: string) => isType(filename, sidecar),
isVideo: (filename: string) => isType(filename, video),
isRaw: (filename: string) => isType(filename, raw),
lookup,
assetType: (filename: string) => {
const contentType = lookup(filename);

View File

@ -757,4 +757,45 @@ export const assetStub = {
fileSizeInByte: 5000,
} as ExifEntity,
}),
imageDng: Object.freeze<AssetEntity>({
id: 'asset-id',
deviceAssetId: 'device-asset-id',
fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
owner: userStub.user1,
ownerId: 'user-id',
deviceId: 'device-id',
originalPath: '/original/path.dng',
previewPath: '/uploads/user-id/thumbs/path.jpg',
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.IMAGE,
thumbnailPath: '/uploads/user-id/webp/path.ext',
thumbhash: Buffer.from('blablabla', 'base64'),
encodedVideoPath: null,
createdAt: new Date('2023-02-23T05:06:29.716Z'),
updatedAt: new Date('2023-02-23T05:06:29.716Z'),
localDateTime: new Date('2023-02-23T05:06:29.716Z'),
isFavorite: true,
isArchived: false,
isReadOnly: false,
duration: null,
isVisible: true,
isExternal: false,
livePhotoVideo: null,
livePhotoVideoId: null,
isOffline: false,
libraryId: 'library-id',
library: libraryStub.uploadLibrary1,
tags: [],
sharedLinks: [],
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
profileDescription: 'Adobe RGB',
bitsPerSample: 14,
} as ExifEntity,
}),
};

View File

@ -4,9 +4,11 @@ import { Mocked, vitest } from 'vitest';
export const newMediaRepositoryMock = (): Mocked<IMediaRepository> => {
return {
generateThumbhash: vitest.fn(),
extract: vitest.fn().mockResolvedValue(false),
resize: vitest.fn(),
crop: vitest.fn(),
probe: vitest.fn(),
transcode: vitest.fn(),
getImageDimensions: vitest.fn(),
};
};

View File

@ -101,6 +101,16 @@
isEdited={config.image.colorspace !== savedConfig.image.colorspace}
{disabled}
/>
<SettingSwitch
id="prefer-embedded"
title="PREFER EMBEDDED PREVIEW"
subtitle="Use embedded previews in RAW photos as the input to image processing when available. This can produce more accurate colors for some images, but the quality of the preview is camera-dependent and the image may have more compression artifacts."
checked={config.image.extractEmbedded}
on:toggle={() => (config.image.extractEmbedded = !config.image.extractEmbedded)}
isEdited={config.image.extractEmbedded !== savedConfig.image.extractEmbedded}
{disabled}
/>
</div>
<div class="ml-4">