From 41c2c8b82d1746a5b63e67223094033331d6b9b6 Mon Sep 17 00:00:00 2001 From: Thomas <9749173+uhthomas@users.noreply.github.com> Date: Thu, 15 Jun 2023 03:34:03 +0100 Subject: [PATCH] use imagemagick and libraw for raw image support (#2668) * use imagemagick and libraw for raw image support imagemagick and libraw have generally good support for raw images, including Sony's ARW format. These tools should also allow Immich to support many more image formats in future without any major code changes. https://www.libraw.org/supported-cameras I've tested and verified this change with .ARW files and other standard formats. Fixes: #2156 * Add additional type for awr * pr feedback --------- Co-authored-by: Alex Tran --- mobile/lib/utils/files_helper.dart | 3 + server/Dockerfile | 4 +- server/package-lock.json | 2 +- server/package.json | 2 +- server/src/domain/media/media.service.ts | 4 +- .../immich/config/asset-upload.config.spec.ts | 102 ++++++------------ .../src/immich/config/asset-upload.config.ts | 2 +- web/src/lib/utils/asset-utils.spec.ts | 49 ++++++++- web/src/lib/utils/asset-utils.ts | 42 +++----- web/src/lib/utils/file-uploader.ts | 2 +- 10 files changed, 104 insertions(+), 108 deletions(-) diff --git a/mobile/lib/utils/files_helper.dart b/mobile/lib/utils/files_helper.dart index 81fae9cee0..5c1cb13a43 100644 --- a/mobile/lib/utils/files_helper.dart +++ b/mobile/lib/utils/files_helper.dart @@ -53,6 +53,9 @@ class FileHelper { case 'insv': return {"type": "video", "subType": "mp4"}; + case 'arw': + return {"type": "image", "subType": "x-sony-arw"}; + default: return {"type": "unsupport", "subType": "unsupport"}; } diff --git a/server/Dockerfile b/server/Dockerfile index 0ff2729b36..c8b807993f 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -2,7 +2,7 @@ FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb849 WORKDIR /usr/src/app -RUN apk add --update-cache build-base python3 vips-heif vips-dev ffmpeg perl +RUN apk add --update-cache build-base imagemagick-dev python3 ffmpeg libraw-dev perl vips-dev vips-heif vips-magick COPY package.json package-lock.json ./ @@ -23,7 +23,7 @@ ENV NODE_ENV=production WORKDIR /usr/src/app -RUN apk add --no-cache vips-heif vips vips-cpp ffmpeg perl +RUN apk add --no-cache ffmpeg imagemagick-dev libraw-dev perl vips-dev vips-heif vips-magick COPY --from=prod /usr/src/app/node_modules ./node_modules COPY --from=prod /usr/src/app/dist ./dist diff --git a/server/package-lock.json b/server/package-lock.json index c1e519fced..1e9f386492 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -45,7 +45,7 @@ "reflect-metadata": "^0.1.13", "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3", - "sharp": "^0.31.0", + "sharp": "^0.31.3", "typeorm": "^0.3.11", "typesense": "^1.5.3", "ua-parser-js": "^1.0.35" diff --git a/server/package.json b/server/package.json index dd2497c53f..eccf501baf 100644 --- a/server/package.json +++ b/server/package.json @@ -74,7 +74,7 @@ "reflect-metadata": "^0.1.13", "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3", - "sharp": "^0.31.0", + "sharp": "^0.31.3", "typeorm": "^0.3.11", "typesense": "^1.5.3", "ua-parser-js": "^1.0.35" diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 5321631ab6..daad0d9c6b 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -60,9 +60,9 @@ export class MediaService { size: JPEG_THUMBNAIL_SIZE, format: 'jpeg', }); - } catch (error) { + } catch (error: any) { this.logger.warn( - `Failed to generate jpeg thumbnail using sharp, trying with exiftool-vendored (asset=${asset.id})`, + `Failed to generate jpeg thumbnail using sharp, trying with exiftool-vendored (asset=${asset.id}): ${error.message}`, ); await this.mediaRepository.extractThumbnailFromExif(asset.originalPath, jpegThumbnailPath); } diff --git a/server/src/immich/config/asset-upload.config.spec.ts b/server/src/immich/config/asset-upload.config.spec.ts index a2e6b261a4..570c89089d 100644 --- a/server/src/immich/config/asset-upload.config.spec.ts +++ b/server/src/immich/config/asset-upload.config.spec.ts @@ -49,77 +49,37 @@ describe('assetUploadOption', () => { expect(name).toBeUndefined(); }); - it('should allow images', async () => { - const file = { mimetype: 'image/jpeg', originalname: 'test.jpg' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow videos', async () => { - const file = { mimetype: 'video/mp4', originalname: 'test.mp4' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow webm videos', async () => { - const file = { mimetype: 'video/webm', originalname: 'test.webm' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .raf recognized', () => { - const file = { mimetype: 'image/x-fuji-raf', originalname: 'test.raf' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .srw recognized', () => { - const file = { mimetype: 'image/x-samsung-srw', originalname: 'test.srw' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .wmv videos', () => { - const file = { mimetype: 'video/x-ms-wmv', originalname: 'test.wmv' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .mkv videos', () => { - const file = { mimetype: 'video/x-matroska', originalname: 'test.mkv' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .mpg videos', () => { - const file = { mimetype: 'video/mpeg', originalname: 'test.mpg' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .flv videos', () => { - const file = { mimetype: 'video/x-flv', originalname: 'test.flv' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .mov videos with video/mov mimetype', () => { - const file = { mimetype: 'video/mov', originalname: 'test.mov' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .avi videos with video/avi mimetype', () => { - const file = { mimetype: 'video/avi', originalname: 'test.avi' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); - - it('should allow .avi videos with video/x-msvideo mimetype', () => { - const file = { mimetype: 'video/x-msvideo', originalname: 'test.avi' } as any; - fileFilter(mock.userRequest, file, callback); - expect(callback).toHaveBeenCalledWith(null, true); - }); + for (const { mimetype, extension } of [ + { mimetype: 'image/dng', extension: 'dng' }, + { mimetype: 'image/gif', extension: 'gif' }, + { mimetype: 'image/heic', extension: 'heic' }, + { mimetype: 'image/heif', extension: 'heif' }, + { mimetype: 'image/jpeg', extension: 'jpeg' }, + { mimetype: 'image/jpeg', extension: 'jpg' }, + { mimetype: 'image/png', extension: 'png' }, + { mimetype: 'image/tiff', extension: 'tiff' }, + { mimetype: 'image/webp', extension: 'webp' }, + { mimetype: 'image/x-adobe-dng', extension: 'dng' }, + { mimetype: 'image/x-fuji-raf', extension: 'raf' }, + { mimetype: 'image/x-nikon-nef', extension: 'nef' }, + { mimetype: 'image/x-samsung-srw', extension: 'srw' }, + { mimetype: 'image/x-sony-arw', extension: 'arw' }, + { mimetype: 'video/avi', extension: 'avi' }, + { mimetype: 'video/mov', extension: 'mov' }, + { mimetype: 'video/mp4', extension: 'mp4' }, + { mimetype: 'video/mpeg', extension: 'mpg' }, + { mimetype: 'video/webm', extension: 'webm' }, + { mimetype: 'video/x-flv', extension: 'flv' }, + { mimetype: 'video/x-matroska', extension: 'mkv' }, + { mimetype: 'video/x-ms-wmv', extension: 'wmv' }, + { mimetype: 'video/x-msvideo', extension: 'avi' }, + ]) { + const name = `test.${extension}`; + it(`should allow ${name} (${mimetype})`, async () => { + fileFilter(mock.userRequest, { mimetype, originalname: name }, callback); + expect(callback).toHaveBeenCalledWith(null, true); + }); + } it('should not allow unknown types', async () => { const file = { mimetype: 'application/html', originalname: 'test.html' } as any; diff --git a/server/src/immich/config/asset-upload.config.ts b/server/src/immich/config/asset-upload.config.ts index 0a1d615130..c640d15b77 100644 --- a/server/src/immich/config/asset-upload.config.ts +++ b/server/src/immich/config/asset-upload.config.ts @@ -55,7 +55,7 @@ function fileFilter(req: AuthRequest, file: any, cb: any) { } if ( file.mimetype.match( - /\/(jpg|jpeg|png|gif|avi|mov|mp4|webm|x-msvideo|quicktime|heic|heif|dng|x-adobe-dng|webp|tiff|3gpp|nef|x-nikon-nef|x-fuji-raf|x-samsung-srw|mpeg|x-flv|x-ms-wmv|x-matroska)$/, + /\/(jpg|jpeg|png|gif|avi|mov|mp4|webm|x-msvideo|quicktime|heic|heif|dng|x-adobe-dng|webp|tiff|3gpp|nef|x-nikon-nef|x-fuji-raf|x-samsung-srw|mpeg|x-flv|x-ms-wmv|x-matroska|x-sony-arw|arw)$/, ) ) { cb(null, true); diff --git a/web/src/lib/utils/asset-utils.spec.ts b/web/src/lib/utils/asset-utils.spec.ts index e11f67f051..ebd82eb8b9 100644 --- a/web/src/lib/utils/asset-utils.spec.ts +++ b/web/src/lib/utils/asset-utils.spec.ts @@ -1,6 +1,6 @@ import type { AssetResponseDto } from '@api'; import { describe, expect, it } from '@jest/globals'; -import { getAssetFilename, getFilenameExtension } from './asset-utils'; +import { getAssetFilename, getFilenameExtension, getFileMimeType } from './asset-utils'; describe('get file extension from filename', () => { it('returns the extension without including the dot', () => { @@ -58,3 +58,50 @@ describe('get asset filename', () => { }); }); }); + +describe('get file mime type', () => { + for (const { extension, mimeType } of [ + { extension: '3gp', mimeType: 'video/3gpp' }, + { extension: 'arw', mimeType: 'image/x-sony-arw' }, + { extension: 'dng', mimeType: 'image/dng' }, + { extension: 'heic', mimeType: 'image/heic' }, + { extension: 'heif', mimeType: 'image/heif' }, + { extension: 'insp', mimeType: 'image/jpeg' }, + { extension: 'insv', mimeType: 'video/mp4' }, + { extension: 'nef', mimeType: 'image/x-nikon-nef' }, + { extension: 'raf', mimeType: 'image/x-fuji-raf' }, + { extension: 'srw', mimeType: 'image/x-samsung-srw' } + ]) { + it(`returns the mime type for ${extension}`, () => { + expect(getFileMimeType({ name: `filename.${extension}` } as File)).toEqual(mimeType); + }); + } + + it('returns the mime type from the file', () => { + [ + { + file: { + name: 'filename.jpg', + type: 'image/jpeg' + }, + result: 'image/jpeg' + }, + { + file: { + name: 'filename.txt', + type: 'text/plain' + }, + result: 'text/plain' + }, + { + file: { + name: 'filename.txt', + type: '' + }, + result: '' + } + ].forEach(({ file, result }) => { + expect(getFileMimeType(file as File)).toEqual(result); + }); + }); +}); diff --git a/web/src/lib/utils/asset-utils.ts b/web/src/lib/utils/asset-utils.ts index c12fe7b441..c471f73c9b 100644 --- a/web/src/lib/utils/asset-utils.ts +++ b/web/src/lib/utils/asset-utils.ts @@ -125,34 +125,20 @@ export function getAssetFilename(asset: AssetResponseDto): string { * Returns the MIME type of the file and an empty string when not found. */ export function getFileMimeType(file: File): string { - if (file.type !== '') { - // Return the MIME type determined by the browser. - return file.type; - } - - // Return MIME type based on the file extension. - switch (getFilenameExtension(file.name)) { - case 'heic': - return 'image/heic'; - case 'heif': - return 'image/heif'; - case 'dng': - return 'image/dng'; - case '3gp': - return 'video/3gpp'; - case 'nef': - return 'image/x-nikon-nef'; - case 'raf': - return 'image/x-fuji-raf'; - case 'srw': - return 'image/x-samsung-srw'; - case 'insp': - return 'image/jpeg'; - case 'insv': - return 'video/mp4'; - default: - return ''; - } + const mimeTypes: Record = { + '3gp': 'video/3gpp', + arw: 'image/x-sony-arw', + dng: 'image/dng', + heic: 'image/heic', + heif: 'image/heif', + insp: 'image/jpeg', + insv: 'video/mp4', + nef: 'image/x-nikon-nef', + raf: 'image/x-fuji-raf', + srw: 'image/x-samsung-srw' + }; + // Return the MIME type determined by the browser or the MIME type based on the file extension. + return file.type || (mimeTypes[getFilenameExtension(file.name)] ?? ''); } /** diff --git a/web/src/lib/utils/file-uploader.ts b/web/src/lib/utils/file-uploader.ts index a97c6f85bb..cc48c974f7 100644 --- a/web/src/lib/utils/file-uploader.ts +++ b/web/src/lib/utils/file-uploader.ts @@ -22,7 +22,7 @@ export const openFileUploadDialog = async ( // When adding a content type that is unsupported by browsers, make sure // to also add it to getFileMimeType() otherwise the upload will fail. - fileSelector.accept = 'image/*,video/*,.heic,.heif,.dng,.3gp,.nef,.srw,.raf,.insp,.insv'; + fileSelector.accept = 'image/*,video/*,.heic,.heif,.dng,.3gp,.nef,.srw,.raf,.insp,.insv,.arw'; fileSelector.onchange = async (e: Event) => { const target = e.target as HTMLInputElement;