self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

refactor(server): make storage core singleton (#4608)

Browse Source
This commit is contained in:
Daniel Dietzler 2023-10-23 17:52:21 +02:00 committed by GitHub
parent 2288b022bc
commit 6b25435b4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 80 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
server/src/domain/asset/asset.service.spec.ts
View File

@ -10,8 +10,6 @@ import {
newCommunicationRepositoryMock,
newCryptoRepositoryMock,
newJobRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
@ -25,8 +23,6 @@ import {
ICommunicationRepository,
ICryptoRepository,
IJobRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
JobItem,
@ -165,8 +161,6 @@ describe(AssetService.name, () => {
let assetMock: jest.Mocked<IAssetRepository>;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
@ -181,21 +175,9 @@ describe(AssetService.name, () => {
communicationMock = newCommunicationRepositoryMock();
cryptoMock = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
configMock = newSystemConfigRepositoryMock();
sut = new AssetService(
accessMock,
assetMock,
cryptoMock,
jobMock,
configMock,
moveMock,
personMock,
storageMock,
communicationMock,
);
sut = new AssetService(accessMock, assetMock, cryptoMock, jobMock, configMock, storageMock, communicationMock);
when(assetMock.getById)
.calledWith(assetStub.livePhotoStillAsset.id)

10
server/src/domain/asset/asset.service.ts
View File

@ -16,8 +16,6 @@ import {
ICommunicationRepository,
ICryptoRepository,
IJobRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
@ -76,7 +74,6 @@ export class AssetService {
private logger = new Logger(AssetService.name);
private access: AccessCore;
private configCore: SystemConfigCore;
private storageCore: StorageCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@ -84,14 +81,11 @@ export class AssetService {
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
}
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {
@ -147,9 +141,9 @@ export class AssetService {
getUploadFolder({ authUser, fieldName }: UploadRequest): string {
authUser = this.access.requireUploadAccess(authUser);
let folder = this.storageCore.getFolderLocation(StorageFolder.UPLOAD, authUser.id);
let folder = StorageCore.getFolderLocation(StorageFolder.UPLOAD, authUser.id);
if (fieldName === UploadFieldName.PROFILE_DATA) {
folder = this.storageCore.getFolderLocation(StorageFolder.PROFILE, authUser.id);
folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, authUser.id);
}
this.storageRepository.mkdirSync(folder);

6
server/src/domain/media/media.service.ts
View File

@ -44,7 +44,7 @@ export class MediaService {
@Inject(IMoveRepository) moveRepository: IMoveRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(this.storageRepository, assetRepository, moveRepository, personRepository);
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
}
async handleQueueGenerateThumbnails({ force }: IBaseJob) {
@ -140,7 +140,7 @@ export class MediaService {
const { thumbnail, ffmpeg } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const path =
format === 'jpeg' ? this.storageCore.getLargeThumbnailPath(asset) : this.storageCore.getSmallThumbnailPath(asset);
format === 'jpeg' ? StorageCore.getLargeThumbnailPath(asset) : StorageCore.getSmallThumbnailPath(asset);
this.storageCore.ensureFolders(path);
switch (asset.type) {
@ -220,7 +220,7 @@ export class MediaService {
}
const input = asset.originalPath;
const output = this.storageCore.getEncodedVideoPath(asset);
const output = StorageCore.getEncodedVideoPath(asset);
this.storageCore.ensureFolders(output);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);

4
server/src/domain/metadata/metadata.service.ts
View File

@ -80,7 +80,7 @@ export class MetadataService {
@Inject(IPersonRepository) personRepository: IPersonRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
this.configCore.config$.subscribe(() => this.init());
}
@ -294,7 +294,7 @@ export class MetadataService {
});
const checksum = this.cryptoRepository.hashSha1(video);
const motionPath = this.storageCore.getAndroidMotionPath(asset);
const motionPath = StorageCore.getAndroidMotionPath(asset);
this.storageCore.ensureFolders(motionPath);
let motionAsset = await this.assetRepository.getByChecksum(asset.ownerId, checksum);

4
server/src/domain/person/person.service.ts
View File

@ -58,7 +58,7 @@ export class PersonService {
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, repository);
this.storageCore = StorageCore.create(assetRepository, moveRepository, repository, storageRepository);
}
async getAll(authUser: AuthUserDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
@ -309,7 +309,7 @@ export class PersonService {
}
this.logger.verbose(`Cropping face for person: ${personId}`);
const thumbnailPath = this.storageCore.getPersonThumbnailPath(person);
const thumbnailPath = StorageCore.getPersonThumbnailPath(person);
this.storageCore.ensureFolders(thumbnailPath);
const halfWidth = (x2 - x1) / 2;

6
server/src/domain/storage-template/storage-template.service.ts
View File

@ -52,7 +52,7 @@ export class StorageTemplateService {
this.configCore = SystemConfigCore.create(configRepository);
this.configCore.addValidator((config) => this.validate(config));
this.configCore.config$.subscribe((config) => this.onConfig(config));
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
}
async handleMigrationSingle({ id }: IEntityJob) {
@ -99,7 +99,7 @@ export class StorageTemplateService {
}
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
if (asset.isReadOnly || asset.isExternal || this.storageCore.isAndroidMotionPath(asset.originalPath)) {
if (asset.isReadOnly || asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) {
// External assets are not affected by storage template
// TODO: shouldn't this only apply to external assets?
return;
@ -131,7 +131,7 @@ export class StorageTemplateService {
const source = asset.originalPath;
const ext = path.extname(source).split('.').pop() as string;
const sanitized = sanitize(path.basename(filename, `.${ext}`));
const rootPath = this.storageCore.getLibraryFolder({ id: asset.ownerId, storageLabel });
const rootPath = StorageCore.getLibraryFolder({ id: asset.ownerId, storageLabel });
const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
const fullPath = path.normalize(path.join(rootPath, storagePath));
let destination = `${fullPath}.${ext}`;

76
server/src/domain/storage/storage.core.ts
View File

@ -21,21 +21,40 @@ export interface MoveRequest {
type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO;
let instance: StorageCore | null;
export class StorageCore {
private logger = new Logger(StorageCore.name);
constructor(
private repository: IStorageRepository,
private constructor(
private assetRepository: IAssetRepository,
private moveRepository: IMoveRepository,
private personRepository: IPersonRepository,
private repository: IStorageRepository,
) {}
getFolderLocation(folder: StorageFolder, userId: string) {
static create(
assetRepository: IAssetRepository,
moveRepository: IMoveRepository,
personRepository: IPersonRepository,
repository: IStorageRepository,
) {
if (!instance) {
instance = new StorageCore(assetRepository, moveRepository, personRepository, repository);
}
return instance;
}
static reset() {
instance = null;
}
static getFolderLocation(folder: StorageFolder, userId: string) {
return join(StorageCore.getBaseFolder(folder), userId);
}
getLibraryFolder(user: { storageLabel: string | null; id: string }) {
static getLibraryFolder(user: { storageLabel: string | null; id: string }) {
return join(StorageCore.getBaseFolder(StorageFolder.LIBRARY), user.storageLabel || user.id);
}
@ -43,27 +62,27 @@ export class StorageCore {
return join(APP_MEDIA_LOCATION, folder);
}
getPersonThumbnailPath(person: PersonEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`);
static getPersonThumbnailPath(person: PersonEntity) {
return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`);
}
getLargeThumbnailPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.jpeg`);
static getLargeThumbnailPath(asset: AssetEntity) {
return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.jpeg`);
}
getSmallThumbnailPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.webp`);
static getSmallThumbnailPath(asset: AssetEntity) {
return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.webp`);
}
getEncodedVideoPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`);
static getEncodedVideoPath(asset: AssetEntity) {
return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`);
}
getAndroidMotionPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`);
static getAndroidMotionPath(asset: AssetEntity) {
return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`);
}
isAndroidMotionPath(originalPath: string) {
static isAndroidMotionPath(originalPath: string) {
return originalPath.startsWith(StorageCore.getBaseFolder(StorageFolder.ENCODED_VIDEO));
}
@ -75,15 +94,25 @@ export class StorageCore {
const { id: entityId, resizePath, webpPath, encodedVideoPath } = asset;
switch (pathType) {
case AssetPathType.JPEG_THUMBNAIL:
return this.moveFile({ entityId, pathType, oldPath: resizePath, newPath: this.getLargeThumbnailPath(asset) });
return this.moveFile({
entityId,
pathType,
oldPath: resizePath,
newPath: StorageCore.getLargeThumbnailPath(asset),
});
case AssetPathType.WEBP_THUMBNAIL:
return this.moveFile({ entityId, pathType, oldPath: webpPath, newPath: this.getSmallThumbnailPath(asset) });
return this.moveFile({
entityId,
pathType,
oldPath: webpPath,
newPath: StorageCore.getSmallThumbnailPath(asset),
});
case AssetPathType.ENCODED_VIDEO:
return this.moveFile({
entityId,
pathType,
oldPath: encodedVideoPath,
newPath: this.getEncodedVideoPath(asset),
newPath: StorageCore.getEncodedVideoPath(asset),
});
}
}
@ -96,7 +125,7 @@ export class StorageCore {
entityId,
pathType,
oldPath: thumbnailPath,
newPath: this.getPersonThumbnailPath(person),
newPath: StorageCore.getPersonThumbnailPath(person),
});
}
}
@ -159,7 +188,12 @@ export class StorageCore {
}
}
private getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
return join(this.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4), filename);
private static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
return join(
StorageCore.getFolderLocation(folder, ownerId),
filename.substring(0, 2),
filename.substring(2, 4),
filename,
);
}
}

20
server/src/domain/user/user.service.spec.ts
View File

@ -11,8 +11,6 @@ import {
newCryptoRepositoryMock,
newJobRepositoryMock,
newLibraryRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newUserRepositoryMock,
userStub,
@ -26,8 +24,6 @@ import {
ICryptoRepository,
IJobRepository,
ILibraryRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
IUserRepository,
} from '../repositories';
@ -139,8 +135,6 @@ describe(UserService.name, () => {
let assetMock: jest.Mocked<IAssetRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let libraryMock: jest.Mocked<ILibraryRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
@ -149,22 +143,10 @@ describe(UserService.name, () => {
cryptoRepositoryMock = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
libraryMock = newLibraryRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
userMock = newUserRepositoryMock();
sut = new UserService(
albumMock,
assetMock,
cryptoRepositoryMock,
jobMock,
libraryMock,
moveMock,
personMock,
storageMock,
userMock,
);
sut = new UserService(albumMock, assetMock, cryptoRepositoryMock, jobMock, libraryMock, storageMock, userMock);
when(userMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
when(userMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);

16
server/src/domain/user/user.service.ts
View File

@ -10,8 +10,6 @@ import {
ICryptoRepository,
IJobRepository,
ILibraryRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
IUserRepository,
} from '../repositories';
@ -30,7 +28,6 @@ import { UserCore } from './user.core';
@Injectable()
export class UserService {
private logger = new Logger(UserService.name);
private storageCore: StorageCore;
private userCore: UserCore;
constructor(
@ -39,12 +36,9 @@ export class UserService {
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
) {
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
this.userCore = UserCore.create(cryptoRepository, libraryRepository, userRepository);
}
@ -171,11 +165,11 @@ export class UserService {
this.logger.log(`Deleting user: ${user.id}`);
const folders = [
this.storageCore.getLibraryFolder(user),
this.storageCore.getFolderLocation(StorageFolder.UPLOAD, user.id),
this.storageCore.getFolderLocation(StorageFolder.PROFILE, user.id),
this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id),
this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id),
StorageCore.getLibraryFolder(user),
StorageCore.getFolderLocation(StorageFolder.UPLOAD, user.id),
StorageCore.getFolderLocation(StorageFolder.PROFILE, user.id),
StorageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id),
StorageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id),
];
for (const folder of folders) {

8
server/test/repositories/storage.repository.mock.ts
View File

@ -1,6 +1,10 @@
import { IStorageRepository } from '@app/domain';
import { IStorageRepository, StorageCore } from '@app/domain';
export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepository> => {
if (reset) {
StorageCore.reset();
}
export const newStorageRepositoryMock = (): jest.Mocked<IStorageRepository> => {
return {
createZipStream: jest.fn(),
createReadStream: jest.fn(),