@@ -44,9 +44,10 @@ Loading ~4000 images/videos
-### Web client -- +### Web +
+ +
# Note @@ -55,26 +56,22 @@ Loading ~4000 images/videos This project is under heavy development, there will be continuous functions, features and api changes. -# Features +# Features + +| | Mobile | Web | +| - | - | - | +| Upload and view videos and photos | Yes | Yes +| Auto backup when app is opened | Yes | N/A +| Selective album(s) for backup | Yes | N/A +| Download photos and videos to local device | Yes | Yes +| Multi-user support | Yes | Yes +| Shared Albums | Yes | No +| Quick navigation with draggable scrollbar | Yes | Yes +| Support RAW (HEIC, HEIF, DNG, Apple ProRaw) | Yes | Yes +| Metadata view (EXIF, map) | Yes | Yes +| Search by metadata, objects and image tags | Yes | No +| Administrative functions (user management) | No | Yes -- Upload and view assets (videos/images). -- Auto Backup. -- Download asset to local device. -- Multi-user supported. -- Quick navigation with drag scroll bar. -- Support HEIC/HEIF Backup. -- Extract and display EXIF info. -- Real-time render from multi-device upload event. -- Image Tagging/Classification based on ImageNet dataset -- Object detection based on COCO SSD. -- Search assets based on tags and exif data (lens, make, model, orientation) -- [Optional] Reverse geocoding using Mapbox (Generous free-tier of 100,000 search/month) -- Show asset's location information on map (OpenStreetMap). -- Show curated places on the search page -- Show curated objects on the search page -- Shared album with users on the same server -- Selective backup - albums can be included and excluded during the backup process. -- Web interface is available for administrative tasks (creating new users) and viewing assets on the server - additional features are coming. # System Requirement @@ -97,7 +94,7 @@ You can use docker compose for development and testing out the application, ther 3. **PostgreSQL** - Main database of the application 4. **Redis** - For sharing websocket instance between docker instances and background tasks message queue. 5. **Nginx** - Load balancing and optimized file uploading. -6. **TensorFlow** - Object Detection and Image Classification. +6. **TensorFlow** - Object Detection (COCO SSD) and Image Classification (ImageNet). ## Step 1: Populate .env file diff --git a/design/dashboard_photos.jpeg b/design/dashboard_photos.jpeg deleted file mode 100644 index c3f47c12de..0000000000 Binary files a/design/dashboard_photos.jpeg and /dev/null differ diff --git a/design/web-admin.jpeg b/design/web-admin.jpeg new file mode 100644 index 0000000000..d5d819ae31 Binary files /dev/null and b/design/web-admin.jpeg differ diff --git a/design/web-detail.jpeg b/design/web-detail.jpeg new file mode 100644 index 0000000000..e6c7c6d2e8 Binary files /dev/null and b/design/web-detail.jpeg differ diff --git a/design/web-home.jpeg b/design/web-home.jpeg new file mode 100644 index 0000000000..f9d16c175f Binary files /dev/null and b/design/web-home.jpeg differ diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 1b763ed6c4..34635f6971 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -60,6 +60,7 @@ services: - NODE_ENV=development depends_on: - database + - immich-server networks: - immich-network diff --git a/server/apps/immich/src/api-v1/asset/asset.controller.ts b/server/apps/immich/src/api-v1/asset/asset.controller.ts index 75331f6d0a..e247d778d8 100644 --- a/server/apps/immich/src/api-v1/asset/asset.controller.ts +++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts @@ -15,6 +15,7 @@ import { Delete, Logger, Patch, + HttpCode, } from '@nestjs/common'; import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard'; import { AssetService } from './asset.service'; @@ -76,6 +77,10 @@ export class AssetController { { asset: assetWithThumbnail, fileName: file.originalname, fileSize: file.size, hasThumbnail: true }, { jobId: savedAsset.id }, ); + + this.wsCommunicateionGateway.server + .to(savedAsset.userId) + .emit('on_upload_success', JSON.stringify(assetWithThumbnail)); } else { await this.assetUploadedQueue.add( 'asset-uploaded', @@ -83,8 +88,6 @@ export class AssetController { { jobId: savedAsset.id }, ); } - - this.wsCommunicateionGateway.server.to(savedAsset.userId).emit('on_upload_success', JSON.stringify(savedAsset)); } catch (e) { Logger.error(`Error receiving upload file ${e}`); } @@ -171,4 +174,20 @@ export class AssetController { return result; } + + /** + * Check duplicated asset before uploading - for Web upload used + */ + @Post('/check') + @HttpCode(200) + async checkDuplicateAsset( + @GetAuthUser() authUser: AuthUserDto, + @Body(ValidationPipe) { deviceAssetId }: { deviceAssetId: string }, + ) { + const res = await this.assetService.checkDuplicatedAsset(authUser, deviceAssetId); + + return { + isExist: res, + }; + } } diff --git a/server/apps/immich/src/api-v1/asset/asset.module.ts b/server/apps/immich/src/api-v1/asset/asset.module.ts index bb9e0823e0..b8ef03b08d 100644 --- a/server/apps/immich/src/api-v1/asset/asset.module.ts +++ b/server/apps/immich/src/api-v1/asset/asset.module.ts @@ -24,6 +24,6 @@ import { CommunicationModule } from '../communication/communication.module'; ], controllers: [AssetController], providers: [AssetService, BackgroundTaskService], - exports: [], + exports: [AssetService], }) export class AssetModule {} diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index 3221c64bc5..5e7f761738 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -1,6 +1,6 @@ import { BadRequestException, Injectable, InternalServerErrorException, Logger, StreamableFile } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { IsNull, Not, Repository } from 'typeorm'; import { AuthUserDto } from '../../decorators/auth-user.decorator'; import { CreateAssetDto } from './dto/create-asset.dto'; import { AssetEntity, AssetType } from '@app/database/entities/asset.entity'; @@ -72,6 +72,7 @@ export class AssetService { return await this.assetRepository.find({ where: { userId: authUser.id, + resizePath: Not(IsNull()), }, relations: ['exifInfo'], order: { @@ -381,4 +382,15 @@ export class AssetService { [authUser.id], ); } + + async checkDuplicatedAsset(authUser: AuthUserDto, deviceAssetId: string) { + const res = await this.assetRepository.findOne({ + where: { + deviceAssetId, + userId: authUser.id, + }, + }); + + return res ? true : false; + } } diff --git a/server/apps/immich/src/api-v1/communication/communication.gateway.ts b/server/apps/immich/src/api-v1/communication/communication.gateway.ts index ae3d4e7031..a7e06bb023 100644 --- a/server/apps/immich/src/api-v1/communication/communication.gateway.ts +++ b/server/apps/immich/src/api-v1/communication/communication.gateway.ts @@ -6,8 +6,9 @@ import { Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { UserEntity } from '@app/database/entities/user.entity'; import { Repository } from 'typeorm'; +import { query } from 'express'; -@WebSocketGateway() +@WebSocketGateway({ cors: true }) export class CommunicationGateway implements OnGatewayConnection, OnGatewayDisconnect { constructor( private immichJwtService: ImmichJwtService, @@ -21,27 +22,33 @@ export class CommunicationGateway implements OnGatewayConnection, OnGatewayDisco handleDisconnect(client: Socket) { client.leave(client.nsp.name); - Logger.log(`Client ${client.id} disconnected`); + Logger.log(`Client ${client.id} disconnected from Websocket`, 'WebsocketConnectionEvent'); } async handleConnection(client: Socket, ...args: any[]) { - Logger.log(`New websocket connection: ${client.id}`, 'NewWebSocketConnection'); - const accessToken = client.handshake.headers.authorization.split(' ')[1]; - const res = await this.immichJwtService.validateToken(accessToken); + try { + Logger.log(`New websocket connection: ${client.id}`, 'WebsocketConnectionEvent'); - if (!res.status) { - client.emit('error', 'unauthorized'); - client.disconnect(); - return; + const accessToken = client.handshake.headers.authorization.split(' ')[1]; + + const res = await this.immichJwtService.validateToken(accessToken); + + if (!res.status) { + client.emit('error', 'unauthorized'); + client.disconnect(); + return; + } + + const user = await this.userRepository.findOne({ where: { id: res.userId } }); + if (!user) { + client.emit('error', 'unauthorized'); + client.disconnect(); + return; + } + + client.join(user.id); + } catch (e) { + // Logger.error(`Error establish websocket conneciton ${e}`, 'HandleWebscoketConnection'); } - - const user = await this.userRepository.findOne({ where: { id: res.userId } }); - if (!user) { - client.emit('error', 'unauthorized'); - client.disconnect(); - return; - } - - client.join(user.id); } } diff --git a/server/apps/microservices/src/main.ts b/server/apps/microservices/src/main.ts index efd6fdad7b..b921a9ca76 100644 --- a/server/apps/microservices/src/main.ts +++ b/server/apps/microservices/src/main.ts @@ -1,10 +1,13 @@ import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; +import { RedisIoAdapter } from '../../immich/src/middlewares/redis-io.adapter.middleware'; import { MicroservicesModule } from './microservices.module'; async function bootstrap() { const app = await NestFactory.create(MicroservicesModule); + app.useWebSocketAdapter(new RedisIoAdapter(app)); + await app.listen(3000, () => { if (process.env.NODE_ENV == 'development') { Logger.log('Running Immich Microservices in DEVELOPMENT environment', 'ImmichMicroservice'); diff --git a/server/apps/microservices/src/microservices.module.ts b/server/apps/microservices/src/microservices.module.ts index 92c1ce4e75..b7258fa3f5 100644 --- a/server/apps/microservices/src/microservices.module.ts +++ b/server/apps/microservices/src/microservices.module.ts @@ -11,6 +11,9 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor'; import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor'; import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; import { VideoTranscodeProcessor } from './processors/video-transcode.processor'; +import { AssetModule } from '../../immich/src/api-v1/asset/asset.module'; +import { CommunicationGateway } from '../../immich/src/api-v1/communication/communication.gateway'; +import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module'; @Module({ imports: [ @@ -56,6 +59,7 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor' removeOnFail: false, }, }), + CommunicationModule, ], controllers: [], providers: [ diff --git a/server/apps/microservices/src/processors/asset-uploaded.processor.ts b/server/apps/microservices/src/processors/asset-uploaded.processor.ts index c8345c5ca3..f616b83222 100644 --- a/server/apps/microservices/src/processors/asset-uploaded.processor.ts +++ b/server/apps/microservices/src/processors/asset-uploaded.processor.ts @@ -46,6 +46,7 @@ export class AssetUploadedProcessor { await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() }); } else { // Generate Thumbnail -> Then generate webp, tag image and detect object + await this.thumbnailGeneratorQueue.add('generate-jpeg-thumbnail', { asset }, { jobId: randomUUID() }); } // Video Conversion @@ -63,5 +64,10 @@ export class AssetUploadedProcessor { { jobId: randomUUID() }, ); } + + // Extract video duration if uploaded from the web + if (asset.type == AssetType.VIDEO && asset.duration == '0:00:00.000000') { + await this.metadataExtractionQueue.add('extract-video-length', { asset }, { jobId: randomUUID() }); + } } } diff --git a/server/apps/microservices/src/processors/metadata-extraction.processor.ts b/server/apps/microservices/src/processors/metadata-extraction.processor.ts index 6e649e4e8a..0a8a205d4b 100644 --- a/server/apps/microservices/src/processors/metadata-extraction.processor.ts +++ b/server/apps/microservices/src/processors/metadata-extraction.processor.ts @@ -12,6 +12,8 @@ import { Logger } from '@nestjs/common'; import axios from 'axios'; import { SmartInfoEntity } from '@app/database/entities/smart-info.entity'; import { ConfigService } from '@nestjs/config'; +import ffmpeg from 'fluent-ffmpeg'; +// import moment from 'moment'; @Processor('metadata-extraction-queue') export class MetadataExtractionProcessor { @@ -129,4 +131,27 @@ export class MetadataExtractionProcessor { Logger.error(`Failed to trigger object detection pipe line ${error.toString()}`); } } + + @Process({ name: 'extract-video-length', concurrency: 2 }) + async extractVideoLength(job: Job) { + const { asset }: { asset: AssetEntity } = job.data; + + ffmpeg.ffprobe(asset.originalPath, async (err, data) => { + if (!err) { + if (data.format.duration) { + const videoDurationInSecond = parseInt(data.format.duration.toString(), 0); + + const hours = Math.floor(videoDurationInSecond / 3600); + const minutes = Math.floor((videoDurationInSecond - hours * 3600) / 60); + const seconds = videoDurationInSecond - hours * 3600 - minutes * 60; + + const durationString = `${hours}:${minutes < 10 ? '0' + minutes.toString() : minutes}:${ + seconds < 10 ? '0' + seconds.toString() : seconds + }.000000`; + + await this.assetRepository.update({ id: asset.id }, { duration: durationString }); + } + } + }); + } } diff --git a/server/apps/microservices/src/processors/thumbnail.processor.ts b/server/apps/microservices/src/processors/thumbnail.processor.ts index 3f3919daa5..55cc77c470 100644 --- a/server/apps/microservices/src/processors/thumbnail.processor.ts +++ b/server/apps/microservices/src/processors/thumbnail.processor.ts @@ -1,22 +1,82 @@ -import { Process, Processor } from '@nestjs/bull'; -import { Job } from 'bull'; -import { AssetEntity } from '@app/database/entities/asset.entity'; +import { InjectQueue, Process, Processor } from '@nestjs/bull'; +import { Job, Queue } from 'bull'; +import { AssetEntity, AssetType } from '@app/database/entities/asset.entity'; import { Repository } from 'typeorm/repository/Repository'; import { InjectRepository } from '@nestjs/typeorm'; import sharp from 'sharp'; +import { existsSync, mkdirSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { CommunicationGateway } from '../../../immich/src/api-v1/communication/communication.gateway'; +import ffmpeg from 'fluent-ffmpeg'; +import { Logger } from '@nestjs/common'; @Processor('thumbnail-generator-queue') export class ThumbnailGeneratorProcessor { constructor( @InjectRepository(AssetEntity) private assetRepository: RepositoryInfo
+Info