1
0
mirror of https://github.com/immich-app/immich.git synced 2024-12-23 02:06:15 +02:00

Added machine learning microservice and object detection (#76)

This commit is contained in:
Alex 2022-03-27 14:58:54 -05:00 committed by GitHub
parent fe693db84f
commit dd9c5244fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 11555 additions and 278 deletions

View File

@ -1,8 +1,8 @@
dev:
docker-compose -f ./docker/docker-compose.yml up
docker-compose -f ./docker/docker-compose.yml up --remove-orphans
dev-update:
docker-compose -f ./docker/docker-compose.yml up --build -V
docker-compose -f ./docker/docker-compose.yml up --build -V --remove-orphans
dev-scale:
docker-compose -f ./docker/docker-compose.yml up --build -V --scale immich_server=3 --remove-orphans

View File

@ -22,6 +22,34 @@ services:
networks:
- immich_network
immich_microservices:
image: immich-microservices-dev:1.3.2
build:
context: ../microservices
target: development
dockerfile: ../microservices/Dockerfile
command: npm run start:dev
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
expose:
- "3001"
volumes:
- ../microservices:/usr/src/app
- ${UPLOAD_LOCATION}:/usr/src/app/upload
- /usr/src/app/node_modules
env_file:
- .env
depends_on:
- database
- immich_server
networks:
- immich_network
redis:
container_name: immich_redis
image: redis:6.2
@ -60,35 +88,6 @@ services:
depends_on:
- immich_server
immich_tf_fastapi:
container_name: immich_tf_fastapi
image: tensor_flow_fastapi:1.0.0
restart: always
command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
build:
context: ../machine_learning
target: gpu
dockerfile: ../machine_learning/Dockerfile
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
volumes:
- ../machine_learning/app:/code/app
- ${UPLOAD_LOCATION}:/code/app/upload
ports:
- 2285:8000
expose:
- "8000"
depends_on:
- database
networks:
- immich_network
networks:
immich_network:
volumes:

View File

@ -23,6 +23,27 @@ services:
networks:
- immich_network
immich_microservices:
image: immich-microservices-dev:1.3.2
build:
context: ../microservices
target: development
dockerfile: ../microservices/Dockerfile
command: npm run start:dev
expose:
- "3001"
volumes:
- ../microservices:/usr/src/app
- ${UPLOAD_LOCATION}:/usr/src/app/upload
- /usr/src/app/node_modules
env_file:
- .env
depends_on:
- database
networks:
- immich_network
redis:
container_name: immich_redis
image: redis:6.2
@ -61,26 +82,26 @@ services:
depends_on:
- immich_server
immich_tf_fastapi:
container_name: immich_tf_fastapi
image: tensor_flow_fastapi:1.0.0
restart: always
command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
build:
context: ../machine_learning
target: cpu
dockerfile: ../machine_learning/Dockerfile
volumes:
- ../machine_learning/app:/code/app
- ${UPLOAD_LOCATION}:/code/app/upload
ports:
- 2285:8000
expose:
- "8000"
depends_on:
- database
networks:
- immich_network
# immich_tf_fastapi:
# container_name: immich_tf_fastapi
# image: tensor_flow_fastapi:1.0.0
# restart: always
# command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
# build:
# context: ../machine_learning
# target: cpu
# dockerfile: ../machine_learning/Dockerfile
# volumes:
# - ../machine_learning/app:/code/app
# - ${UPLOAD_LOCATION}:/code/app/upload
# ports:
# - 2285:8000
# expose:
# - "8000"
# depends_on:
# - database
# networks:
# - immich_network
networks:
immich_network:

View File

@ -13,7 +13,7 @@ server {
client_max_body_size 50000M;
listen 80;
access_log off;
location / {
proxy_buffering off;
proxy_buffer_size 16k;

View File

@ -0,0 +1,4 @@
node_modules/
upload/
dist/

43
microservices/Dockerfile Normal file
View File

@ -0,0 +1,43 @@
##################################
# DEVELOPMENT
##################################
FROM node:16-bullseye-slim AS development
ARG DEBIAN_FRONTEND=noninteractive
WORKDIR /usr/src/app
COPY package.json package-lock.json ./
RUN apt-get update
RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
RUN npm install
COPY . .
RUN npm run build
#################################
# PRODUCTION
#################################
FROM node:16-bullseye-slim AS production
ARG DEBIAN_FRONTEND=noninteractive
ARG NODE_ENV=production
ENV NODE_ENV=${NODE_ENV}
WORKDIR /usr/src/app
COPY package.json package-lock.json ./
RUN apt-get update
RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
RUN npm install --only=production
COPY . .
COPY --from=development /usr/src/app/dist ./dist
CMD ["node", "dist/main"]

View File

@ -1,73 +1,4 @@
<p align="center">
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
</p>
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
[circleci-url]: https://circleci.com/gh/nestjs/nest
# Microservices for Immich
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
<p align="center">
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
</p>
<!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
[![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
## Description
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
## Installation
```bash
$ npm install
```
## Running the app
```bash
# development
$ npm run start
# watch mode
$ npm run start:dev
# production mode
$ npm run start:prod
```
## Test
```bash
# unit tests
$ npm run test
# e2e tests
$ npm run test:e2e
# test coverage
$ npm run test:cov
```
## Support
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
## Stay in touch
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
- Website - [https://nestjs.com](https://nestjs.com/)
- Twitter - [@nestframework](https://twitter.com/nestframework)
## License
Nest is [MIT licensed](LICENSE).
## Image Classifier

File diff suppressed because it is too large Load Diff

View File

@ -23,13 +23,25 @@
"dependencies": {
"@nestjs/common": "^8.0.0",
"@nestjs/core": "^8.0.0",
"@nestjs/mapped-types": "^1.0.1",
"@nestjs/platform-express": "^8.0.0",
"@nestjs/typeorm": "^8.0.3",
"@tensorflow-models/coco-ssd": "^2.2.2",
"@tensorflow-models/mobilenet": "^2.1.0",
"@tensorflow/tfjs": "^3.15.0",
"@tensorflow/tfjs-converter": "^3.15.0",
"@tensorflow/tfjs-core": "^3.15.0",
"@tensorflow/tfjs-node": "^3.15.0",
"@tensorflow/tfjs-node-gpu": "^3.15.0",
"@trpc/server": "^9.20.3",
"pg": "^8.7.3",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0"
"rxjs": "^7.2.0",
"typeorm": "^0.2.45"
},
"devDependencies": {
"@nestjs/cli": "^8.0.0",
"@nestjs/cli": "^8.2.4",
"@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0",
"@types/express": "^4.17.13",

View File

@ -1,12 +0,0 @@
import { Controller, Get } from '@nestjs/common';
import { AppService } from './app.service';
@Controller()
export class AppController {
constructor(private readonly appService: AppService) {}
@Get()
getHello(): string {
return this.appService.getHello();
}
}

View File

@ -1,10 +1,16 @@
import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import { ImageClassifierModule } from './image-classifier/image-classifier.module';
import { databaseConfig } from './config/database.config';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ObjectDetectionModule } from './object-detection/object-detection.module';
@Module({
imports: [],
controllers: [AppController],
providers: [AppService],
imports: [
TypeOrmModule.forRoot(databaseConfig),
ImageClassifierModule,
ObjectDetectionModule,
],
controllers: [],
providers: [],
})
export class AppModule {}

View File

@ -1,9 +0,0 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class AppService {
getHello(): string {
console.log('Hello World 123');
return 'Hello World!';
}
}

View File

@ -0,0 +1,11 @@
import { TypeOrmModuleOptions } from '@nestjs/typeorm';
export const databaseConfig: TypeOrmModuleOptions = {
type: 'postgres',
host: 'immich_postgres',
port: 5432,
username: process.env.DB_USERNAME,
password: process.env.DB_PASSWORD,
database: process.env.DB_DATABASE_NAME,
synchronize: false,
};

View File

@ -0,0 +1,14 @@
import { Body, Controller, Post } from '@nestjs/common';
import { ImageClassifierService } from './image-classifier.service';
@Controller('image-classifier')
export class ImageClassifierController {
constructor(
private readonly imageClassifierService: ImageClassifierService,
) {}
@Post('/tagImage')
async tagImage(@Body('thumbnailPath') thumbnailPath: string) {
return await this.imageClassifierService.tagImage(thumbnailPath);
}
}

View File

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { ImageClassifierService } from './image-classifier.service';
import { ImageClassifierController } from './image-classifier.controller';
@Module({
controllers: [ImageClassifierController],
providers: [ImageClassifierService],
})
export class ImageClassifierModule {}

View File

@ -0,0 +1,48 @@
import { Injectable, Logger } from '@nestjs/common';
import * as mobilenet from '@tensorflow-models/mobilenet';
import * as cocoSsd from '@tensorflow-models/coco-ssd';
import * as tf from '@tensorflow/tfjs-node';
import * as fs from 'fs';
@Injectable()
export class ImageClassifierService {
private readonly MOBILENET_VERSION = 2;
private readonly MOBILENET_ALPHA = 1.0;
private mobileNetModel: mobilenet.MobileNet;
constructor() {
Logger.log(
`Running Node TensorFlow Version : ${tf.version['tfjs']}`,
'ImageClassifier',
);
mobilenet
.load({
version: this.MOBILENET_VERSION,
alpha: this.MOBILENET_ALPHA,
})
.then((mobilenetModel) => (this.mobileNetModel = mobilenetModel));
}
async tagImage(thumbnailPath: string) {
try {
const isExist = fs.existsSync(thumbnailPath);
if (isExist) {
const tags = [];
const image = fs.readFileSync(thumbnailPath);
const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
const predictions = await this.mobileNetModel.classify(decodedImage);
for (const prediction of predictions) {
if (prediction.probability >= 0.1) {
tags.push(...prediction.className.split(',').map((e) => e.trim()));
}
}
return tags;
}
} catch (e) {
console.log('Error reading file ', e);
}
}
}

View File

@ -1,15 +1,10 @@
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
import { AppService } from './app.service';
async function bootstrap() {
const app = await NestFactory.createApplicationContext(AppModule);
const app = await NestFactory.create(AppModule);
const appService = app.get(AppService);
appService.getHello();
await app.close();
await app.listen(3001);
}
bootstrap();

View File

@ -0,0 +1,14 @@
import { Body, Controller, Post } from '@nestjs/common';
import { ObjectDetectionService } from './object-detection.service';
@Controller('object-detection')
export class ObjectDetectionController {
constructor(
private readonly objectDetectionService: ObjectDetectionService,
) {}
@Post('/detectObject')
async detectObject(@Body('thumbnailPath') thumbnailPath: string) {
return await this.objectDetectionService.detectObject(thumbnailPath);
}
}

View File

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { ObjectDetectionService } from './object-detection.service';
import { ObjectDetectionController } from './object-detection.controller';
@Module({
controllers: [ObjectDetectionController],
providers: [ObjectDetectionService],
})
export class ObjectDetectionModule {}

View File

@ -0,0 +1,38 @@
import { Injectable, Logger } from '@nestjs/common';
import * as cocoSsd from '@tensorflow-models/coco-ssd';
import * as tf from '@tensorflow/tfjs-node';
import * as fs from 'fs';
@Injectable()
export class ObjectDetectionService {
private cocoSsdModel: cocoSsd.ObjectDetection;
constructor() {
Logger.log(
`Running Node TensorFlow Version : ${tf.version['tfjs']}`,
'ObjectDetection',
);
cocoSsd.load().then((model) => (this.cocoSsdModel = model));
}
async detectObject(thumbnailPath: string) {
try {
const isExist = fs.existsSync(thumbnailPath);
if (isExist) {
const tags = new Set();
const image = fs.readFileSync(thumbnailPath);
const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
const predictions = await this.cocoSsdModel.detect(decodedImage);
for (const result of predictions) {
if (result.score > 0.5) {
tags.add(result.class);
}
}
return [...tags];
}
} catch (e) {
console.log('Error reading file ', e);
}
}
}

View File

@ -1,8 +1,9 @@
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import * as request from 'supertest';
import { AppModule } from './../src/app.module';
import { AppModule } from '../src/app.module';
// End to End test
describe('AppController (e2e)', () => {
let app: INestApplication;

View File

@ -79,4 +79,4 @@ SPEC CHECKSUMS:
PODFILE CHECKSUM: 05c3056158482c567a3e0cdab1351ceeee238a07
COCOAPODS: 1.10.1
COCOAPODS: 1.11.3

View File

@ -341,7 +341,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
MTL_ENABLE_DEBUG_INFO = NO;
NEW_SETTING = "";
SDKROOT = iphoneos;
@ -425,7 +425,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
MTL_ENABLE_DEBUG_INFO = YES;
NEW_SETTING = "";
ONLY_ACTIVE_ARCH = YES;
@ -475,7 +475,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
MTL_ENABLE_DEBUG_INFO = NO;
NEW_SETTING = "";
SDKROOT = iphoneos;

View File

@ -15,7 +15,7 @@ class LoginForm extends HookConsumerWidget {
Widget build(BuildContext context, WidgetRef ref) {
final usernameController = useTextEditingController(text: 'testuser@email.com');
final passwordController = useTextEditingController(text: 'password');
final serverEndpointController = useTextEditingController(text: 'http://192.168.1.103:2283');
final serverEndpointController = useTextEditingController(text: 'http://192.168.1.216:2283');
return Center(
child: ConstrainedBox(

View File

@ -0,0 +1,80 @@
import 'dart:convert';
class CuratedObject {
final String id;
final String object;
final String resizePath;
final String deviceAssetId;
final String deviceId;
CuratedObject({
required this.id,
required this.object,
required this.resizePath,
required this.deviceAssetId,
required this.deviceId,
});
CuratedObject copyWith({
String? id,
String? object,
String? resizePath,
String? deviceAssetId,
String? deviceId,
}) {
return CuratedObject(
id: id ?? this.id,
object: object ?? this.object,
resizePath: resizePath ?? this.resizePath,
deviceAssetId: deviceAssetId ?? this.deviceAssetId,
deviceId: deviceId ?? this.deviceId,
);
}
Map<String, dynamic> toMap() {
final result = <String, dynamic>{};
result.addAll({'id': id});
result.addAll({'object': object});
result.addAll({'resizePath': resizePath});
result.addAll({'deviceAssetId': deviceAssetId});
result.addAll({'deviceId': deviceId});
return result;
}
factory CuratedObject.fromMap(Map<String, dynamic> map) {
return CuratedObject(
id: map['id'] ?? '',
object: map['object'] ?? '',
resizePath: map['resizePath'] ?? '',
deviceAssetId: map['deviceAssetId'] ?? '',
deviceId: map['deviceId'] ?? '',
);
}
String toJson() => json.encode(toMap());
factory CuratedObject.fromJson(String source) => CuratedObject.fromMap(json.decode(source));
@override
String toString() {
return 'CuratedObject(id: $id, object: $object, resizePath: $resizePath, deviceAssetId: $deviceAssetId, deviceId: $deviceId)';
}
@override
bool operator ==(Object other) {
if (identical(this, other)) return true;
return other is CuratedObject &&
other.id == id &&
other.object == object &&
other.resizePath == resizePath &&
other.deviceAssetId == deviceAssetId &&
other.deviceId == deviceId;
}
@override
int get hashCode {
return id.hashCode ^ object.hashCode ^ resizePath.hashCode ^ deviceAssetId.hashCode ^ deviceId.hashCode;
}
}

View File

@ -1,5 +1,6 @@
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
import 'package:immich_mobile/modules/search/models/search_page_state.model.dart';
import 'package:immich_mobile/modules/search/services/search.service.dart';
@ -64,3 +65,14 @@ final getCuratedLocationProvider = FutureProvider.autoDispose<List<CuratedLocati
return [];
}
});
final getCuratedObjectProvider = FutureProvider.autoDispose<List<CuratedObject>>((ref) async {
final SearchService _searchService = SearchService();
var curatedObject = await _searchService.getCuratedObjects();
if (curatedObject != null) {
return curatedObject;
} else {
return [];
}
});

View File

@ -2,6 +2,7 @@ import 'dart:convert';
import 'package:flutter/material.dart';
import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
import 'package:immich_mobile/shared/models/immich_asset.model.dart';
import 'package:immich_mobile/shared/services/network.service.dart';
@ -52,4 +53,19 @@ class SearchService {
throw Error();
}
}
Future<List<CuratedObject>?> getCuratedObjects() async {
try {
var res = await _networkService.getRequest(url: "asset/allObjects");
List<dynamic> decodedData = jsonDecode(res.toString());
List<CuratedObject> result = List.from(decodedData.map((a) => CuratedObject.fromMap(a)));
return result;
} catch (e) {
debugPrint("[ERROR] [CuratedObject] ${e.toString()}");
throw Error();
}
}
}

View File

@ -6,10 +6,12 @@ import 'package:hive_flutter/hive_flutter.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/constants/hive_box.dart';
import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
import 'package:immich_mobile/modules/search/providers/search_page_state.provider.dart';
import 'package:immich_mobile/modules/search/ui/search_bar.dart';
import 'package:immich_mobile/modules/search/ui/search_suggestion_list.dart';
import 'package:immich_mobile/routing/router.dart';
import 'package:immich_mobile/utils/capitalize_first_letter.dart';
// ignore: must_be_immutable
class SearchPage extends HookConsumerWidget {
@ -22,6 +24,7 @@ class SearchPage extends HookConsumerWidget {
var box = Hive.box(userInfoBox);
final isSearchEnabled = ref.watch(searchPageStateProvider).isSearchEnabled;
AsyncValue<List<CuratedLocation>> curatedLocation = ref.watch(getCuratedLocationProvider);
AsyncValue<List<CuratedObject>> curatedObjects = ref.watch(getCuratedObjectProvider);
useEffect(() {
searchFocusNode = FocusNode();
@ -82,6 +85,54 @@ class SearchPage extends HookConsumerWidget {
);
}
_buildThings() {
return curatedObjects.when(
loading: () => const CircularProgressIndicator(),
error: (err, stack) => Text('Error: $err'),
data: (objects) {
return objects.isNotEmpty
? SizedBox(
height: MediaQuery.of(context).size.width / 3,
child: ListView.builder(
padding: const EdgeInsets.only(left: 16),
scrollDirection: Axis.horizontal,
itemCount: curatedObjects.value?.length,
itemBuilder: ((context, index) {
CuratedObject curatedObjectInfo = objects[index];
var thumbnailRequestUrl =
'${box.get(serverEndpointKey)}/asset/file?aid=${curatedObjectInfo.deviceAssetId}&did=${curatedObjectInfo.deviceId}&isThumb=true';
return ThumbnailWithInfo(
imageUrl: thumbnailRequestUrl,
textInfo: curatedObjectInfo.object,
onTap: () {
AutoRouter.of(context)
.push(SearchResultRoute(searchTerm: curatedObjectInfo.object.capitalizeFirstLetter()));
},
);
}),
),
)
: SizedBox(
height: MediaQuery.of(context).size.width / 3,
child: ListView.builder(
padding: const EdgeInsets.only(left: 16),
scrollDirection: Axis.horizontal,
itemCount: 1,
itemBuilder: ((context, index) {
return ThumbnailWithInfo(
imageUrl:
'https://images.unsplash.com/photo-1612178537253-bccd437b730e?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxzZWFyY2h8NXx8Ymxhbmt8ZW58MHx8MHx8&auto=format&fit=crop&w=700&q=60',
textInfo: 'No Object Info Available',
onTap: () {},
);
}),
),
);
},
);
}
return Scaffold(
appBar: SearchBar(
searchFocusNode: searchFocusNode,
@ -104,6 +155,14 @@ class SearchPage extends HookConsumerWidget {
),
),
_buildPlaces(),
const Padding(
padding: EdgeInsets.all(16.0),
child: Text(
"Things",
style: TextStyle(fontWeight: FontWeight.bold, fontSize: 16),
),
),
_buildThings()
],
),
isSearchEnabled ? SearchSuggestionList(onSubmitted: _onSearchSubmitted) : Container(),
@ -160,7 +219,7 @@ class ThumbnailWithInfo extends StatelessWidget {
child: SizedBox(
width: MediaQuery.of(context).size.width / 3,
child: Text(
textInfo,
textInfo.capitalizeFirstLetter(),
style: const TextStyle(
color: Colors.white,
fontWeight: FontWeight.bold,

View File

@ -26,6 +26,7 @@ class TabNavigationObserver extends AutoRouterObserver {
if (route.name == 'SearchRoute') {
// Refresh Location State
ref.refresh(getCuratedLocationProvider);
ref.refresh(getCuratedObjectProvider);
}
ref.watch(serverInfoProvider.notifier).getServerVersion();

View File

@ -0,0 +1,5 @@
extension StringExtension on String {
String capitalizeFirstLetter() {
return "${this[0].toUpperCase()}${substring(1).toLowerCase()}";
}
}

View File

@ -1,12 +1,12 @@
{
"name": "immich",
"version": "0.0.1",
"version": "1.3.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "immich",
"version": "0.0.1",
"version": "1.3.2",
"license": "UNLICENSED",
"dependencies": {
"@mapbox/mapbox-sdk": "^0.13.3",
@ -1547,6 +1547,66 @@
}
}
},
"node_modules/@nestjs/microservices": {
"version": "8.4.3",
"resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
"integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
"optional": true,
"peer": true,
"dependencies": {
"iterare": "1.2.1",
"json-socket": "0.3.0",
"tslib": "2.3.1"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/nest"
},
"peerDependencies": {
"@grpc/grpc-js": "*",
"@nestjs/common": "^8.0.0",
"@nestjs/core": "^8.0.0",
"@nestjs/websockets": "^8.0.0",
"amqp-connection-manager": "*",
"amqplib": "*",
"cache-manager": "*",
"kafkajs": "*",
"mqtt": "*",
"nats": "*",
"redis": "*",
"reflect-metadata": "^0.1.12",
"rxjs": "^7.1.0"
},
"peerDependenciesMeta": {
"@grpc/grpc-js": {
"optional": true
},
"@nestjs/websockets": {
"optional": true
},
"amqp-connection-manager": {
"optional": true
},
"amqplib": {
"optional": true
},
"cache-manager": {
"optional": true
},
"kafkajs": {
"optional": true
},
"mqtt": {
"optional": true
},
"nats": {
"optional": true
},
"redis": {
"optional": true
}
}
},
"node_modules/@nestjs/passport": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@ -7053,6 +7113,13 @@
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"node_modules/json-socket": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
"integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
"optional": true,
"peer": true
},
"node_modules/json-stable-stringify-without-jsonify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
@ -11943,6 +12010,18 @@
"integrity": "sha512-NFvofzSinp00j5rzUd4tf+xi9od6383iY0JP7o0Bnu1fuItAUkWBgc4EKuIQ3D+c2QI3i9pG1kDWAeY27EMGtg==",
"requires": {}
},
"@nestjs/microservices": {
"version": "8.4.3",
"resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
"integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
"optional": true,
"peer": true,
"requires": {
"iterare": "1.2.1",
"json-socket": "0.3.0",
"tslib": "2.3.1"
}
},
"@nestjs/passport": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@ -16243,6 +16322,13 @@
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"json-socket": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
"integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
"optional": true,
"peer": true
},
"json-stable-stringify-without-jsonify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",

View File

@ -16,13 +16,12 @@ import {
} from '@nestjs/common';
import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard';
import { AssetService } from './asset.service';
import { FileFieldsInterceptor, FilesInterceptor } from '@nestjs/platform-express';
import { FileFieldsInterceptor } from '@nestjs/platform-express';
import { multerOption } from '../../config/multer-option.config';
import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
import { CreateAssetDto } from './dto/create-asset.dto';
import { ServeFileDto } from './dto/serve-file.dto';
import { AssetOptimizeService } from '../../modules/image-optimize/image-optimize.service';
import { AssetEntity, AssetType } from './entities/asset.entity';
import { AssetEntity } from './entities/asset.entity';
import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
import { Response as Res } from 'express';
import { GetNewAssetQueryDto } from './dto/get-new-asset-query.dto';
@ -61,6 +60,7 @@ export class AssetController {
if (uploadFiles.thumbnailData != null) {
await this.assetService.updateThumbnailInfo(savedAsset.id, uploadFiles.thumbnailData[0].path);
await this.backgroundTaskService.tagImage(uploadFiles.thumbnailData[0].path, savedAsset);
await this.backgroundTaskService.detectObject(uploadFiles.thumbnailData[0].path, savedAsset);
}
await this.backgroundTaskService.extractExif(savedAsset, file.originalname, file.size);
@ -81,6 +81,11 @@ export class AssetController {
return this.assetService.serveFile(authUser, query, res, headers);
}
@Get('/allObjects')
async getCuratedObject(@GetAuthUser() authUser: AuthUserDto) {
return this.assetService.getCuratedObject(authUser);
}
@Get('/allLocation')
async getCuratedLocation(@GetAuthUser() authUser: AuthUserDto) {
return this.assetService.getCuratedLocation(authUser);

View File

@ -3,9 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { MoreThan, Repository } from 'typeorm';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { CreateAssetDto } from './dto/create-asset.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { AssetEntity, AssetType } from './entities/asset.entity';
import _, { result } from 'lodash';
import _ from 'lodash';
import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
import { GetAllAssetReponseDto } from './dto/get-all-asset-response.dto';
import { createReadStream, stat } from 'fs';
@ -44,9 +43,7 @@ export class AssetService {
asset.duration = assetInfo.duration;
try {
const res = await this.assetRepository.save(asset);
return res;
return await this.assetRepository.save(asset);
} catch (e) {
Logger.error(`Error Create New Asset ${e}`, 'createUserAsset');
}
@ -68,13 +65,11 @@ export class AssetService {
public async getAllAssetsNoPagination(authUser: AuthUserDto) {
try {
const assets = await this.assetRepository
return await this.assetRepository
.createQueryBuilder('a')
.where('a."userId" = :userId', { userId: authUser.id })
.orderBy('a."createdAt"::date', 'DESC')
.getMany();
return assets;
} catch (e) {
Logger.error(e, 'getAllAssets');
}
@ -226,10 +221,10 @@ export class AssetService {
}
public async deleteAssetById(authUser: AuthUserDto, assetIds: DeleteAssetDto) {
let result = [];
const result = [];
const target = assetIds.ids;
for (let assetId of target) {
for (const assetId of target) {
const res = await this.assetRepository.delete({
id: assetId,
userId: authUser.id,
@ -251,11 +246,11 @@ export class AssetService {
return result;
}
async getAssetSearchTerm(authUser: AuthUserDto): Promise<String[]> {
const possibleSearchTerm = new Set<String>();
async getAssetSearchTerm(authUser: AuthUserDto): Promise<string[]> {
const possibleSearchTerm = new Set<string>();
const rows = await this.assetRepository.query(
`
select distinct si.tags, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
select distinct si.tags, si.objects, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
from assets a
left join exif e on a.id = e."assetId"
left join smart_info si on a.id = si."assetId"
@ -268,6 +263,9 @@ export class AssetService {
// tags
row['tags']?.map((tag) => possibleSearchTerm.add(tag?.toLowerCase()));
// objects
row['objects']?.map((object) => possibleSearchTerm.add(object?.toLowerCase()));
// asset's tyoe
possibleSearchTerm.add(row['type']?.toLowerCase());
@ -300,18 +298,17 @@ export class AssetService {
WHERE a."userId" = $1
AND
(
TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
e.exif_text_searchable_column @@ PLAINTO_TSQUERY('english', $2)
);
`;
const rows = await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
return rows;
return await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
}
async getCuratedLocation(authUser: AuthUserDto) {
const rows = await this.assetRepository.query(
return await this.assetRepository.query(
`
select distinct on (e.city) a.id, e.city, a."resizePath", a."deviceAssetId", a."deviceId"
from assets a
@ -322,7 +319,18 @@ export class AssetService {
`,
[authUser.id],
);
}
return rows;
async getCuratedObject(authUser: AuthUserDto) {
return await this.assetRepository.query(
`
select distinct on (unnest(si.objects)) a.id, unnest(si.objects) as "object", a."resizePath", a."deviceAssetId", a."deviceId"
from assets a
left join smart_info si on a.id = si."assetId"
where a."userId" = $1
and si.objects is not null
`,
[authUser.id],
);
}
}

View File

@ -13,6 +13,9 @@ export class SmartInfoEntity {
@Column({ type: 'text', array: true, nullable: true })
tags: string[];
@Column({ type: 'text', array: true, nullable: true })
objects: string[];
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
asset: SmartInfoEntity;

View File

@ -5,7 +5,6 @@ import { UserModule } from './api-v1/user/user.module';
import { AssetModule } from './api-v1/asset/asset.module';
import { AuthModule } from './api-v1/auth/auth.module';
import { ImmichJwtModule } from './modules/immich-jwt/immich-jwt.module';
import { JwtModule } from '@nestjs/jwt';
import { DeviceInfoModule } from './api-v1/device-info/device-info.module';
import { AppLoggerMiddleware } from './middlewares/app-logger.middleware';
import { ConfigModule, ConfigService } from '@nestjs/config';
@ -26,14 +25,12 @@ import { CommunicationModule } from './api-v1/communication/communication.module
ImmichJwtModule,
DeviceInfoModule,
BullModule.forRootAsync({
imports: [ConfigModule],
useFactory: async (configService: ConfigService) => ({
useFactory: async () => ({
redis: {
host: 'immich_redis',
port: 6379,
},
}),
inject: [ConfigService],
}),
ImageOptimizeModule,

View File

@ -0,0 +1,20 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddObjectColumnToSmartInfo1648317474768
implements MigrationInterface
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
ALTER TABLE smart_info
ADD COLUMN objects text[];
`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
ALTER TABLE smart_info
DROP COLUMN objects;
`);
}
}

View File

@ -6,7 +6,7 @@ import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
import { ConfigService } from '@nestjs/config';
import exifr from 'exifr';
import { readFile } from 'fs/promises';
import fs, { rmSync } from 'fs';
import fs from 'fs';
import { Logger } from '@nestjs/common';
import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
import axios from 'axios';
@ -114,14 +114,37 @@ export class BackgroundTaskProcessor {
@Process('tag-image')
async tagImage(job) {
const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
const res = await axios.post('http://immich_tf_fastapi:8000/tagImage', { thumbnail_path: thumbnailPath });
if (res.status == 200) {
const res = await axios.post('http://immich_microservices:3001/image-classifier/tagImage', {
thumbnailPath: thumbnailPath,
});
if (res.status == 201 && res.data.length > 0) {
const smartInfo = new SmartInfoEntity();
smartInfo.assetId = asset.id;
smartInfo.tags = [...res.data];
await this.smartInfoRepository.save(smartInfo);
await this.smartInfoRepository.upsert(smartInfo, {
conflictPaths: ['assetId'],
});
}
}
@Process('detect-object')
async detectObject(job) {
const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
const res = await axios.post('http://immich_microservices:3001/object-detection/detectObject', {
thumbnailPath: thumbnailPath,
});
if (res.status == 201 && res.data.length > 0) {
const smartInfo = new SmartInfoEntity();
smartInfo.assetId = asset.id;
smartInfo.objects = [...res.data];
await this.smartInfoRepository.upsert(smartInfo, {
conflictPaths: ['assetId'],
});
}
}
}

View File

@ -43,4 +43,15 @@ export class BackgroundTaskService {
{ jobId: randomUUID() },
);
}
async detectObject(thumbnailPath: string, asset: AssetEntity) {
await this.backgroundTaskQueue.add(
'detect-object',
{
thumbnailPath,
asset,
},
{ jobId: randomUUID() },
);
}
}