feat(server): search across own+partner assets (#5966)

* feat(server): search across own+partner assets

* generate sql

* fix sql parameter

server/src/domain/person/person.service.ts

@@ -334,7 +334,7 @@ export class PersonService {
 
     for (const { embedding, ...rest } of faces) {
       const matches = await this.smartInfoRepository.searchFaces({
-        ownerId: asset.ownerId,
+        userIds: [asset.ownerId],
         embedding,
         numResults: 1,
         maxDistance: machineLearning.facialRecognition.maxDistance,

server/src/domain/repositories/asset.repository.ts

@@ -199,5 +199,5 @@ export interface IAssetRepository {
   search(options: AssetSearchOptions): Promise<AssetEntity[]>;
   getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
   getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
-  searchMetadata(query: string, userId: string, options: MetadataSearchOptions): Promise<AssetEntity[]>;
+  searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
 }

server/src/domain/repositories/smart-info.repository.ts

@@ -5,7 +5,7 @@ export const ISmartInfoRepository = 'ISmartInfoRepository';
 export type Embedding = number[];
 
 export interface EmbeddingSearch {
-  ownerId: string;
+  userIds: string[];
   embedding: Embedding;
   numResults: number;
   maxDistance?: number;

server/src/domain/search/search.service.spec.ts

@@ -4,6 +4,7 @@ import {
   authStub,
   newAssetRepositoryMock,
   newMachineLearningRepositoryMock,
+  newPartnerRepositoryMock,
   newPersonRepositoryMock,
   newSmartInfoRepositoryMock,
   newSystemConfigRepositoryMock,
@@ -13,6 +14,7 @@ import { mapAsset } from '../asset';
 import {
   IAssetRepository,
   IMachineLearningRepository,
+  IPartnerRepository,
   IPersonRepository,
   ISmartInfoRepository,
   ISystemConfigRepository,
@@ -29,6 +31,7 @@ describe(SearchService.name, () => {
   let machineMock: jest.Mocked<IMachineLearningRepository>;
   let personMock: jest.Mocked<IPersonRepository>;
   let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
+  let partnerMock: jest.Mocked<IPartnerRepository>;
 
   beforeEach(() => {
     assetMock = newAssetRepositoryMock();
@@ -36,7 +39,8 @@ describe(SearchService.name, () => {
     machineMock = newMachineLearningRepositoryMock();
     personMock = newPersonRepositoryMock();
     smartInfoMock = newSmartInfoRepositoryMock();
-    sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock);
+    partnerMock = newPartnerRepositoryMock();
+    sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock, partnerMock);
   });
 
   it('should work', () => {
@@ -87,6 +91,7 @@ describe(SearchService.name, () => {
     it('should search by metadata if `clip` option is false', async () => {
       const dto: SearchDto = { q: 'test query', clip: false };
       assetMock.searchMetadata.mockResolvedValueOnce([assetStub.image]);
+      partnerMock.getAll.mockResolvedValueOnce([]);
       const expectedResponse = {
         albums: {
           total: 0,
@@ -105,7 +110,7 @@ describe(SearchService.name, () => {
       const result = await sut.search(authStub.user1, dto);
 
       expect(result).toEqual(expectedResponse);
-      expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, authStub.user1.user.id, { numResults: 250 });
+      expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, [authStub.user1.user.id], { numResults: 250 });
       expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled();
     });
 
@@ -114,6 +119,7 @@ describe(SearchService.name, () => {
       const embedding = [1, 2, 3];
       smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
       machineMock.encodeText.mockResolvedValueOnce(embedding);
+      partnerMock.getAll.mockResolvedValueOnce([]);
       const expectedResponse = {
         albums: {
           total: 0,
@@ -133,7 +139,7 @@ describe(SearchService.name, () => {
 
       expect(result).toEqual(expectedResponse);
       expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({
-        ownerId: authStub.user1.user.id,
+        userIds: [authStub.user1.user.id],
         embedding,
         numResults: 100,
       });

server/src/domain/search/search.service.ts

@@ -7,6 +7,7 @@ import { PersonResponseDto } from '../person';
 import {
   IAssetRepository,
   IMachineLearningRepository,
+  IPartnerRepository,
   IPersonRepository,
   ISmartInfoRepository,
   ISystemConfigRepository,
@@ -28,6 +29,7 @@ export class SearchService {
     @Inject(IPersonRepository) private personRepository: IPersonRepository,
     @Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
   ) {
     this.configCore = SystemConfigCore.create(configRepository);
   }
@@ -64,6 +66,7 @@ export class SearchService {
       throw new Error('CLIP is not enabled');
     }
     const strategy = dto.clip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
+    const userIds = await this.getUserIdsToSearch(auth);
 
     let assets: AssetEntity[] = [];
 
@@ -74,10 +77,10 @@ export class SearchService {
           { text: query },
           machineLearning.clip,
         );
-        assets = await this.smartInfoRepository.searchCLIP({ ownerId: auth.user.id, embedding, numResults: 100 });
+        assets = await this.smartInfoRepository.searchCLIP({ userIds: userIds, embedding, numResults: 100 });
         break;
       case SearchStrategy.TEXT:
-        assets = await this.assetRepository.searchMetadata(query, auth.user.id, { numResults: 250 });
+        assets = await this.assetRepository.searchMetadata(query, userIds, { numResults: 250 });
       default:
         break;
     }
@@ -97,4 +100,14 @@ export class SearchService {
       },
     };
   }
+
+  private async getUserIdsToSearch(auth: AuthDto): Promise<string[]> {
+    const userIds: string[] = [auth.user.id];
+    const partners = await this.partnerRepository.getAll(auth.user.id);
+    const partnersIds = partners
+      .filter((partner) => partner.sharedBy && partner.inTimeline)
+      .map((partner) => partner.sharedById);
+    userIds.push(...partnersIds);
+    return userIds;
+  }
 }

server/src/infra/repositories/asset.repository.ts

@@ -804,10 +804,14 @@ export class AssetRepository implements IAssetRepository {
     return builder;
   }
 
-  @GenerateSql({ params: [DummyValue.STRING, DummyValue.UUID, { numResults: 250 }] })
+  @GenerateSql({ params: [DummyValue.STRING, [DummyValue.UUID], { numResults: 250 }] })
-  async searchMetadata(query: string, ownerId: string, { numResults }: MetadataSearchOptions): Promise<AssetEntity[]> {
+  async searchMetadata(
+    query: string,
+    userIds: string[],
+    { numResults }: MetadataSearchOptions,
+  ): Promise<AssetEntity[]> {
     const rows = await this.getBuilder({
-      userIds: [ownerId],
+      userIds: userIds,
       exifInfo: false,
       isArchived: false,
     })

server/src/infra/repositories/smart-info.repository.ts

@@ -41,9 +41,9 @@ export class SmartInfoRepository implements ISmartInfoRepository {
   }
 
   @GenerateSql({
-    params: [{ ownerId: DummyValue.UUID, embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
+    params: [{ userIds: [DummyValue.UUID], embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
   })
-  async searchCLIP({ ownerId, embedding, numResults }: EmbeddingSearch): Promise<AssetEntity[]> {
+  async searchCLIP({ userIds, embedding, numResults }: EmbeddingSearch): Promise<AssetEntity[]> {
     if (!isValidInteger(numResults, { min: 1 })) {
       throw new Error(`Invalid value for 'numResults': ${numResults}`);
     }
@@ -55,13 +55,13 @@ export class SmartInfoRepository implements ISmartInfoRepository {
       results = await manager
         .createQueryBuilder(AssetEntity, 'a')
         .innerJoin('a.smartSearch', 's')
-        .where('a.ownerId = :ownerId')
+        .where('a.ownerId IN (:...userIds )')
         .andWhere('a.isVisible = true')
         .andWhere('a.isArchived = false')
         .andWhere('a.fileCreatedAt < NOW()')
         .leftJoinAndSelect('a.exifInfo', 'e')
         .orderBy('s.embedding <=> :embedding')
-        .setParameters({ ownerId, embedding: asVector(embedding) })
+        .setParameters({ userIds, embedding: asVector(embedding) })
         .limit(numResults)
         .getMany();
     });
@@ -72,14 +72,14 @@ export class SmartInfoRepository implements ISmartInfoRepository {
   @GenerateSql({
     params: [
       {
-        ownerId: DummyValue.UUID,
+        userIds: [DummyValue.UUID],
         embedding: Array.from({ length: 512 }, Math.random),
         numResults: 100,
         maxDistance: 0.6,
       },
     ],
   })
-  async searchFaces({ ownerId, embedding, numResults, maxDistance }: EmbeddingSearch): Promise<AssetFaceEntity[]> {
+  async searchFaces({ userIds, embedding, numResults, maxDistance }: EmbeddingSearch): Promise<AssetFaceEntity[]> {
     if (!isValidInteger(numResults, { min: 1 })) {
       throw new Error(`Invalid value for 'numResults': ${numResults}`);
     }
@@ -91,9 +91,9 @@ export class SmartInfoRepository implements ISmartInfoRepository {
         .createQueryBuilder(AssetFaceEntity, 'faces')
         .select('1 + (faces.embedding <=> :embedding)', 'distance')
         .innerJoin('faces.asset', 'asset')
-        .where('asset.ownerId = :ownerId')
+        .where('asset.ownerId IN (:...userIds )')
         .orderBy('1 + (faces.embedding <=> :embedding)')
-        .setParameters({ ownerId, embedding: asVector(embedding) })
+        .setParameters({ userIds, embedding: asVector(embedding) })
         .limit(numResults);
 
       this.faceColumns.forEach((col) => cte.addSelect(`faces.${col}`, col));

server/src/infra/sql/smart.info.repository.sql

@@ -69,7 +69,7 @@ FROM
   LEFT JOIN "exif" "e" ON "e"."assetId" = "a"."id"
 WHERE
   (
-    "a"."ownerId" = $1
+    "a"."ownerId" IN ($1)
     AND "a"."isVisible" = true
     AND "a"."isArchived" = false
     AND "a"."fileCreatedAt" < NOW()
@@ -103,7 +103,7 @@ WITH
       INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
       AND ("asset"."deletedAt" IS NULL)
     WHERE
-      "asset"."ownerId" = $2
+      "asset"."ownerId" IN ($2)
     ORDER BY
       1 + ("faces"."embedding" <= > $3) ASC
     LIMIT