diff --git a/server/src/domain/smart-info/smart-info.service.spec.ts b/server/src/domain/smart-info/smart-info.service.spec.ts
index 3a31366372..2250d1796a 100644
--- a/server/src/domain/smart-info/smart-info.service.spec.ts
+++ b/server/src/domain/smart-info/smart-info.service.spec.ts
@@ -91,13 +91,13 @@ describe(SmartInfoService.name, () => {
       });
     });
 
-    it('should no update the smart info if no tags were returned', async () => {
+    it('should always overwrite old tags', async () => {
       machineMock.classifyImage.mockResolvedValue([]);
 
       await sut.handleClassifyImage({ id: asset.id });
 
       expect(machineMock.classifyImage).toHaveBeenCalled();
-      expect(smartMock.upsert).not.toHaveBeenCalled();
+      expect(smartMock.upsert).toHaveBeenCalled();
     });
   });
 
diff --git a/server/src/domain/smart-info/smart-info.service.ts b/server/src/domain/smart-info/smart-info.service.ts
index cc3a8a08ec..c1341a04b6 100644
--- a/server/src/domain/smart-info/smart-info.service.ts
+++ b/server/src/domain/smart-info/smart-info.service.ts
@@ -41,10 +41,6 @@ export class SmartInfoService {
     }
 
     const tags = await this.machineLearning.classifyImage({ imagePath: asset.resizePath });
-    if (tags.length === 0) {
-      return false;
-    }
-
     await this.repository.upsert({ assetId: asset.id, tags });
 
     return true;