feat(cli): Implement logic for --skip-hash (#8561)

* feat(cli): Implement logic for --skip-hash

* feat: better output for duplicates

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

cli/src/commands/asset.ts

@@ -56,14 +56,13 @@ class UploadFile extends File {
 export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => {
   await authenticate(baseOptions);
 
-  const files = await scan(paths, options);
+  const scanFiles = await scan(paths, options);
-  if (files.length === 0) {
+  if (scanFiles.length === 0) {
     console.log('No files found, exiting');
     return;
   }
 
-  const { newFiles, duplicates } = await checkForDuplicates(files, options);
+  const { newFiles, duplicates } = await checkForDuplicates(scanFiles, options);
-
   const newAssets = await uploadFiles(newFiles, options);
   await updateAlbums([...newAssets, ...duplicates], options);
   await deleteFiles(newFiles, options);
@@ -84,7 +83,12 @@ const scan = async (pathsToCrawl: string[], options: UploadOptionsDto) => {
   return files;
 };
 
-const checkForDuplicates = async (files: string[], { concurrency }: UploadOptionsDto) => {
+const checkForDuplicates = async (files: string[], { concurrency, skipHash }: UploadOptionsDto) => {
+  if (skipHash) {
+    console.log('Skipping hash check, assuming all files are new');
+    return { newFiles: files, duplicates: [] };
+  }
+
   const progressBar = new SingleBar(
     { format: 'Checking files | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
     Presets.shades_classic,
@@ -147,17 +151,32 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio
   uploadProgress.start(totalSize, 0);
   uploadProgress.update({ value_formatted: 0, total_formatted: byteSize(totalSize) });
 
-  let totalSizeUploaded = 0;
+  let duplicateCount = 0;
+  let duplicateSize = 0;
+  let successCount = 0;
+  let successSize = 0;
+
   const newAssets: Asset[] = [];
+
   try {
     for (const items of chunk(files, concurrency)) {
       await Promise.all(
         items.map(async (filepath) => {
           const stats = statsMap.get(filepath) as Stats;
           const response = await uploadFile(filepath, stats);
-          totalSizeUploaded += stats.size ?? 0;
+
-          uploadProgress.update(totalSizeUploaded, { value_formatted: byteSize(totalSizeUploaded) });
           newAssets.push({ id: response.id, filepath });
+
+          if (response.duplicate) {
+            duplicateCount++;
+            duplicateSize += stats.size ?? 0;
+          } else {
+            successCount++;
+            successSize += stats.size ?? 0;
+          }
+
+          uploadProgress.update(successSize, { value_formatted: byteSize(successSize + duplicateSize) });
+
           return response;
         }),
       );
@@ -166,7 +185,10 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio
     uploadProgress.stop();
   }
 
-  console.log(`Successfully uploaded ${newAssets.length} asset${s(newAssets.length)} (${byteSize(totalSizeUploaded)})`);
+  console.log(`Successfully uploaded ${successCount} new asset${s(successCount)} (${byteSize(successSize)})`);
+  if (duplicateCount > 0) {
+    console.log(`Skipped ${duplicateCount} duplicate asset${s(duplicateCount)} (${byteSize(duplicateSize)})`);
+  }
   return newAssets;
 };
 

e2e/src/cli/specs/upload.e2e-spec.ts

@@ -1,4 +1,5 @@
 import { LoginResponseDto, getAllAlbums, getAllAssets } from '@immich/sdk';
+import { readFileSync } from 'node:fs';
 import { mkdir, readdir, rm, symlink } from 'node:fs/promises';
 import { asKeyAuth, immichCli, testAssetDir, utils } from 'src/utils';
 import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
@@ -23,7 +24,7 @@ describe(`immich upload`, () => {
       const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
       expect(stderr).toBe('');
       expect(stdout.split('\n')).toEqual(
-        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]),
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
       );
       expect(exitCode).toBe(0);
 
@@ -35,7 +36,7 @@ describe(`immich upload`, () => {
       const first = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
       expect(first.stderr).toBe('');
       expect(first.stdout.split('\n')).toEqual(
-        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]),
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
       );
       expect(first.exitCode).toBe(0);
 
@@ -69,7 +70,7 @@ describe(`immich upload`, () => {
       const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']);
       expect(stderr).toBe('');
       expect(stdout.split('\n')).toEqual(
-        expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
       );
       expect(exitCode).toBe(0);
 
@@ -88,7 +89,7 @@ describe(`immich upload`, () => {
       ]);
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
-          expect.stringContaining('Successfully uploaded 9 assets'),
+          expect.stringContaining('Successfully uploaded 9 new assets'),
           expect.stringContaining('Successfully created 1 new album'),
           expect.stringContaining('Successfully updated 9 assets'),
         ]),
@@ -107,7 +108,7 @@ describe(`immich upload`, () => {
     it('should add existing assets to albums', async () => {
       const response1 = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']);
       expect(response1.stdout.split('\n')).toEqual(
-        expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
       );
       expect(response1.stderr).toBe('');
       expect(response1.exitCode).toBe(0);
@@ -147,7 +148,7 @@ describe(`immich upload`, () => {
       ]);
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
-          expect.stringContaining('Successfully uploaded 9 assets'),
+          expect.stringContaining('Successfully uploaded 9 new assets'),
           expect.stringContaining('Successfully created 1 new album'),
           expect.stringContaining('Successfully updated 9 assets'),
         ]),
@@ -180,7 +181,7 @@ describe(`immich upload`, () => {
 
       expect(stdout.split('\n')).toEqual(
         expect.arrayContaining([
-          expect.stringContaining('Successfully uploaded 9 assets'),
+          expect.stringContaining('Successfully uploaded 9 new assets'),
           expect.stringContaining('Deleting assets that have been uploaded'),
         ]),
       );
@@ -192,6 +193,32 @@ describe(`immich upload`, () => {
     });
   });
 
+  describe('immich upload --skip-hash', () => {
+    it('should skip hashing', async () => {
+      const filename = `albums/nature/silver_fir.jpg`;
+      await utils.createAsset(admin.accessToken, {
+        assetData: {
+          bytes: readFileSync(`${testAssetDir}/${filename}`),
+          filename: 'silver_fit.jpg',
+        },
+      });
+      const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/${filename}`, '--skip-hash']);
+
+      expect(stderr).toBe('');
+      expect(stdout.split('\n')).toEqual(
+        expect.arrayContaining([
+          'Skipping hash check, assuming all files are new',
+          expect.stringContaining('Successfully uploaded 0 new assets'),
+          expect.stringContaining('Skipped 1 duplicate asset'),
+        ]),
+      );
+      expect(exitCode).toBe(0);
+
+      const assets = await getAllAssets({}, { headers: asKeyAuth(key) });
+      expect(assets.length).toBe(1);
+    });
+  });
+
   describe('immich upload --concurrency <number>', () => {
     it('should work', async () => {
       const { stderr, stdout, exitCode } = await immichCli([
@@ -203,7 +230,10 @@ describe(`immich upload`, () => {
 
       expect(stderr).toBe('');
       expect(stdout.split('\n')).toEqual(
-        expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
+        expect.arrayContaining([
+          'Found 9 new files and 0 duplicates',
+          expect.stringContaining('Successfully uploaded 9 new assets'),
+        ]),
       );
       expect(exitCode).toBe(0);