self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-26 10:50:29 +02:00
Code Issues Releases Activity

feat(cli): Implement logic for --skip-hash (#8561)

Browse Source 
* feat(cli): Implement logic for --skip-hash

* feat: better output for duplicates

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
pedrxd 2024-04-08 17:40:32 +02:00 committed by GitHub
parent 29e47dd7c1
commit 0075243ed5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 69 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
cli/src/commands/asset.ts
View File

@ -56,14 +56,13 @@ class UploadFile extends File {
export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => {
await authenticate(baseOptions);
const files = await scan(paths, options);
if (files.length === 0) {
const scanFiles = await scan(paths, options);
if (scanFiles.length === 0) {
console.log('No files found, exiting');
return;
}
const { newFiles, duplicates } = await checkForDuplicates(files, options);
const { newFiles, duplicates } = await checkForDuplicates(scanFiles, options);
const newAssets = await uploadFiles(newFiles, options);
await updateAlbums([...newAssets, ...duplicates], options);
await deleteFiles(newFiles, options);
@ -84,7 +83,12 @@ const scan = async (pathsToCrawl: string[], options: UploadOptionsDto) => {
return files;
};
const checkForDuplicates = async (files: string[], { concurrency }: UploadOptionsDto) => {
const checkForDuplicates = async (files: string[], { concurrency, skipHash }: UploadOptionsDto) => {
if (skipHash) {
console.log('Skipping hash check, assuming all files are new');
return { newFiles: files, duplicates: [] };
}
const progressBar = new SingleBar(
{ format: 'Checking files | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
Presets.shades_classic,
@ -147,17 +151,32 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio
uploadProgress.start(totalSize, 0);
uploadProgress.update({ value_formatted: 0, total_formatted: byteSize(totalSize) });
let totalSizeUploaded = 0;
let duplicateCount = 0;
let duplicateSize = 0;
let successCount = 0;
let successSize = 0;
const newAssets: Asset[] = [];
try {
for (const items of chunk(files, concurrency)) {
await Promise.all(
items.map(async (filepath) => {
const stats = statsMap.get(filepath) as Stats;
const response = await uploadFile(filepath, stats);
totalSizeUploaded += stats.size ?? 0;
uploadProgress.update(totalSizeUploaded, { value_formatted: byteSize(totalSizeUploaded) });
newAssets.push({ id: response.id, filepath });
if (response.duplicate) {
duplicateCount++;
duplicateSize += stats.size ?? 0;
} else {
successCount++;
successSize += stats.size ?? 0;
}
uploadProgress.update(successSize, { value_formatted: byteSize(successSize + duplicateSize) });
return response;
}),
);
@ -166,7 +185,10 @@ const uploadFiles = async (files: string[], { dryRun, concurrency }: UploadOptio
uploadProgress.stop();
}
console.log(`Successfully uploaded ${newAssets.length} asset${s(newAssets.length)} (${byteSize(totalSizeUploaded)})`);
console.log(`Successfully uploaded ${successCount} new asset${s(successCount)} (${byteSize(successSize)})`);
if (duplicateCount > 0) {
console.log(`Skipped ${duplicateCount} duplicate asset${s(duplicateCount)} (${byteSize(duplicateSize)})`);
}
return newAssets;
};

46
e2e/src/cli/specs/upload.e2e-spec.ts
View File

@ -1,4 +1,5 @@
import { LoginResponseDto, getAllAlbums, getAllAssets } from '@immich/sdk';
import { readFileSync } from 'node:fs';
import { mkdir, readdir, rm, symlink } from 'node:fs/promises';
import { asKeyAuth, immichCli, testAssetDir, utils } from 'src/utils';
import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
@ -23,7 +24,7 @@ describe(`immich upload`, () => {
const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
expect(stderr).toBe('');
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]),
expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
);
expect(exitCode).toBe(0);
@ -35,7 +36,7 @@ describe(`immich upload`, () => {
const first = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
expect(first.stderr).toBe('');
expect(first.stdout.split('\n')).toEqual(
expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 asset')]),
expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
);
expect(first.exitCode).toBe(0);
@ -69,7 +70,7 @@ describe(`immich upload`, () => {
const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']);
expect(stderr).toBe('');
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
);
expect(exitCode).toBe(0);
@ -88,7 +89,7 @@ describe(`immich upload`, () => {
]);
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
expect.stringContaining('Successfully uploaded 9 new assets'),
expect.stringContaining('Successfully created 1 new album'),
expect.stringContaining('Successfully updated 9 assets'),
]),
@ -107,7 +108,7 @@ describe(`immich upload`, () => {
it('should add existing assets to albums', async () => {
const response1 = await immichCli(['upload', `${testAssetDir}/albums/nature/`, '--recursive']);
expect(response1.stdout.split('\n')).toEqual(
expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 new assets')]),
);
expect(response1.stderr).toBe('');
expect(response1.exitCode).toBe(0);
@ -147,7 +148,7 @@ describe(`immich upload`, () => {
]);
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
expect.stringContaining('Successfully uploaded 9 new assets'),
expect.stringContaining('Successfully created 1 new album'),
expect.stringContaining('Successfully updated 9 assets'),
]),
@ -180,7 +181,7 @@ describe(`immich upload`, () => {
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
expect.stringContaining('Successfully uploaded 9 new assets'),
expect.stringContaining('Deleting assets that have been uploaded'),
]),
);
@ -192,6 +193,32 @@ describe(`immich upload`, () => {
});
});
describe('immich upload --skip-hash', () => {
it('should skip hashing', async () => {
const filename = `albums/nature/silver_fir.jpg`;
await utils.createAsset(admin.accessToken, {
assetData: {
bytes: readFileSync(`${testAssetDir}/${filename}`),
filename: 'silver_fit.jpg',
},
});
const { stderr, stdout, exitCode } = await immichCli(['upload', `${testAssetDir}/${filename}`, '--skip-hash']);
expect(stderr).toBe('');
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
'Skipping hash check, assuming all files are new',
expect.stringContaining('Successfully uploaded 0 new assets'),
expect.stringContaining('Skipped 1 duplicate asset'),
]),
);
expect(exitCode).toBe(0);
const assets = await getAllAssets({}, { headers: asKeyAuth(key) });
expect(assets.length).toBe(1);
});
});
describe('immich upload --concurrency <number>', () => {
it('should work', async () => {
const { stderr, stdout, exitCode } = await immichCli([
@ -203,7 +230,10 @@ describe(`immich upload`, () => {
expect(stderr).toBe('');
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([expect.stringContaining('Successfully uploaded 9 assets')]),
expect.arrayContaining([
'Found 9 new files and 0 duplicates',
expect.stringContaining('Successfully uploaded 9 new assets'),
]),
);
expect(exitCode).toBe(0);