1
0
mirror of https://github.com/immich-app/immich.git synced 2025-10-31 00:18:28 +02:00

feat(server): support for read-only assets and importing existing items in the filesystem (#2715)

* Added read-only flag for assets, endpoint to trigger file import vs upload

* updated fixtures with new property

* if upload is 'read-only', ensure there is no existing asset at the designated originalPath

* added test for file import as well as detecting existing image at read-only destination location

* Added storage service test for a case where it should not move read-only assets

* upload doesn't need the read-only flag available, just importing

* default isReadOnly on import endpoint to true

* formatting fixes

* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation

* updated code to reflect changes in MR

* fixed read stream promise return type

* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates

* refactor: import asset

* chore: open api

* chore: tests

* Added externalPath support for individual users, updated UI to allow this to be set by admin

* added missing var for externalPath in ui

* chore: open api

* fix: compilation issues

* fix: server test

* built api, fixed user-response dto to include externalPath

* reverted accidental commit

* bad commit of duplicate externalPath in user response  dto

* fixed tests to include externalPath on expected result

* fix: unit tests

* centralized supported filetypes, perform file type checking of asset and sidecar during file import process

* centralized supported filetype check method to keep regex DRY

* fixed typo

* combined migrations into one

* update api

* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not

* update mimetype

* Fixed detect correct mimetype

* revert asset-upload config

* reverted domain.constant

* refactor

* fix mime-type issue

* fix format

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Alex Phillips
2023-06-21 22:33:20 -04:00
committed by GitHub
parent 7f44d508dc
commit e171fec5aa
55 changed files with 1321 additions and 128 deletions

View File

@@ -85,6 +85,7 @@ part 'model/download_files_dto.dart';
part 'model/exif_response_dto.dart';
part 'model/get_asset_by_time_bucket_dto.dart';
part 'model/get_asset_count_by_time_bucket_dto.dart';
part 'model/import_asset_dto.dart';
part 'model/job_command.dart';
part 'model/job_command_dto.dart';
part 'model/job_counts_dto.dart';

View File

@@ -1123,6 +1123,53 @@ class AssetApi {
return null;
}
/// Performs an HTTP 'POST /asset/import' operation and returns the [Response].
/// Parameters:
///
/// * [ImportAssetDto] importAssetDto (required):
Future<Response> importFileWithHttpInfo(ImportAssetDto importAssetDto,) async {
// ignore: prefer_const_declarations
final path = r'/asset/import';
// ignore: prefer_final_locals
Object? postBody = importAssetDto;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
const contentTypes = <String>['application/json'];
return apiClient.invokeAPI(
path,
'POST',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Parameters:
///
/// * [ImportAssetDto] importAssetDto (required):
Future<AssetFileUploadResponseDto?> importFile(ImportAssetDto importAssetDto,) async {
final response = await importFileWithHttpInfo(importAssetDto,);
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetFileUploadResponseDto',) as AssetFileUploadResponseDto;
}
return null;
}
/// Performs an HTTP 'POST /asset/search' operation and returns the [Response].
/// Parameters:
///
@@ -1307,6 +1354,8 @@ class AssetApi {
///
/// * [MultipartFile] assetData (required):
///
/// * [String] fileExtension (required):
///
/// * [String] deviceAssetId (required):
///
/// * [String] deviceId (required):
@@ -1317,20 +1366,20 @@ class AssetApi {
///
/// * [bool] isFavorite (required):
///
/// * [String] fileExtension (required):
///
/// * [String] key:
///
/// * [MultipartFile] livePhotoData:
///
/// * [MultipartFile] sidecarData:
///
/// * [bool] isReadOnly:
///
/// * [bool] isArchived:
///
/// * [bool] isVisible:
///
/// * [String] duration:
Future<Response> uploadFileWithHttpInfo(AssetTypeEnum assetType, MultipartFile assetData, String deviceAssetId, String deviceId, DateTime fileCreatedAt, DateTime fileModifiedAt, bool isFavorite, String fileExtension, { String? key, MultipartFile? livePhotoData, MultipartFile? sidecarData, bool? isArchived, bool? isVisible, String? duration, }) async {
Future<Response> uploadFileWithHttpInfo(AssetTypeEnum assetType, MultipartFile assetData, String fileExtension, String deviceAssetId, String deviceId, DateTime fileCreatedAt, DateTime fileModifiedAt, bool isFavorite, { String? key, MultipartFile? livePhotoData, MultipartFile? sidecarData, bool? isReadOnly, bool? isArchived, bool? isVisible, String? duration, }) async {
// ignore: prefer_const_declarations
final path = r'/asset/upload';
@@ -1368,6 +1417,14 @@ class AssetApi {
mp.fields[r'sidecarData'] = sidecarData.field;
mp.files.add(sidecarData);
}
if (isReadOnly != null) {
hasFields = true;
mp.fields[r'isReadOnly'] = parameterToString(isReadOnly);
}
if (fileExtension != null) {
hasFields = true;
mp.fields[r'fileExtension'] = parameterToString(fileExtension);
}
if (deviceAssetId != null) {
hasFields = true;
mp.fields[r'deviceAssetId'] = parameterToString(deviceAssetId);
@@ -1396,10 +1453,6 @@ class AssetApi {
hasFields = true;
mp.fields[r'isVisible'] = parameterToString(isVisible);
}
if (fileExtension != null) {
hasFields = true;
mp.fields[r'fileExtension'] = parameterToString(fileExtension);
}
if (duration != null) {
hasFields = true;
mp.fields[r'duration'] = parameterToString(duration);
@@ -1425,6 +1478,8 @@ class AssetApi {
///
/// * [MultipartFile] assetData (required):
///
/// * [String] fileExtension (required):
///
/// * [String] deviceAssetId (required):
///
/// * [String] deviceId (required):
@@ -1435,21 +1490,21 @@ class AssetApi {
///
/// * [bool] isFavorite (required):
///
/// * [String] fileExtension (required):
///
/// * [String] key:
///
/// * [MultipartFile] livePhotoData:
///
/// * [MultipartFile] sidecarData:
///
/// * [bool] isReadOnly:
///
/// * [bool] isArchived:
///
/// * [bool] isVisible:
///
/// * [String] duration:
Future<AssetFileUploadResponseDto?> uploadFile(AssetTypeEnum assetType, MultipartFile assetData, String deviceAssetId, String deviceId, DateTime fileCreatedAt, DateTime fileModifiedAt, bool isFavorite, String fileExtension, { String? key, MultipartFile? livePhotoData, MultipartFile? sidecarData, bool? isArchived, bool? isVisible, String? duration, }) async {
final response = await uploadFileWithHttpInfo(assetType, assetData, deviceAssetId, deviceId, fileCreatedAt, fileModifiedAt, isFavorite, fileExtension, key: key, livePhotoData: livePhotoData, sidecarData: sidecarData, isArchived: isArchived, isVisible: isVisible, duration: duration, );
Future<AssetFileUploadResponseDto?> uploadFile(AssetTypeEnum assetType, MultipartFile assetData, String fileExtension, String deviceAssetId, String deviceId, DateTime fileCreatedAt, DateTime fileModifiedAt, bool isFavorite, { String? key, MultipartFile? livePhotoData, MultipartFile? sidecarData, bool? isReadOnly, bool? isArchived, bool? isVisible, String? duration, }) async {
final response = await uploadFileWithHttpInfo(assetType, assetData, fileExtension, deviceAssetId, deviceId, fileCreatedAt, fileModifiedAt, isFavorite, key: key, livePhotoData: livePhotoData, sidecarData: sidecarData, isReadOnly: isReadOnly, isArchived: isArchived, isVisible: isVisible, duration: duration, );
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}

View File

@@ -265,6 +265,8 @@ class ApiClient {
return GetAssetByTimeBucketDto.fromJson(value);
case 'GetAssetCountByTimeBucketDto':
return GetAssetCountByTimeBucketDto.fromJson(value);
case 'ImportAssetDto':
return ImportAssetDto.fromJson(value);
case 'JobCommand':
return JobCommandTypeTransformer().decode(value);
case 'JobCommandDto':

View File

@@ -18,6 +18,7 @@ class CreateUserDto {
required this.firstName,
required this.lastName,
this.storageLabel,
this.externalPath,
});
String email;
@@ -30,13 +31,16 @@ class CreateUserDto {
String? storageLabel;
String? externalPath;
@override
bool operator ==(Object other) => identical(this, other) || other is CreateUserDto &&
other.email == email &&
other.password == password &&
other.firstName == firstName &&
other.lastName == lastName &&
other.storageLabel == storageLabel;
other.storageLabel == storageLabel &&
other.externalPath == externalPath;
@override
int get hashCode =>
@@ -45,10 +49,11 @@ class CreateUserDto {
(password.hashCode) +
(firstName.hashCode) +
(lastName.hashCode) +
(storageLabel == null ? 0 : storageLabel!.hashCode);
(storageLabel == null ? 0 : storageLabel!.hashCode) +
(externalPath == null ? 0 : externalPath!.hashCode);
@override
String toString() => 'CreateUserDto[email=$email, password=$password, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel]';
String toString() => 'CreateUserDto[email=$email, password=$password, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel, externalPath=$externalPath]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -61,6 +66,11 @@ class CreateUserDto {
} else {
// json[r'storageLabel'] = null;
}
if (this.externalPath != null) {
json[r'externalPath'] = this.externalPath;
} else {
// json[r'externalPath'] = null;
}
return json;
}
@@ -88,6 +98,7 @@ class CreateUserDto {
firstName: mapValueOfType<String>(json, r'firstName')!,
lastName: mapValueOfType<String>(json, r'lastName')!,
storageLabel: mapValueOfType<String>(json, r'storageLabel'),
externalPath: mapValueOfType<String>(json, r'externalPath'),
);
}
return null;

View File

@@ -0,0 +1,232 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class ImportAssetDto {
/// Returns a new [ImportAssetDto] instance.
ImportAssetDto({
required this.assetType,
this.isReadOnly = true,
required this.assetPath,
this.sidecarPath,
required this.deviceAssetId,
required this.deviceId,
required this.fileCreatedAt,
required this.fileModifiedAt,
required this.isFavorite,
this.isArchived,
this.isVisible,
this.duration,
});
AssetTypeEnum assetType;
bool isReadOnly;
String assetPath;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
String? sidecarPath;
String deviceAssetId;
String deviceId;
DateTime fileCreatedAt;
DateTime fileModifiedAt;
bool isFavorite;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
bool? isArchived;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
bool? isVisible;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
String? duration;
@override
bool operator ==(Object other) => identical(this, other) || other is ImportAssetDto &&
other.assetType == assetType &&
other.isReadOnly == isReadOnly &&
other.assetPath == assetPath &&
other.sidecarPath == sidecarPath &&
other.deviceAssetId == deviceAssetId &&
other.deviceId == deviceId &&
other.fileCreatedAt == fileCreatedAt &&
other.fileModifiedAt == fileModifiedAt &&
other.isFavorite == isFavorite &&
other.isArchived == isArchived &&
other.isVisible == isVisible &&
other.duration == duration;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(assetType.hashCode) +
(isReadOnly.hashCode) +
(assetPath.hashCode) +
(sidecarPath == null ? 0 : sidecarPath!.hashCode) +
(deviceAssetId.hashCode) +
(deviceId.hashCode) +
(fileCreatedAt.hashCode) +
(fileModifiedAt.hashCode) +
(isFavorite.hashCode) +
(isArchived == null ? 0 : isArchived!.hashCode) +
(isVisible == null ? 0 : isVisible!.hashCode) +
(duration == null ? 0 : duration!.hashCode);
@override
String toString() => 'ImportAssetDto[assetType=$assetType, isReadOnly=$isReadOnly, assetPath=$assetPath, sidecarPath=$sidecarPath, deviceAssetId=$deviceAssetId, deviceId=$deviceId, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, isFavorite=$isFavorite, isArchived=$isArchived, isVisible=$isVisible, duration=$duration]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'assetType'] = this.assetType;
json[r'isReadOnly'] = this.isReadOnly;
json[r'assetPath'] = this.assetPath;
if (this.sidecarPath != null) {
json[r'sidecarPath'] = this.sidecarPath;
} else {
// json[r'sidecarPath'] = null;
}
json[r'deviceAssetId'] = this.deviceAssetId;
json[r'deviceId'] = this.deviceId;
json[r'fileCreatedAt'] = this.fileCreatedAt.toUtc().toIso8601String();
json[r'fileModifiedAt'] = this.fileModifiedAt.toUtc().toIso8601String();
json[r'isFavorite'] = this.isFavorite;
if (this.isArchived != null) {
json[r'isArchived'] = this.isArchived;
} else {
// json[r'isArchived'] = null;
}
if (this.isVisible != null) {
json[r'isVisible'] = this.isVisible;
} else {
// json[r'isVisible'] = null;
}
if (this.duration != null) {
json[r'duration'] = this.duration;
} else {
// json[r'duration'] = null;
}
return json;
}
/// Returns a new [ImportAssetDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static ImportAssetDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "ImportAssetDto[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "ImportAssetDto[$key]" has a null value in JSON.');
});
return true;
}());
return ImportAssetDto(
assetType: AssetTypeEnum.fromJson(json[r'assetType'])!,
isReadOnly: mapValueOfType<bool>(json, r'isReadOnly') ?? true,
assetPath: mapValueOfType<String>(json, r'assetPath')!,
sidecarPath: mapValueOfType<String>(json, r'sidecarPath'),
deviceAssetId: mapValueOfType<String>(json, r'deviceAssetId')!,
deviceId: mapValueOfType<String>(json, r'deviceId')!,
fileCreatedAt: mapDateTime(json, r'fileCreatedAt', '')!,
fileModifiedAt: mapDateTime(json, r'fileModifiedAt', '')!,
isFavorite: mapValueOfType<bool>(json, r'isFavorite')!,
isArchived: mapValueOfType<bool>(json, r'isArchived'),
isVisible: mapValueOfType<bool>(json, r'isVisible'),
duration: mapValueOfType<String>(json, r'duration'),
);
}
return null;
}
static List<ImportAssetDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <ImportAssetDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = ImportAssetDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, ImportAssetDto> mapFromJson(dynamic json) {
final map = <String, ImportAssetDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = ImportAssetDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of ImportAssetDto-objects as value to a dart map
static Map<String, List<ImportAssetDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<ImportAssetDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = ImportAssetDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'assetType',
'assetPath',
'deviceAssetId',
'deviceId',
'fileCreatedAt',
'fileModifiedAt',
'isFavorite',
};
}

View File

@@ -19,6 +19,7 @@ class UpdateUserDto {
this.firstName,
this.lastName,
this.storageLabel,
this.externalPath,
this.isAdmin,
this.shouldChangePassword,
});
@@ -65,6 +66,14 @@ class UpdateUserDto {
///
String? storageLabel;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
String? externalPath;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
@@ -89,6 +98,7 @@ class UpdateUserDto {
other.firstName == firstName &&
other.lastName == lastName &&
other.storageLabel == storageLabel &&
other.externalPath == externalPath &&
other.isAdmin == isAdmin &&
other.shouldChangePassword == shouldChangePassword;
@@ -101,11 +111,12 @@ class UpdateUserDto {
(firstName == null ? 0 : firstName!.hashCode) +
(lastName == null ? 0 : lastName!.hashCode) +
(storageLabel == null ? 0 : storageLabel!.hashCode) +
(externalPath == null ? 0 : externalPath!.hashCode) +
(isAdmin == null ? 0 : isAdmin!.hashCode) +
(shouldChangePassword == null ? 0 : shouldChangePassword!.hashCode);
@override
String toString() => 'UpdateUserDto[id=$id, email=$email, password=$password, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel, isAdmin=$isAdmin, shouldChangePassword=$shouldChangePassword]';
String toString() => 'UpdateUserDto[id=$id, email=$email, password=$password, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel, externalPath=$externalPath, isAdmin=$isAdmin, shouldChangePassword=$shouldChangePassword]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -135,6 +146,11 @@ class UpdateUserDto {
} else {
// json[r'storageLabel'] = null;
}
if (this.externalPath != null) {
json[r'externalPath'] = this.externalPath;
} else {
// json[r'externalPath'] = null;
}
if (this.isAdmin != null) {
json[r'isAdmin'] = this.isAdmin;
} else {
@@ -173,6 +189,7 @@ class UpdateUserDto {
firstName: mapValueOfType<String>(json, r'firstName'),
lastName: mapValueOfType<String>(json, r'lastName'),
storageLabel: mapValueOfType<String>(json, r'storageLabel'),
externalPath: mapValueOfType<String>(json, r'externalPath'),
isAdmin: mapValueOfType<bool>(json, r'isAdmin'),
shouldChangePassword: mapValueOfType<bool>(json, r'shouldChangePassword'),
);

View File

@@ -18,6 +18,7 @@ class UserResponseDto {
required this.firstName,
required this.lastName,
required this.storageLabel,
required this.externalPath,
required this.profileImagePath,
required this.shouldChangePassword,
required this.isAdmin,
@@ -37,6 +38,8 @@ class UserResponseDto {
String? storageLabel;
String? externalPath;
String profileImagePath;
bool shouldChangePassword;
@@ -58,6 +61,7 @@ class UserResponseDto {
other.firstName == firstName &&
other.lastName == lastName &&
other.storageLabel == storageLabel &&
other.externalPath == externalPath &&
other.profileImagePath == profileImagePath &&
other.shouldChangePassword == shouldChangePassword &&
other.isAdmin == isAdmin &&
@@ -74,6 +78,7 @@ class UserResponseDto {
(firstName.hashCode) +
(lastName.hashCode) +
(storageLabel == null ? 0 : storageLabel!.hashCode) +
(externalPath == null ? 0 : externalPath!.hashCode) +
(profileImagePath.hashCode) +
(shouldChangePassword.hashCode) +
(isAdmin.hashCode) +
@@ -83,7 +88,7 @@ class UserResponseDto {
(oauthId.hashCode);
@override
String toString() => 'UserResponseDto[id=$id, email=$email, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel, profileImagePath=$profileImagePath, shouldChangePassword=$shouldChangePassword, isAdmin=$isAdmin, createdAt=$createdAt, deletedAt=$deletedAt, updatedAt=$updatedAt, oauthId=$oauthId]';
String toString() => 'UserResponseDto[id=$id, email=$email, firstName=$firstName, lastName=$lastName, storageLabel=$storageLabel, externalPath=$externalPath, profileImagePath=$profileImagePath, shouldChangePassword=$shouldChangePassword, isAdmin=$isAdmin, createdAt=$createdAt, deletedAt=$deletedAt, updatedAt=$updatedAt, oauthId=$oauthId]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -95,6 +100,11 @@ class UserResponseDto {
json[r'storageLabel'] = this.storageLabel;
} else {
// json[r'storageLabel'] = null;
}
if (this.externalPath != null) {
json[r'externalPath'] = this.externalPath;
} else {
// json[r'externalPath'] = null;
}
json[r'profileImagePath'] = this.profileImagePath;
json[r'shouldChangePassword'] = this.shouldChangePassword;
@@ -134,6 +144,7 @@ class UserResponseDto {
firstName: mapValueOfType<String>(json, r'firstName')!,
lastName: mapValueOfType<String>(json, r'lastName')!,
storageLabel: mapValueOfType<String>(json, r'storageLabel'),
externalPath: mapValueOfType<String>(json, r'externalPath'),
profileImagePath: mapValueOfType<String>(json, r'profileImagePath')!,
shouldChangePassword: mapValueOfType<bool>(json, r'shouldChangePassword')!,
isAdmin: mapValueOfType<bool>(json, r'isAdmin')!,
@@ -193,6 +204,7 @@ class UserResponseDto {
'firstName',
'lastName',
'storageLabel',
'externalPath',
'profileImagePath',
'shouldChangePassword',
'isAdmin',