diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index dc81c10dece..4c06edc8c96 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -56,6 +56,7 @@ custom_lint: allowed: # required / wanted - 'lib/infrastructure/repositories/album_media.repository.dart' + - 'lib/infrastructure/repositories/storage.repository.dart' - 'lib/repositories/{album,asset,file}_media.repository.dart' # acceptable exceptions for the time being - lib/entities/asset.entity.dart # to provide local AssetEntity for now diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt index f4dbda730b0..18a788903a0 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt @@ -247,6 +247,7 @@ interface NativeSyncApi { fun getAlbums(): List fun getAssetsCountSince(albumId: String, timestamp: Long): Long fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List + fun hashPaths(paths: List): List companion object { /** The codec used by NativeSyncApi. */ @@ -388,6 +389,23 @@ interface NativeSyncApi { channel.setMessageHandler(null) } } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val pathsArg = args[0] as List + val wrapped: List = try { + listOf(api.hashPaths(pathsArg)) + } catch (exception: Throwable) { + MessagesPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } } } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt index 23228553077..70fc045d5b7 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt @@ -4,7 +4,10 @@ import android.annotation.SuppressLint import android.content.Context import android.database.Cursor import android.provider.MediaStore +import android.util.Log import java.io.File +import java.io.FileInputStream +import java.security.MessageDigest sealed class AssetResult { data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult() @@ -16,6 +19,8 @@ open class NativeSyncApiImplBase(context: Context) { private val ctx: Context = context.applicationContext companion object { + private const val TAG = "NativeSyncApiImplBase" + const val MEDIA_SELECTION = "(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)" val MEDIA_SELECTION_ARGS = arrayOf( @@ -34,6 +39,8 @@ open class NativeSyncApiImplBase(context: Context) { MediaStore.MediaColumns.BUCKET_ID, MediaStore.MediaColumns.DURATION ) + + const val HASH_BUFFER_SIZE = 2 * 1024 * 1024 } protected fun getCursor( @@ -174,4 +181,24 @@ open class NativeSyncApiImplBase(context: Context) { .mapNotNull { result -> (result as? AssetResult.ValidAsset)?.asset } .toList() } + + fun hashPaths(paths: List): List { + val buffer = ByteArray(HASH_BUFFER_SIZE) + val digest = MessageDigest.getInstance("SHA-1") + + return paths.map { path -> + try { + FileInputStream(path).use { file -> + var bytesRead: Int + while (file.read(buffer).also { bytesRead = it } > 0) { + digest.update(buffer, 0, bytesRead) + } + } + digest.digest() + } catch (e: Exception) { + Log.w(TAG, "Failed to hash file $path: $e") + null + } + } + } } diff --git a/mobile/drift_schemas/main/drift_schema_v1.json b/mobile/drift_schemas/main/drift_schema_v1.json index 5cdec3d9245..ee8e41aea21 100644 --- a/mobile/drift_schemas/main/drift_schema_v1.json +++ b/mobile/drift_schemas/main/drift_schema_v1.json @@ -1 +1 @@ -{"_meta":{"description":"This file contains a serialized version of schema entities for drift.","version":"1.2.0"},"options":{"store_date_time_values_as_text":true},"entities":[{"id":0,"references":[],"type":"table","data":{"name":"user_entity","was_declared_in_moor":false,"columns":[{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_admin","getter_name":"isAdmin","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_admin\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_admin\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]},{"name":"email","getter_name":"email","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"profile_image_path","getter_name":"profileImagePath","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"quota_size_in_bytes","getter_name":"quotaSizeInBytes","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"quota_usage_in_bytes","getter_name":"quotaUsageInBytes","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":1,"references":[0],"type":"table","data":{"name":"user_metadata_entity","was_declared_in_moor":false,"columns":[{"name":"user_id","getter_name":"userId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"preferences","getter_name":"preferences","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"userPreferenceConverter","dart_type_name":"UserPreferences"}}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["user_id"]}},{"id":2,"references":[0],"type":"table","data":{"name":"partner_entity","was_declared_in_moor":false,"columns":[{"name":"shared_by_id","getter_name":"sharedById","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"shared_with_id","getter_name":"sharedWithId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"in_timeline","getter_name":"inTimeline","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"in_timeline\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"in_timeline\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["shared_by_id","shared_with_id"]}},{"id":3,"references":[],"type":"table","data":{"name":"local_album_entity","was_declared_in_moor":false,"columns":[{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"backup_selection","getter_name":"backupSelection","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(BackupSelection.values)","dart_type_name":"BackupSelection"}},{"name":"marker","getter_name":"marker_","moor_type":"bool","nullable":true,"customConstraints":null,"defaultConstraints":"CHECK (\"marker\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"marker\" IN (0, 1))"},"default_dart":null,"default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":4,"references":[],"type":"table","data":{"name":"local_asset_entity","was_declared_in_moor":false,"columns":[{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"type","getter_name":"type","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetType.values)","dart_type_name":"AssetType"}},{"name":"created_at","getter_name":"createdAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"duration_in_seconds","getter_name":"durationInSeconds","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"checksum","getter_name":"checksum","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_favorite","getter_name":"isFavorite","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_favorite\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_favorite\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":5,"references":[4,3],"type":"table","data":{"name":"local_album_asset_entity","was_declared_in_moor":false,"columns":[{"name":"asset_id","getter_name":"assetId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES local_asset_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES local_asset_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"album_id","getter_name":"albumId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES local_album_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES local_album_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["asset_id","album_id"]}},{"id":6,"references":[0],"type":"table","data":{"name":"remote_asset_entity","was_declared_in_moor":false,"columns":[{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"type","getter_name":"type","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetType.values)","dart_type_name":"AssetType"}},{"name":"created_at","getter_name":"createdAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"duration_in_seconds","getter_name":"durationInSeconds","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"checksum","getter_name":"checksum","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_favorite","getter_name":"isFavorite","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_favorite\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_favorite\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]},{"name":"owner_id","getter_name":"ownerId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"local_date_time","getter_name":"localDateTime","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"thumb_hash","getter_name":"thumbHash","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"deleted_at","getter_name":"deletedAt","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"visibility","getter_name":"visibility","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetVisibility.values)","dart_type_name":"AssetVisibility"}}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":7,"references":[6],"type":"table","data":{"name":"remote_exif_entity","was_declared_in_moor":false,"columns":[{"name":"asset_id","getter_name":"assetId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES remote_asset_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES remote_asset_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"city","getter_name":"city","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"state","getter_name":"state","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"country","getter_name":"country","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"date_time_original","getter_name":"dateTimeOriginal","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"description","getter_name":"description","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"height","getter_name":"height","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"width","getter_name":"width","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"exposure_time","getter_name":"exposureTime","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"f_number","getter_name":"fNumber","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"file_size","getter_name":"fileSize","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"focal_length","getter_name":"focalLength","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"latitude","getter_name":"latitude","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"longitude","getter_name":"longitude","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"iso","getter_name":"iso","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"make","getter_name":"make","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"model","getter_name":"model","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"orientation","getter_name":"orientation","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"time_zone","getter_name":"timeZone","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"rating","getter_name":"rating","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"projection_type","getter_name":"projectionType","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["asset_id"]}},{"id":8,"references":[4],"type":"index","data":{"on":4,"name":"idx_local_asset_checksum","sql":null,"unique":false,"columns":["checksum"]}},{"id":9,"references":[6],"type":"index","data":{"on":6,"name":"UQ_remote_asset_owner_checksum","sql":null,"unique":true,"columns":["checksum","owner_id"]}}]} \ No newline at end of file +{"_meta":{"description":"This file contains a serialized version of schema entities for drift.","version":"1.2.0"},"options":{"store_date_time_values_as_text":true},"entities":[{"id":0,"references":[],"type":"table","data":{"name":"user_entity","was_declared_in_moor":false,"columns":[{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_admin","getter_name":"isAdmin","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_admin\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_admin\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]},{"name":"email","getter_name":"email","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"profile_image_path","getter_name":"profileImagePath","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"quota_size_in_bytes","getter_name":"quotaSizeInBytes","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"quota_usage_in_bytes","getter_name":"quotaUsageInBytes","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":1,"references":[0],"type":"table","data":{"name":"user_metadata_entity","was_declared_in_moor":false,"columns":[{"name":"user_id","getter_name":"userId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"preferences","getter_name":"preferences","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"userPreferenceConverter","dart_type_name":"UserPreferences"}}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["user_id"]}},{"id":2,"references":[0],"type":"table","data":{"name":"partner_entity","was_declared_in_moor":false,"columns":[{"name":"shared_by_id","getter_name":"sharedById","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"shared_with_id","getter_name":"sharedWithId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"in_timeline","getter_name":"inTimeline","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"in_timeline\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"in_timeline\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["shared_by_id","shared_with_id"]}},{"id":3,"references":[],"type":"table","data":{"name":"local_album_entity","was_declared_in_moor":false,"columns":[{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"backup_selection","getter_name":"backupSelection","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(BackupSelection.values)","dart_type_name":"BackupSelection"}},{"name":"is_ios_shared_album","getter_name":"isIosSharedAlbum","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_ios_shared_album\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_ios_shared_album\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]},{"name":"marker","getter_name":"marker_","moor_type":"bool","nullable":true,"customConstraints":null,"defaultConstraints":"CHECK (\"marker\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"marker\" IN (0, 1))"},"default_dart":null,"default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":4,"references":[],"type":"table","data":{"name":"local_asset_entity","was_declared_in_moor":false,"columns":[{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"type","getter_name":"type","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetType.values)","dart_type_name":"AssetType"}},{"name":"created_at","getter_name":"createdAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"duration_in_seconds","getter_name":"durationInSeconds","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"checksum","getter_name":"checksum","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_favorite","getter_name":"isFavorite","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_favorite\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_favorite\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":5,"references":[4,3],"type":"table","data":{"name":"local_album_asset_entity","was_declared_in_moor":false,"columns":[{"name":"asset_id","getter_name":"assetId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES local_asset_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES local_asset_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"album_id","getter_name":"albumId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES local_album_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES local_album_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["asset_id","album_id"]}},{"id":6,"references":[0],"type":"table","data":{"name":"remote_asset_entity","was_declared_in_moor":false,"columns":[{"name":"name","getter_name":"name","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"type","getter_name":"type","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetType.values)","dart_type_name":"AssetType"}},{"name":"created_at","getter_name":"createdAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"updated_at","getter_name":"updatedAt","moor_type":"dateTime","nullable":false,"customConstraints":null,"default_dart":"const CustomExpression('CURRENT_TIMESTAMP')","default_client_dart":null,"dsl_features":[]},{"name":"duration_in_seconds","getter_name":"durationInSeconds","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"id","getter_name":"id","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"checksum","getter_name":"checksum","moor_type":"string","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"is_favorite","getter_name":"isFavorite","moor_type":"bool","nullable":false,"customConstraints":null,"defaultConstraints":"CHECK (\"is_favorite\" IN (0, 1))","dialectAwareDefaultConstraints":{"sqlite":"CHECK (\"is_favorite\" IN (0, 1))"},"default_dart":"const CustomExpression('0')","default_client_dart":null,"dsl_features":[]},{"name":"owner_id","getter_name":"ownerId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES user_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES user_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"local_date_time","getter_name":"localDateTime","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"thumb_hash","getter_name":"thumbHash","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"deleted_at","getter_name":"deletedAt","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"visibility","getter_name":"visibility","moor_type":"int","nullable":false,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[],"type_converter":{"dart_expr":"const EnumIndexConverter(AssetVisibility.values)","dart_type_name":"AssetVisibility"}}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["id"]}},{"id":7,"references":[6],"type":"table","data":{"name":"remote_exif_entity","was_declared_in_moor":false,"columns":[{"name":"asset_id","getter_name":"assetId","moor_type":"string","nullable":false,"customConstraints":null,"defaultConstraints":"REFERENCES remote_asset_entity (id) ON DELETE CASCADE","dialectAwareDefaultConstraints":{"sqlite":"REFERENCES remote_asset_entity (id) ON DELETE CASCADE"},"default_dart":null,"default_client_dart":null,"dsl_features":["unknown"]},{"name":"city","getter_name":"city","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"state","getter_name":"state","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"country","getter_name":"country","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"date_time_original","getter_name":"dateTimeOriginal","moor_type":"dateTime","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"description","getter_name":"description","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"height","getter_name":"height","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"width","getter_name":"width","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"exposure_time","getter_name":"exposureTime","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"f_number","getter_name":"fNumber","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"file_size","getter_name":"fileSize","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"focal_length","getter_name":"focalLength","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"latitude","getter_name":"latitude","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"longitude","getter_name":"longitude","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"iso","getter_name":"iso","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"make","getter_name":"make","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"model","getter_name":"model","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"orientation","getter_name":"orientation","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"time_zone","getter_name":"timeZone","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"rating","getter_name":"rating","moor_type":"int","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]},{"name":"projection_type","getter_name":"projectionType","moor_type":"string","nullable":true,"customConstraints":null,"default_dart":null,"default_client_dart":null,"dsl_features":[]}],"is_virtual":false,"without_rowid":true,"constraints":[],"strict":true,"explicit_pk":["asset_id"]}},{"id":8,"references":[4],"type":"index","data":{"on":4,"name":"idx_local_asset_checksum","sql":null,"unique":false,"columns":["checksum"]}},{"id":9,"references":[6],"type":"index","data":{"on":6,"name":"UQ_remote_asset_owner_checksum","sql":null,"unique":true,"columns":["checksum","owner_id"]}}]} \ No newline at end of file diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swift index 0d7a3026884..eb765337c3d 100644 --- a/mobile/ios/Runner/Sync/Messages.g.swift +++ b/mobile/ios/Runner/Sync/Messages.g.swift @@ -307,6 +307,7 @@ protocol NativeSyncApi { func getAlbums() throws -> [PlatformAlbum] func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64 func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset] + func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -442,5 +443,22 @@ class NativeSyncApiSetup { } else { getAssetsForAlbumChannel.setMessageHandler(nil) } + let hashPathsChannel = taskQueue == nil + ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) + if let api = api { + hashPathsChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let pathsArg = args[0] as! [String] + do { + let result = try api.hashPaths(paths: pathsArg) + reply(wrapResult(result)) + } catch { + reply(wrapError(error)) + } + } + } else { + hashPathsChannel.setMessageHandler(nil) + } } } diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift index 5d2f08691d2..06c958b88aa 100644 --- a/mobile/ios/Runner/Sync/MessagesImpl.swift +++ b/mobile/ios/Runner/Sync/MessagesImpl.swift @@ -1,4 +1,5 @@ import Photos +import CryptoKit struct AssetWrapper: Hashable, Equatable { let asset: PlatformAsset @@ -34,6 +35,8 @@ class NativeSyncApiImpl: NativeSyncApi { private let changeTokenKey = "immich:changeToken" private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum] + private let hashBufferSize = 2 * 1024 * 1024 + init(with defaults: UserDefaults = .standard) { self.defaults = defaults } @@ -243,4 +246,24 @@ class NativeSyncApiImpl: NativeSyncApi { } return assets } + + func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] { + return paths.map { path in + guard let file = FileHandle(forReadingAtPath: path) else { + print("Cannot open file: \(path)") + return nil + } + + var hasher = Insecure.SHA1() + while autoreleasepool(invoking: { + let chunk = file.readData(ofLength: hashBufferSize) + guard !chunk.isEmpty else { return false } + hasher.update(data: chunk) + return true + }) { } + + let digest = hasher.finalize() + return FlutterStandardTypedData(bytes: Data(digest)) + } + } } diff --git a/mobile/lib/domain/interfaces/local_album.interface.dart b/mobile/lib/domain/interfaces/local_album.interface.dart index 35cfad44552..d7b38c567f6 100644 --- a/mobile/lib/domain/interfaces/local_album.interface.dart +++ b/mobile/lib/domain/interfaces/local_album.interface.dart @@ -29,6 +29,8 @@ abstract interface class ILocalAlbumRepository implements IDatabaseRepository { String albumId, Iterable assetIdsToKeep, ); + + Future> getAssetsToHash(String albumId); } enum SortLocalAlbumsBy { id } diff --git a/mobile/lib/domain/interfaces/local_asset.interface.dart b/mobile/lib/domain/interfaces/local_asset.interface.dart new file mode 100644 index 00000000000..5792ebe5d92 --- /dev/null +++ b/mobile/lib/domain/interfaces/local_asset.interface.dart @@ -0,0 +1,6 @@ +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; + +abstract interface class ILocalAssetRepository implements IDatabaseRepository { + Future updateHashes(Iterable hashes); +} diff --git a/mobile/lib/domain/interfaces/storage.interface.dart b/mobile/lib/domain/interfaces/storage.interface.dart new file mode 100644 index 00000000000..ea6513e7f2a --- /dev/null +++ b/mobile/lib/domain/interfaces/storage.interface.dart @@ -0,0 +1,7 @@ +import 'dart:io'; + +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; + +abstract interface class IStorageRepository { + Future getFileForAsset(LocalAsset asset); +} diff --git a/mobile/lib/domain/models/local_album.model.dart b/mobile/lib/domain/models/local_album.model.dart index 95c56627bbd..ee27d91a718 100644 --- a/mobile/lib/domain/models/local_album.model.dart +++ b/mobile/lib/domain/models/local_album.model.dart @@ -1,13 +1,19 @@ enum BackupSelection { - none, - selected, - excluded, + none._(1), + selected._(0), + excluded._(2); + + // Used to sort albums based on the backupSelection + // selected -> none -> excluded + final int sortOrder; + const BackupSelection._(this.sortOrder); } class LocalAlbum { final String id; final String name; final DateTime updatedAt; + final bool isIosSharedAlbum; final int assetCount; final BackupSelection backupSelection; @@ -18,6 +24,7 @@ class LocalAlbum { required this.updatedAt, this.assetCount = 0, this.backupSelection = BackupSelection.none, + this.isIosSharedAlbum = false, }); LocalAlbum copyWith({ @@ -26,6 +33,7 @@ class LocalAlbum { DateTime? updatedAt, int? assetCount, BackupSelection? backupSelection, + bool? isIosSharedAlbum, }) { return LocalAlbum( id: id ?? this.id, @@ -33,6 +41,7 @@ class LocalAlbum { updatedAt: updatedAt ?? this.updatedAt, assetCount: assetCount ?? this.assetCount, backupSelection: backupSelection ?? this.backupSelection, + isIosSharedAlbum: isIosSharedAlbum ?? this.isIosSharedAlbum, ); } @@ -45,7 +54,8 @@ class LocalAlbum { other.name == name && other.updatedAt == updatedAt && other.assetCount == assetCount && - other.backupSelection == backupSelection; + other.backupSelection == backupSelection && + other.isIosSharedAlbum == isIosSharedAlbum; } @override @@ -54,7 +64,8 @@ class LocalAlbum { name.hashCode ^ updatedAt.hashCode ^ assetCount.hashCode ^ - backupSelection.hashCode; + backupSelection.hashCode ^ + isIosSharedAlbum.hashCode; } @override @@ -65,6 +76,7 @@ name: $name, updatedAt: $updatedAt, assetCount: $assetCount, backupSelection: $backupSelection, +isIosSharedAlbum: $isIosSharedAlbum }'''; } } diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dart new file mode 100644 index 00000000000..9820453db1b --- /dev/null +++ b/mobile/lib/domain/services/hash.service.dart @@ -0,0 +1,121 @@ +import 'dart:convert'; + +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/local_album.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart'; +import 'package:logging/logging.dart'; + +class HashService { + final int batchSizeLimit; + final int batchFileLimit; + final ILocalAlbumRepository _localAlbumRepository; + final ILocalAssetRepository _localAssetRepository; + final IStorageRepository _storageRepository; + final NativeSyncApi _nativeSyncApi; + final _log = Logger('HashService'); + + HashService({ + required ILocalAlbumRepository localAlbumRepository, + required ILocalAssetRepository localAssetRepository, + required IStorageRepository storageRepository, + required NativeSyncApi nativeSyncApi, + this.batchSizeLimit = kBatchHashSizeLimit, + this.batchFileLimit = kBatchHashFileLimit, + }) : _localAlbumRepository = localAlbumRepository, + _localAssetRepository = localAssetRepository, + _storageRepository = storageRepository, + _nativeSyncApi = nativeSyncApi; + + Future hashAssets() async { + final Stopwatch stopwatch = Stopwatch()..start(); + // Sorted by backupSelection followed by isCloud + final localAlbums = await _localAlbumRepository.getAll(); + localAlbums.sort((a, b) { + final backupComparison = + a.backupSelection.sortOrder.compareTo(b.backupSelection.sortOrder); + + if (backupComparison != 0) { + return backupComparison; + } + + // Local albums come before iCloud albums + return (a.isIosSharedAlbum ? 1 : 0).compareTo(b.isIosSharedAlbum ? 1 : 0); + }); + + for (final album in localAlbums) { + final assetsToHash = + await _localAlbumRepository.getAssetsToHash(album.id); + if (assetsToHash.isNotEmpty) { + await _hashAssets(assetsToHash); + } + } + + stopwatch.stop(); + _log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms"); + DLog.log("Hashing took - ${stopwatch.elapsedMilliseconds}ms"); + } + + /// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB + /// with hash for those that were successfully hashed. Hashes are looked up in a table + /// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB. + Future _hashAssets(List assetsToHash) async { + int bytesProcessed = 0; + final toHash = <_AssetToPath>[]; + + for (final asset in assetsToHash) { + final file = await _storageRepository.getFileForAsset(asset); + if (file == null) { + continue; + } + + bytesProcessed += await file.length(); + toHash.add(_AssetToPath(asset: asset, path: file.path)); + + if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) { + await _processBatch(toHash); + toHash.clear(); + bytesProcessed = 0; + } + } + + await _processBatch(toHash); + } + + /// Processes a batch of assets. + Future _processBatch(List<_AssetToPath> toHash) async { + if (toHash.isEmpty) { + return; + } + + _log.fine("Hashing ${toHash.length} files"); + + final hashed = []; + final hashes = + await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList()); + + for (final (index, hash) in hashes.indexed) { + final asset = toHash[index].asset; + if (hash?.length == 20) { + hashed.add(asset.copyWith(checksum: base64.encode(hash!))); + } else { + _log.warning("Failed to hash file ${asset.id}"); + } + } + + _log.fine("Hashed ${hashed.length}/${toHash.length} assets"); + DLog.log("Hashed ${hashed.length}/${toHash.length} assets"); + + await _localAssetRepository.updateHashes(hashed); + } +} + +class _AssetToPath { + final LocalAsset asset; + final String path; + + const _AssetToPath({required this.asset, required this.path}); +} diff --git a/mobile/lib/domain/services/local_sync.service.dart b/mobile/lib/domain/services/local_sync.service.dart index e07595b6dbd..e39999f2226 100644 --- a/mobile/lib/domain/services/local_sync.service.dart +++ b/mobile/lib/domain/services/local_sync.service.dart @@ -365,6 +365,7 @@ extension on Iterable { (e) => LocalAsset( id: e.id, name: e.name, + checksum: null, type: AssetType.values.elementAtOrNull(e.type) ?? AssetType.other, createdAt: e.createdAt == null ? DateTime.now() diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart index 6a694ee44ac..c8d2e2b624a 100644 --- a/mobile/lib/domain/utils/background_sync.dart +++ b/mobile/lib/domain/utils/background_sync.dart @@ -7,6 +7,7 @@ import 'package:worker_manager/worker_manager.dart'; class BackgroundSyncManager { Cancelable? _syncTask; Cancelable? _deviceAlbumSyncTask; + Cancelable? _hashTask; BackgroundSyncManager(); @@ -45,6 +46,20 @@ class BackgroundSyncManager { }); } +// No need to cancel the task, as it can also be run when the user logs out + Future hashAssets() { + if (_hashTask != null) { + return _hashTask!.future; + } + + _hashTask = runInIsolateGentle( + computation: (ref) => ref.read(hashServiceProvider).hashAssets(), + ); + return _hashTask!.whenComplete(() { + _hashTask = null; + }); + } + Future syncRemote() { if (_syncTask != null) { return _syncTask!.future; diff --git a/mobile/lib/infrastructure/entities/local_album.entity.dart b/mobile/lib/infrastructure/entities/local_album.entity.dart index 74c3e7a8f73..9657173c3cf 100644 --- a/mobile/lib/infrastructure/entities/local_album.entity.dart +++ b/mobile/lib/infrastructure/entities/local_album.entity.dart @@ -9,6 +9,8 @@ class LocalAlbumEntity extends Table with DriftDefaultsMixin { TextColumn get name => text()(); DateTimeColumn get updatedAt => dateTime().withDefault(currentDateAndTime)(); IntColumn get backupSelection => intEnum()(); + BoolColumn get isIosSharedAlbum => + boolean().withDefault(const Constant(false))(); // Used for mark & sweep BoolColumn get marker_ => boolean().nullable()(); diff --git a/mobile/lib/infrastructure/entities/local_album.entity.drift.dart b/mobile/lib/infrastructure/entities/local_album.entity.drift.dart index 5955742ec0e..ff6226ba3f2 100644 --- a/mobile/lib/infrastructure/entities/local_album.entity.drift.dart +++ b/mobile/lib/infrastructure/entities/local_album.entity.drift.dart @@ -14,6 +14,7 @@ typedef $$LocalAlbumEntityTableCreateCompanionBuilder required String name, i0.Value updatedAt, required i2.BackupSelection backupSelection, + i0.Value isIosSharedAlbum, i0.Value marker_, }); typedef $$LocalAlbumEntityTableUpdateCompanionBuilder @@ -22,6 +23,7 @@ typedef $$LocalAlbumEntityTableUpdateCompanionBuilder i0.Value name, i0.Value updatedAt, i0.Value backupSelection, + i0.Value isIosSharedAlbum, i0.Value marker_, }); @@ -48,6 +50,10 @@ class $$LocalAlbumEntityTableFilterComposer column: $table.backupSelection, builder: (column) => i0.ColumnWithTypeConverterFilters(column)); + i0.ColumnFilters get isIosSharedAlbum => $composableBuilder( + column: $table.isIosSharedAlbum, + builder: (column) => i0.ColumnFilters(column)); + i0.ColumnFilters get marker_ => $composableBuilder( column: $table.marker_, builder: (column) => i0.ColumnFilters(column)); } @@ -75,6 +81,10 @@ class $$LocalAlbumEntityTableOrderingComposer column: $table.backupSelection, builder: (column) => i0.ColumnOrderings(column)); + i0.ColumnOrderings get isIosSharedAlbum => $composableBuilder( + column: $table.isIosSharedAlbum, + builder: (column) => i0.ColumnOrderings(column)); + i0.ColumnOrderings get marker_ => $composableBuilder( column: $table.marker_, builder: (column) => i0.ColumnOrderings(column)); } @@ -101,6 +111,9 @@ class $$LocalAlbumEntityTableAnnotationComposer get backupSelection => $composableBuilder( column: $table.backupSelection, builder: (column) => column); + i0.GeneratedColumn get isIosSharedAlbum => $composableBuilder( + column: $table.isIosSharedAlbum, builder: (column) => column); + i0.GeneratedColumn get marker_ => $composableBuilder(column: $table.marker_, builder: (column) => column); } @@ -139,6 +152,7 @@ class $$LocalAlbumEntityTableTableManager extends i0.RootTableManager< i0.Value updatedAt = const i0.Value.absent(), i0.Value backupSelection = const i0.Value.absent(), + i0.Value isIosSharedAlbum = const i0.Value.absent(), i0.Value marker_ = const i0.Value.absent(), }) => i1.LocalAlbumEntityCompanion( @@ -146,6 +160,7 @@ class $$LocalAlbumEntityTableTableManager extends i0.RootTableManager< name: name, updatedAt: updatedAt, backupSelection: backupSelection, + isIosSharedAlbum: isIosSharedAlbum, marker_: marker_, ), createCompanionCallback: ({ @@ -153,6 +168,7 @@ class $$LocalAlbumEntityTableTableManager extends i0.RootTableManager< required String name, i0.Value updatedAt = const i0.Value.absent(), required i2.BackupSelection backupSelection, + i0.Value isIosSharedAlbum = const i0.Value.absent(), i0.Value marker_ = const i0.Value.absent(), }) => i1.LocalAlbumEntityCompanion.insert( @@ -160,6 +176,7 @@ class $$LocalAlbumEntityTableTableManager extends i0.RootTableManager< name: name, updatedAt: updatedAt, backupSelection: backupSelection, + isIosSharedAlbum: isIosSharedAlbum, marker_: marker_, ), withReferenceMapper: (p0) => p0 @@ -218,6 +235,16 @@ class $LocalAlbumEntityTable extends i3.LocalAlbumEntity type: i0.DriftSqlType.int, requiredDuringInsert: true) .withConverter( i1.$LocalAlbumEntityTable.$converterbackupSelection); + static const i0.VerificationMeta _isIosSharedAlbumMeta = + const i0.VerificationMeta('isIosSharedAlbum'); + @override + late final i0.GeneratedColumn isIosSharedAlbum = + i0.GeneratedColumn('is_ios_shared_album', aliasedName, false, + type: i0.DriftSqlType.bool, + requiredDuringInsert: false, + defaultConstraints: i0.GeneratedColumn.constraintIsAlways( + 'CHECK ("is_ios_shared_album" IN (0, 1))'), + defaultValue: const i4.Constant(false)); static const i0.VerificationMeta _marker_Meta = const i0.VerificationMeta('marker_'); @override @@ -229,7 +256,7 @@ class $LocalAlbumEntityTable extends i3.LocalAlbumEntity i0.GeneratedColumn.constraintIsAlways('CHECK ("marker" IN (0, 1))')); @override List get $columns => - [id, name, updatedAt, backupSelection, marker_]; + [id, name, updatedAt, backupSelection, isIosSharedAlbum, marker_]; @override String get aliasedName => _alias ?? actualTableName; @override @@ -256,6 +283,12 @@ class $LocalAlbumEntityTable extends i3.LocalAlbumEntity context.handle(_updatedAtMeta, updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta)); } + if (data.containsKey('is_ios_shared_album')) { + context.handle( + _isIosSharedAlbumMeta, + isIosSharedAlbum.isAcceptableOrUnknown( + data['is_ios_shared_album']!, _isIosSharedAlbumMeta)); + } if (data.containsKey('marker')) { context.handle(_marker_Meta, marker_.isAcceptableOrUnknown(data['marker']!, _marker_Meta)); @@ -279,6 +312,8 @@ class $LocalAlbumEntityTable extends i3.LocalAlbumEntity backupSelection: i1.$LocalAlbumEntityTable.$converterbackupSelection .fromSql(attachedDatabase.typeMapping.read(i0.DriftSqlType.int, data['${effectivePrefix}backup_selection'])!), + isIosSharedAlbum: attachedDatabase.typeMapping.read( + i0.DriftSqlType.bool, data['${effectivePrefix}is_ios_shared_album'])!, marker_: attachedDatabase.typeMapping .read(i0.DriftSqlType.bool, data['${effectivePrefix}marker']), ); @@ -305,12 +340,14 @@ class LocalAlbumEntityData extends i0.DataClass final String name; final DateTime updatedAt; final i2.BackupSelection backupSelection; + final bool isIosSharedAlbum; final bool? marker_; const LocalAlbumEntityData( {required this.id, required this.name, required this.updatedAt, required this.backupSelection, + required this.isIosSharedAlbum, this.marker_}); @override Map toColumns(bool nullToAbsent) { @@ -323,6 +360,7 @@ class LocalAlbumEntityData extends i0.DataClass .$LocalAlbumEntityTable.$converterbackupSelection .toSql(backupSelection)); } + map['is_ios_shared_album'] = i0.Variable(isIosSharedAlbum); if (!nullToAbsent || marker_ != null) { map['marker'] = i0.Variable(marker_); } @@ -338,6 +376,7 @@ class LocalAlbumEntityData extends i0.DataClass updatedAt: serializer.fromJson(json['updatedAt']), backupSelection: i1.$LocalAlbumEntityTable.$converterbackupSelection .fromJson(serializer.fromJson(json['backupSelection'])), + isIosSharedAlbum: serializer.fromJson(json['isIosSharedAlbum']), marker_: serializer.fromJson(json['marker_']), ); } @@ -351,6 +390,7 @@ class LocalAlbumEntityData extends i0.DataClass 'backupSelection': serializer.toJson(i1 .$LocalAlbumEntityTable.$converterbackupSelection .toJson(backupSelection)), + 'isIosSharedAlbum': serializer.toJson(isIosSharedAlbum), 'marker_': serializer.toJson(marker_), }; } @@ -360,12 +400,14 @@ class LocalAlbumEntityData extends i0.DataClass String? name, DateTime? updatedAt, i2.BackupSelection? backupSelection, + bool? isIosSharedAlbum, i0.Value marker_ = const i0.Value.absent()}) => i1.LocalAlbumEntityData( id: id ?? this.id, name: name ?? this.name, updatedAt: updatedAt ?? this.updatedAt, backupSelection: backupSelection ?? this.backupSelection, + isIosSharedAlbum: isIosSharedAlbum ?? this.isIosSharedAlbum, marker_: marker_.present ? marker_.value : this.marker_, ); LocalAlbumEntityData copyWithCompanion(i1.LocalAlbumEntityCompanion data) { @@ -376,6 +418,9 @@ class LocalAlbumEntityData extends i0.DataClass backupSelection: data.backupSelection.present ? data.backupSelection.value : this.backupSelection, + isIosSharedAlbum: data.isIosSharedAlbum.present + ? data.isIosSharedAlbum.value + : this.isIosSharedAlbum, marker_: data.marker_.present ? data.marker_.value : this.marker_, ); } @@ -387,14 +432,15 @@ class LocalAlbumEntityData extends i0.DataClass ..write('name: $name, ') ..write('updatedAt: $updatedAt, ') ..write('backupSelection: $backupSelection, ') + ..write('isIosSharedAlbum: $isIosSharedAlbum, ') ..write('marker_: $marker_') ..write(')')) .toString(); } @override - int get hashCode => - Object.hash(id, name, updatedAt, backupSelection, marker_); + int get hashCode => Object.hash( + id, name, updatedAt, backupSelection, isIosSharedAlbum, marker_); @override bool operator ==(Object other) => identical(this, other) || @@ -403,6 +449,7 @@ class LocalAlbumEntityData extends i0.DataClass other.name == this.name && other.updatedAt == this.updatedAt && other.backupSelection == this.backupSelection && + other.isIosSharedAlbum == this.isIosSharedAlbum && other.marker_ == this.marker_); } @@ -412,12 +459,14 @@ class LocalAlbumEntityCompanion final i0.Value name; final i0.Value updatedAt; final i0.Value backupSelection; + final i0.Value isIosSharedAlbum; final i0.Value marker_; const LocalAlbumEntityCompanion({ this.id = const i0.Value.absent(), this.name = const i0.Value.absent(), this.updatedAt = const i0.Value.absent(), this.backupSelection = const i0.Value.absent(), + this.isIosSharedAlbum = const i0.Value.absent(), this.marker_ = const i0.Value.absent(), }); LocalAlbumEntityCompanion.insert({ @@ -425,6 +474,7 @@ class LocalAlbumEntityCompanion required String name, this.updatedAt = const i0.Value.absent(), required i2.BackupSelection backupSelection, + this.isIosSharedAlbum = const i0.Value.absent(), this.marker_ = const i0.Value.absent(), }) : id = i0.Value(id), name = i0.Value(name), @@ -434,6 +484,7 @@ class LocalAlbumEntityCompanion i0.Expression? name, i0.Expression? updatedAt, i0.Expression? backupSelection, + i0.Expression? isIosSharedAlbum, i0.Expression? marker_, }) { return i0.RawValuesInsertable({ @@ -441,6 +492,7 @@ class LocalAlbumEntityCompanion if (name != null) 'name': name, if (updatedAt != null) 'updated_at': updatedAt, if (backupSelection != null) 'backup_selection': backupSelection, + if (isIosSharedAlbum != null) 'is_ios_shared_album': isIosSharedAlbum, if (marker_ != null) 'marker': marker_, }); } @@ -450,12 +502,14 @@ class LocalAlbumEntityCompanion i0.Value? name, i0.Value? updatedAt, i0.Value? backupSelection, + i0.Value? isIosSharedAlbum, i0.Value? marker_}) { return i1.LocalAlbumEntityCompanion( id: id ?? this.id, name: name ?? this.name, updatedAt: updatedAt ?? this.updatedAt, backupSelection: backupSelection ?? this.backupSelection, + isIosSharedAlbum: isIosSharedAlbum ?? this.isIosSharedAlbum, marker_: marker_ ?? this.marker_, ); } @@ -477,6 +531,9 @@ class LocalAlbumEntityCompanion .$LocalAlbumEntityTable.$converterbackupSelection .toSql(backupSelection.value)); } + if (isIosSharedAlbum.present) { + map['is_ios_shared_album'] = i0.Variable(isIosSharedAlbum.value); + } if (marker_.present) { map['marker'] = i0.Variable(marker_.value); } @@ -490,6 +547,7 @@ class LocalAlbumEntityCompanion ..write('name: $name, ') ..write('updatedAt: $updatedAt, ') ..write('backupSelection: $backupSelection, ') + ..write('isIosSharedAlbum: $isIosSharedAlbum, ') ..write('marker_: $marker_') ..write(')')) .toString(); diff --git a/mobile/lib/infrastructure/repositories/local_album.repository.dart b/mobile/lib/infrastructure/repositories/local_album.repository.dart index 650b7a1aab1..5100b7a192d 100644 --- a/mobile/lib/infrastructure/repositories/local_album.repository.dart +++ b/mobile/lib/infrastructure/repositories/local_album.repository.dart @@ -98,12 +98,24 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository name: localAlbum.name, updatedAt: Value(localAlbum.updatedAt), backupSelection: localAlbum.backupSelection, + isIosSharedAlbum: Value(localAlbum.isIosSharedAlbum), ); return _db.transaction(() async { await _db.localAlbumEntity .insertOne(companion, onConflict: DoUpdate((_) => companion)); - await _addAssets(localAlbum.id, toUpsert); + if (toUpsert.isNotEmpty) { + await _upsertAssets(toUpsert); + await _db.localAlbumAssetEntity.insertAll( + toUpsert.map( + (a) => LocalAlbumAssetEntityCompanion.insert( + assetId: a.id, + albumId: localAlbum.id, + ), + ), + mode: InsertMode.insertOrIgnore, + ); + } await _removeAssets(localAlbum.id, toDelete); }); } @@ -122,6 +134,7 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository name: album.name, updatedAt: Value(album.updatedAt), backupSelection: album.backupSelection, + isIosSharedAlbum: Value(album.isIosSharedAlbum), marker_: const Value(null), ); @@ -226,21 +239,52 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository }); } - Future _addAssets(String albumId, Iterable assets) { - if (assets.isEmpty) { + @override + Future> getAssetsToHash(String albumId) { + final query = _db.localAlbumAssetEntity.select().join( + [ + innerJoin( + _db.localAssetEntity, + _db.localAlbumAssetEntity.assetId.equalsExp(_db.localAssetEntity.id), + ), + ], + ) + ..where( + _db.localAlbumAssetEntity.albumId.equals(albumId) & + _db.localAssetEntity.checksum.isNull(), + ) + ..orderBy([OrderingTerm.asc(_db.localAssetEntity.id)]); + + return query + .map((row) => row.readTable(_db.localAssetEntity).toDto()) + .get(); + } + + Future _upsertAssets(Iterable localAssets) { + if (localAssets.isEmpty) { return Future.value(); } - return transaction(() async { - await _upsertAssets(assets); - await _db.localAlbumAssetEntity.insertAll( - assets.map( - (a) => LocalAlbumAssetEntityCompanion.insert( - assetId: a.id, - albumId: albumId, + + return _db.batch((batch) async { + for (final asset in localAssets) { + final companion = LocalAssetEntityCompanion.insert( + name: asset.name, + type: asset.type, + createdAt: Value(asset.createdAt), + updatedAt: Value(asset.updatedAt), + durationInSeconds: Value.absentIfNull(asset.durationInSeconds), + id: asset.id, + checksum: const Value(null), + ); + batch.insert<$LocalAssetEntityTable, LocalAssetEntityData>( + _db.localAssetEntity, + companion, + onConflict: DoUpdate( + (_) => companion, + where: (old) => old.updatedAt.isNotValue(asset.updatedAt), ), - ), - mode: InsertMode.insertOrIgnore, - ); + ); + } }); } @@ -301,40 +345,14 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository return query.map((row) => row.read(assetId)!).get(); } - Future _upsertAssets(Iterable localAssets) { - if (localAssets.isEmpty) { - return Future.value(); - } - - return _db.batch((batch) async { - batch.insertAllOnConflictUpdate( - _db.localAssetEntity, - localAssets.map( - (a) => LocalAssetEntityCompanion.insert( - name: a.name, - type: a.type, - createdAt: Value(a.createdAt), - updatedAt: Value(a.updatedAt), - durationInSeconds: Value.absentIfNull(a.durationInSeconds), - id: a.id, - checksum: Value.absentIfNull(a.checksum), - ), - ), - ); - }); - } - Future _deleteAssets(Iterable ids) { if (ids.isEmpty) { return Future.value(); } - return _db.batch( - (batch) => batch.deleteWhere( - _db.localAssetEntity, - (f) => f.id.isIn(ids), - ), - ); + return _db.batch((batch) { + batch.deleteWhere(_db.localAssetEntity, (f) => f.id.isIn(ids)); + }); } } diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dart new file mode 100644 index 00000000000..350a8dcd32e --- /dev/null +++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart @@ -0,0 +1,28 @@ +import 'package:drift/drift.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; + +class DriftLocalAssetRepository extends DriftDatabaseRepository + implements ILocalAssetRepository { + final Drift _db; + const DriftLocalAssetRepository(this._db) : super(_db); + + @override + Future updateHashes(Iterable hashes) { + if (hashes.isEmpty) { + return Future.value(); + } + + return _db.batch((batch) async { + for (final asset in hashes) { + batch.update( + _db.localAssetEntity, + LocalAssetEntityCompanion(checksum: Value(asset.checksum)), + where: (e) => e.id.equals(asset.id), + ); + } + }); + } +} diff --git a/mobile/lib/infrastructure/repositories/storage.repository.dart b/mobile/lib/infrastructure/repositories/storage.repository.dart new file mode 100644 index 00000000000..57dfc42135a --- /dev/null +++ b/mobile/lib/infrastructure/repositories/storage.repository.dart @@ -0,0 +1,31 @@ +import 'dart:io'; + +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; + +class StorageRepository implements IStorageRepository { + final _log = Logger('StorageRepository'); + + @override + Future getFileForAsset(LocalAsset asset) async { + File? file; + try { + final entity = await AssetEntity.fromId(asset.id); + file = await entity?.originFile; + if (file == null) { + _log.warning( + "Cannot get file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + ); + } + } catch (error, stackTrace) { + _log.warning( + "Error getting file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + error, + stackTrace, + ); + } + return file; + } +} diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 32bb025916d..46955404247 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -18,8 +18,8 @@ import 'package:immich_mobile/providers/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/locale_provider.dart'; import 'package:immich_mobile/providers/theme.provider.dart'; -import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/app_navigation_observer.dart'; +import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/background.service.dart'; import 'package:immich_mobile/services/local_notification.service.dart'; import 'package:immich_mobile/theme/dynamic_theme.dart'; diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dart index c4e4c467d41..ffcef67962e 100644 --- a/mobile/lib/platform/native_sync_api.g.dart +++ b/mobile/lib/platform/native_sync_api.g.dart @@ -498,4 +498,35 @@ class NativeSyncApi { return (pigeonVar_replyList[0] as List?)!.cast(); } } + + Future> hashPaths(List paths) async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = + BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = + pigeonVar_channel.send([paths]); + final List? pigeonVar_replyList = + await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as List?)!.cast(); + } + } } diff --git a/mobile/lib/presentation/pages/dev/dev_logger.dart b/mobile/lib/presentation/pages/dev/dev_logger.dart index 6d179241a48..ab9849f87c1 100644 --- a/mobile/lib/presentation/pages/dev/dev_logger.dart +++ b/mobile/lib/presentation/pages/dev/dev_logger.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/models/log.model.dart'; @@ -15,7 +16,6 @@ abstract final class DLog { static Stream> watchLog() { final db = Isar.getInstance(); if (db == null) { - debugPrint('Isar is not initialized'); return const Stream.empty(); } @@ -30,7 +30,6 @@ abstract final class DLog { static void clearLog() { final db = Isar.getInstance(); if (db == null) { - debugPrint('Isar is not initialized'); return; } @@ -40,7 +39,9 @@ abstract final class DLog { } static void log(String message, [Object? error, StackTrace? stackTrace]) { - debugPrint('[$kDevLoggerTag] [${DateTime.now()}] $message'); + if (!Platform.environment.containsKey('FLUTTER_TEST')) { + debugPrint('[$kDevLoggerTag] [${DateTime.now()}] $message'); + } if (error != null) { debugPrint('Error: $error'); } @@ -50,7 +51,6 @@ abstract final class DLog { final isar = Isar.getInstance(); if (isar == null) { - debugPrint('Isar is not initialized'); return; } diff --git a/mobile/lib/presentation/pages/dev/feat_in_development.page.dart b/mobile/lib/presentation/pages/dev/feat_in_development.page.dart index 3ff0b12b959..edbbd23796a 100644 --- a/mobile/lib/presentation/pages/dev/feat_in_development.page.dart +++ b/mobile/lib/presentation/pages/dev/feat_in_development.page.dart @@ -26,6 +26,11 @@ final _features = [ icon: Icons.photo_library_rounded, onTap: (_, ref) => ref.read(backgroundSyncProvider).syncLocal(full: true), ), + _Feature( + name: 'Hash Local Assets', + icon: Icons.numbers_outlined, + onTap: (_, ref) => ref.read(backgroundSyncProvider).hashAssets(), + ), _Feature( name: 'Sync Remote', icon: Icons.refresh_rounded, diff --git a/mobile/lib/presentation/pages/dev/media_stat.page.dart b/mobile/lib/presentation/pages/dev/media_stat.page.dart index 5debeff31d8..c074e524bf6 100644 --- a/mobile/lib/presentation/pages/dev/media_stat.page.dart +++ b/mobile/lib/presentation/pages/dev/media_stat.page.dart @@ -4,7 +4,6 @@ import 'package:auto_route/auto_route.dart'; import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; -import 'package:immich_mobile/domain/models/local_album.model.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/providers/infrastructure/album.provider.dart'; @@ -94,9 +93,8 @@ class LocalMediaSummaryPage extends StatelessWidget { ), FutureBuilder( future: albumsFuture, - initialData: [], builder: (_, snap) { - final albums = snap.data!; + final albums = snap.data ?? []; if (albums.isEmpty) { return const SliverToBoxAdapter(child: SizedBox.shrink()); } diff --git a/mobile/lib/providers/infrastructure/asset.provider.dart b/mobile/lib/providers/infrastructure/asset.provider.dart new file mode 100644 index 00000000000..d714571473f --- /dev/null +++ b/mobile/lib/providers/infrastructure/asset.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; + +final localAssetRepository = Provider( + (ref) => DriftLocalAssetRepository(ref.watch(driftProvider)), +); diff --git a/mobile/lib/providers/infrastructure/storage.provider.dart b/mobile/lib/providers/infrastructure/storage.provider.dart new file mode 100644 index 00000000000..d8ac79f1c14 --- /dev/null +++ b/mobile/lib/providers/infrastructure/storage.provider.dart @@ -0,0 +1,7 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; + +final storageRepositoryProvider = Provider( + (ref) => StorageRepository(), +); diff --git a/mobile/lib/providers/infrastructure/sync.provider.dart b/mobile/lib/providers/infrastructure/sync.provider.dart index 96e470eba26..359af63232f 100644 --- a/mobile/lib/providers/infrastructure/sync.provider.dart +++ b/mobile/lib/providers/infrastructure/sync.provider.dart @@ -1,13 +1,16 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/services/hash.service.dart'; import 'package:immich_mobile/domain/services/local_sync.service.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart'; import 'package:immich_mobile/providers/api.provider.dart'; import 'package:immich_mobile/providers/infrastructure/album.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/asset.provider.dart'; import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/storage.provider.dart'; import 'package:immich_mobile/providers/infrastructure/store.provider.dart'; final syncStreamServiceProvider = Provider( @@ -33,3 +36,12 @@ final localSyncServiceProvider = Provider( storeService: ref.watch(storeServiceProvider), ), ); + +final hashServiceProvider = Provider( + (ref) => HashService( + localAlbumRepository: ref.watch(localAlbumRepository), + localAssetRepository: ref.watch(localAssetRepository), + storageRepository: ref.watch(storageRepositoryProvider), + nativeSyncApi: ref.watch(nativeSyncApiProvider), + ), +); diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 4519c6d803e..a31e441b1f5 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -1,10 +1,13 @@ // ignore_for_file: avoid-unsafe-collection-methods import 'dart:async'; +import 'dart:convert'; import 'dart:io'; +import 'package:drift/drift.dart'; import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/android_device_asset.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -13,14 +16,16 @@ import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; +import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart'; import 'package:immich_mobile/infrastructure/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/user.entity.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/utils/diff.dart'; import 'package:isar/isar.dart'; // ignore: import_rule_photo_manager import 'package:photo_manager/photo_manager.dart'; -const int targetVersion = 11; +const int targetVersion = 12; Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, targetVersion); @@ -45,7 +50,15 @@ Future migrateDatabaseIfNeeded(Isar db) async { await _migrateDeviceAsset(db); } - final shouldTruncate = version < 8 && version < targetVersion; + if (version < 12 && (!kReleaseMode)) { + final backgroundSync = BackgroundSyncManager(); + await backgroundSync.syncLocal(); + final drift = Drift(); + await _migrateDeviceAssetToSqlite(db, drift); + await drift.close(); + } + + final shouldTruncate = version < 8 || version < targetVersion; if (shouldTruncate) { await _migrateTo(db, targetVersion); } @@ -154,6 +167,28 @@ Future _migrateDeviceAsset(Isar db) async { }); } +Future _migrateDeviceAssetToSqlite(Isar db, Drift drift) async { + final isarDeviceAssets = + await db.deviceAssetEntitys.where().sortByAssetId().findAll(); + await drift.batch((batch) { + for (final deviceAsset in isarDeviceAssets) { + final companion = LocalAssetEntityCompanion( + updatedAt: Value(deviceAsset.modifiedTime), + id: Value(deviceAsset.assetId), + checksum: Value(base64.encode(deviceAsset.hash)), + ); + batch.insert<$LocalAssetEntityTable, LocalAssetEntityData>( + drift.localAssetEntity, + companion, + onConflict: DoUpdate( + (_) => companion, + where: (old) => old.updatedAt.equals(deviceAsset.modifiedTime), + ), + ); + } + }); +} + class _DeviceAsset { final String assetId; final List? hash; diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart index b8a7500d6e5..9bcb816a64c 100644 --- a/mobile/pigeon/native_sync_api.dart +++ b/mobile/pigeon/native_sync_api.dart @@ -86,4 +86,7 @@ abstract class NativeSyncApi { @TaskQueue(type: TaskQueueType.serialBackgroundThread) List getAssetsForAlbum(String albumId, {int? updatedTimeCond}); + + @TaskQueue(type: TaskQueueType.serialBackgroundThread) + List hashPaths(List paths); } diff --git a/mobile/test/domain/service.mock.dart b/mobile/test/domain/service.mock.dart index 97a3f30294b..f4c5a32a4b3 100644 --- a/mobile/test/domain/service.mock.dart +++ b/mobile/test/domain/service.mock.dart @@ -1,6 +1,7 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/domain/services/user.service.dart'; import 'package:immich_mobile/domain/utils/background_sync.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; import 'package:mocktail/mocktail.dart'; class MockStoreService extends Mock implements StoreService {} @@ -8,3 +9,5 @@ class MockStoreService extends Mock implements StoreService {} class MockUserService extends Mock implements UserService {} class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {} + +class MockNativeSyncApi extends Mock implements NativeSyncApi {} diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart index 2da41cd704e..1401f5d2a0c 100644 --- a/mobile/test/domain/services/hash_service_test.dart +++ b/mobile/test/domain/services/hash_service_test.dart @@ -1,425 +1,292 @@ import 'dart:convert'; import 'dart:io'; -import 'dart:math'; +import 'dart:typed_data'; -import 'package:collection/collection.dart'; -import 'package:file/memory.dart'; -import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; -import 'package:immich_mobile/domain/models/device_asset.model.dart'; -import 'package:immich_mobile/entities/asset.entity.dart'; -import 'package:immich_mobile/services/background.service.dart'; -import 'package:immich_mobile/services/hash.service.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/domain/models/local_album.model.dart'; +import 'package:immich_mobile/domain/services/hash.service.dart'; import 'package:mocktail/mocktail.dart'; -import 'package:photo_manager/photo_manager.dart'; +import '../../fixtures/album.stub.dart'; import '../../fixtures/asset.stub.dart'; import '../../infrastructure/repository.mock.dart'; -import '../../service.mocks.dart'; +import '../service.mock.dart'; -class MockAsset extends Mock implements Asset {} - -class MockAssetEntity extends Mock implements AssetEntity {} +class MockFile extends Mock implements File {} void main() { late HashService sut; - late BackgroundService mockBackgroundService; - late IDeviceAssetRepository mockDeviceAssetRepository; + late MockLocalAlbumRepository mockAlbumRepo; + late MockLocalAssetRepository mockAssetRepo; + late MockStorageRepository mockStorageRepo; + late MockNativeSyncApi mockNativeApi; setUp(() { - mockBackgroundService = MockBackgroundService(); - mockDeviceAssetRepository = MockDeviceAssetRepository(); + mockAlbumRepo = MockLocalAlbumRepository(); + mockAssetRepo = MockLocalAssetRepository(); + mockStorageRepo = MockStorageRepository(); + mockNativeApi = MockNativeSyncApi(); sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, ); - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?)?.call(); - }); - when(() => mockDeviceAssetRepository.updateAll(any())) - .thenAnswer((_) async => true); - when(() => mockDeviceAssetRepository.deleteIds(any())) - .thenAnswer((_) async => true); + registerFallbackValue(LocalAlbumStub.recent); + registerFallbackValue(LocalAssetStub.image1); + + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); }); - group("HashService: No DeviceAsset entry", () { - test("hash successfully", () async { - final (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); - - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - // No DB entries for this asset - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => []); - - final result = await sut.hashAssets([mockAsset]); - - // Verify we stored the new hash in DB - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - verify( - () => mockDeviceAssetRepository.updateAll([ - deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), - ]), - ).called(1); - verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); - }); - expect( - result, - [AssetStub.image1.copyWith(checksum: base64.encode(hash))], + group('HashService hashAssets', () { + test('processes albums in correct order', () async { + final album1 = LocalAlbumStub.recent + .copyWith(id: "1", backupSelection: BackupSelection.none); + final album2 = LocalAlbumStub.recent + .copyWith(id: "2", backupSelection: BackupSelection.excluded); + final album3 = LocalAlbumStub.recent + .copyWith(id: "3", backupSelection: BackupSelection.selected); + final album4 = LocalAlbumStub.recent.copyWith( + id: "4", + backupSelection: BackupSelection.selected, + isIosSharedAlbum: true, ); - }); - }); - group("HashService: Has DeviceAsset entry", () { - test("when the asset is not modified", () async { - final hash = utf8.encode("image1-hash"); + when(() => mockAlbumRepo.getAll()) + .thenAnswer((_) async => [album1, album2, album4, album3]); + when(() => mockAlbumRepo.getAssetsToHash(any())) + .thenAnswer((_) async => []); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer( - (_) async => [ - DeviceAsset( - assetId: AssetStub.image1.localId!, - hash: hash, - modifiedTime: AssetStub.image1.fileModifiedAt, - ), - ], - ); - final result = await sut.hashAssets([AssetStub.image1]); + await sut.hashAssets(); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockBackgroundService.digestFile(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); - - expect(result, [ - AssetStub.image1.copyWith(checksum: base64.encode(hash)), + verifyInOrder([ + () => mockAlbumRepo.getAll(), + () => mockAlbumRepo.getAssetsToHash(album3.id), + () => mockAlbumRepo.getAssetsToHash(album4.id), + () => mockAlbumRepo.getAssetsToHash(album1.id), + () => mockAlbumRepo.getAssetsToHash(album2.id), ]); }); - test("hashed successful when asset is modified", () async { - final (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); + test('skips albums with no assets to hash', () async { + when(() => mockAlbumRepo.getAll()).thenAnswer( + (_) async => [LocalAlbumStub.recent.copyWith(assetCount: 0)], + ); + when(() => mockAlbumRepo.getAssetsToHash(LocalAlbumStub.recent.id)) + .thenAnswer((_) async => []); - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => [deviceAsset]); + await sut.hashAssets(); - final result = await sut.hashAssets([mockAsset]); - - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - verify( - () => mockDeviceAssetRepository.updateAll([ - deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), - ]), - ).called(1); - verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); - }); - - verify(() => mockBackgroundService.digestFiles([file.path])).called(1); - - expect(result, [ - AssetStub.image1.copyWith(checksum: base64.encode(hash)), - ]); + verifyNever(() => mockStorageRepo.getFileForAsset(any())); + verifyNever(() => mockNativeApi.hashPaths(any())); }); }); - group("HashService: Cleanup", () { - late Asset mockAsset; - late Uint8List hash; - late DeviceAsset deviceAsset; - late File file; + group('HashService _hashAssets', () { + test('skips assets without files', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => null); - setUp(() async { - (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); + await sut.hashAssets(); - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => [deviceAsset]); + verifyNever(() => mockNativeApi.hashPaths(any())); }); - test("cleanups DeviceAsset when local file cannot be obtained", () async { - when(() => mockAsset.local).thenThrow(Exception("File not found")); - final result = await sut.hashAssets([mockAsset]); + test('processes assets when available', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + final hash = Uint8List.fromList(List.generate(20, (i) => i)); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockBackgroundService.digestFile(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verify( - () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), - ).called(1); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); - expect(result, isEmpty); - }); - - test("cleanups DeviceAsset when hashing failed", () async { - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - - // Verify the callback inside the transaction because, doing it outside results - // in a small delay before the callback is invoked, resulting in other LOCs getting executed - // resulting in an incorrect state - // - // i.e, consider the following piece of code - // await _deviceAssetRepository.transaction(() async { - // await _deviceAssetRepository.updateAll(toBeAdded); - // await _deviceAssetRepository.deleteIds(toBeDeleted); - // }); - // toBeDeleted.clear(); - // since the transaction method is mocked, the callback is not invoked until it is captured - // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed - // immediately once the transaction stub is executed, resulting in the deleteIds method being - // called with an empty list. - // - // To avoid this, we capture the callback and execute it within the transaction stub itself - // and verify the results inside the transaction stub - verify(() => mockDeviceAssetRepository.updateAll([])).called(1); - verify( - () => - mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), - ).called(1); - }); - - when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer( - // Invalid hash, length != 20 - (_) async => [Uint8List.fromList(hash.slice(2).toList())], + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer( + (_) async => [hash], ); - final result = await sut.hashAssets([mockAsset]); + await sut.hashAssets(); - verify(() => mockBackgroundService.digestFiles([file.path])).called(1); - expect(result, isEmpty); - }); - }); - - group("HashService: Batch processing", () { - test("processes assets in batches when size limit is reached", () async { - // Setup multiple assets with large file sizes - final (mock1, mock2, mock3) = await ( - _createAssetMock(AssetStub.image1), - _createAssetMock(AssetStub.image2), - _createAssetMock(AssetStub.image3), - ).wait; - - final (asset1, file1, deviceAsset1, hash1) = mock1; - final (asset2, file2, deviceAsset2, hash2) = mock2; - final (asset3, file3, deviceAsset3, hash3) = mock3; - - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); - - // Setup for multiple batch processing calls - when(() => mockBackgroundService.digestFiles([file1.path, file2.path])) - .thenAnswer((_) async => [hash1, hash2]); - when(() => mockBackgroundService.digestFiles([file3.path])) - .thenAnswer((_) async => [hash3]); - - final size = await file1.length() + await file2.length(); - - sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, - batchSizeLimit: size, - ); - final result = await sut.hashAssets([asset1, asset2, asset3]); - - // Verify multiple batch process calls - verify(() => mockBackgroundService.digestFiles([file1.path, file2.path])) - .called(1); - verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); - - expect( - result, - [ - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ], - ); + verify(() => mockNativeApi.hashPaths(['image-path'])).called(1); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 1); + expect(captured[0].checksum, base64.encode(hash)); }); - test("processes assets in batches when file limit is reached", () async { - // Setup multiple assets with large file sizes - final (mock1, mock2, mock3) = await ( - _createAssetMock(AssetStub.image1), - _createAssetMock(AssetStub.image2), - _createAssetMock(AssetStub.image3), - ).wait; + test('handles failed hashes', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); - final (asset1, file1, deviceAsset1, hash1) = mock1; - final (asset2, file2, deviceAsset2, hash2) = mock2; - final (asset3, file3, deviceAsset3, hash3) = mock3; + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + when(() => mockNativeApi.hashPaths(['image-path'])) + .thenAnswer((_) async => [null]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); + await sut.hashAssets(); - when(() => mockBackgroundService.digestFiles([file1.path])) - .thenAnswer((_) async => [hash1]); - when(() => mockBackgroundService.digestFiles([file2.path])) - .thenAnswer((_) async => [hash2]); - when(() => mockBackgroundService.digestFiles([file3.path])) - .thenAnswer((_) async => [hash3]); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 0); + }); - sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, + test('handles invalid hash length', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); + + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + + final invalidHash = Uint8List.fromList([1, 2, 3]); + when(() => mockNativeApi.hashPaths(['image-path'])) + .thenAnswer((_) async => [invalidHash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 0); + }); + + test('batches by file count limit', () async { + final sut = HashService( + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, batchFileLimit: 1, ); - final result = await sut.hashAssets([asset1, asset2, asset3]); - // Verify multiple batch process calls - verify(() => mockBackgroundService.digestFiles([file1.path])).called(1); - verify(() => mockBackgroundService.digestFiles([file2.path])).called(1); - verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - expect( - result, - [ - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ], - ); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); + + final hash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(any())) + .thenAnswer((_) async => [hash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); + verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); + verify(() => mockAssetRepo.updateHashes(any())).called(2); }); - test("HashService: Sort & Process different states", () async { - final (asset1, file1, deviceAsset1, hash1) = - await _createAssetMock(AssetStub.image1); // Will need rehashing - final (asset2, file2, deviceAsset2, hash2) = - await _createAssetMock(AssetStub.image2); // Will have matching hash - final (asset3, file3, deviceAsset3, hash3) = - await _createAssetMock(AssetStub.image3); // No DB entry - final asset4 = - AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed - - when(() => mockBackgroundService.digestFiles([file1.path, file3.path])) - .thenAnswer((_) async => [hash1, hash3]); - // DB entries are not sorted and a dummy entry added - when( - () => mockDeviceAssetRepository.getByIds([ - AssetStub.image1.localId!, - AssetStub.image2.localId!, - AssetStub.image3.localId!, - asset4.localId!, - ]), - ).thenAnswer( - (_) async => [ - // Same timestamp to reuse deviceAsset - deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt), - deviceAsset1, - deviceAsset3.copyWith(assetId: asset4.localId!), - ], + test('batches by size limit', () async { + final sut = HashService( + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, + batchSizeLimit: 80, ); - final result = await sut.hashAssets([asset1, asset2, asset3, asset4]); + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - // Verify correct processing of all assets - verify(() => mockBackgroundService.digestFiles([file1.path, file3.path])) - .called(1); - expect(result.length, 3); - expect(result, [ - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ]); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); + + final hash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(any())) + .thenAnswer((_) async => [hash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); + verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); + verify(() => mockAssetRepo.updateHashes(any())).called(2); }); - group("HashService: Edge cases", () { - test("handles empty list of assets", () async { - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); + test('handles mixed success and failure in batch', () async { + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - final result = await sut.hashAssets([]); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + final validHash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(['path-1', 'path-2'])) + .thenAnswer((_) async => [validHash, null]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - expect(result, isEmpty); - }); + await sut.hashAssets(); - test("handles all file access failures", () async { - // No DB entries - when( - () => mockDeviceAssetRepository.getByIds( - [AssetStub.image1.localId!, AssetStub.image2.localId!], - ), - ).thenAnswer((_) async => []); - - final result = await sut.hashAssets([ - AssetStub.image1, - AssetStub.image2, - ]); - - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - expect(result, isEmpty); - }); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 1); + expect(captured.first.id, asset1.id); }); }); } - -Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock( - Asset asset, -) async { - final random = Random(); - final hash = - Uint8List.fromList(List.generate(20, (i) => random.nextInt(255))); - final mockAsset = MockAsset(); - final mockAssetEntity = MockAssetEntity(); - final fs = MemoryFileSystem(); - final deviceAsset = DeviceAsset( - assetId: asset.localId!, - hash: Uint8List.fromList(hash), - modifiedTime: DateTime.now(), - ); - final tmp = await fs.systemTempDirectory.createTemp(); - final file = tmp.childFile("${asset.fileName}-path"); - await file.writeAsString("${asset.fileName}-content"); - - when(() => mockAsset.localId).thenReturn(asset.localId); - when(() => mockAsset.fileName).thenReturn(asset.fileName); - when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt); - when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt); - when(() => mockAsset.copyWith(checksum: any(named: "checksum"))) - .thenReturn(asset.copyWith(checksum: base64.encode(hash))); - when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity); - when(() => mockAssetEntity.originFile).thenAnswer((_) async => file); - - return (mockAsset, file, deviceAsset, hash); -} diff --git a/mobile/test/fixtures/album.stub.dart b/mobile/test/fixtures/album.stub.dart index c6ea199c0f3..1432d359010 100644 --- a/mobile/test/fixtures/album.stub.dart +++ b/mobile/test/fixtures/album.stub.dart @@ -1,3 +1,4 @@ +import 'package:immich_mobile/domain/models/local_album.model.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/infrastructure/entities/user.entity.dart'; @@ -101,3 +102,16 @@ final class AlbumStub { endDate: DateTime(2026), ); } + +abstract final class LocalAlbumStub { + const LocalAlbumStub._(); + + static final recent = LocalAlbum( + id: "recent-local-id", + name: "Recent", + updatedAt: DateTime(2023), + assetCount: 1000, + backupSelection: BackupSelection.none, + isIosSharedAlbum: false, + ); +} diff --git a/mobile/test/fixtures/asset.stub.dart b/mobile/test/fixtures/asset.stub.dart index 771b2dda96a..8d920119992 100644 --- a/mobile/test/fixtures/asset.stub.dart +++ b/mobile/test/fixtures/asset.stub.dart @@ -1,10 +1,11 @@ +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/exif.model.dart'; -import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/entities/asset.entity.dart' as old; final class AssetStub { const AssetStub._(); - static final image1 = Asset( + static final image1 = old.Asset( checksum: "image1-checksum", localId: "image1", remoteId: 'image1-remote', @@ -13,7 +14,7 @@ final class AssetStub { fileModifiedAt: DateTime(2020), updatedAt: DateTime.now(), durationInSeconds: 0, - type: AssetType.image, + type: old.AssetType.image, fileName: "image1.jpg", isFavorite: true, isArchived: false, @@ -21,7 +22,7 @@ final class AssetStub { exifInfo: const ExifInfo(isFlipped: false), ); - static final image2 = Asset( + static final image2 = old.Asset( checksum: "image2-checksum", localId: "image2", remoteId: 'image2-remote', @@ -30,7 +31,7 @@ final class AssetStub { fileModifiedAt: DateTime(2010), updatedAt: DateTime.now(), durationInSeconds: 60, - type: AssetType.video, + type: old.AssetType.video, fileName: "image2.jpg", isFavorite: false, isArchived: false, @@ -38,7 +39,7 @@ final class AssetStub { exifInfo: const ExifInfo(isFlipped: true), ); - static final image3 = Asset( + static final image3 = old.Asset( checksum: "image3-checksum", localId: "image3", ownerId: 1, @@ -46,10 +47,30 @@ final class AssetStub { fileModifiedAt: DateTime(2025), updatedAt: DateTime.now(), durationInSeconds: 60, - type: AssetType.image, + type: old.AssetType.image, fileName: "image3.jpg", isFavorite: true, isArchived: false, isTrashed: false, ); } + +abstract final class LocalAssetStub { + const LocalAssetStub._(); + + static final image1 = LocalAsset( + id: "image1", + name: "image1.jpg", + type: AssetType.image, + createdAt: DateTime(2025), + updatedAt: DateTime(2025, 2), + ); + + static final image2 = LocalAsset( + id: "image2", + name: "image2.jpg", + type: AssetType.image, + createdAt: DateTime(2000), + updatedAt: DateTime(20021), + ); +} diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index c4a5680f718..0dc241ca947 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,5 +1,8 @@ import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_album.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; @@ -18,6 +21,12 @@ class MockDeviceAssetRepository extends Mock class MockSyncStreamRepository extends Mock implements ISyncStreamRepository {} +class MockLocalAlbumRepository extends Mock implements ILocalAlbumRepository {} + +class MockLocalAssetRepository extends Mock implements ILocalAssetRepository {} + +class MockStorageRepository extends Mock implements IStorageRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} diff --git a/mobile/test/services/hash_service_test.dart b/mobile/test/services/hash_service_test.dart new file mode 100644 index 00000000000..e278199e4f5 --- /dev/null +++ b/mobile/test/services/hash_service_test.dart @@ -0,0 +1,425 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:collection/collection.dart'; +import 'package:file/memory.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/models/device_asset.model.dart'; +import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/services/background.service.dart'; +import 'package:immich_mobile/services/hash.service.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:photo_manager/photo_manager.dart'; + +import '../fixtures/asset.stub.dart'; +import '../infrastructure/repository.mock.dart'; +import '../service.mocks.dart'; + +class MockAsset extends Mock implements Asset {} + +class MockAssetEntity extends Mock implements AssetEntity {} + +void main() { + late HashService sut; + late BackgroundService mockBackgroundService; + late IDeviceAssetRepository mockDeviceAssetRepository; + + setUp(() { + mockBackgroundService = MockBackgroundService(); + mockDeviceAssetRepository = MockDeviceAssetRepository(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + ); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?)?.call(); + }); + when(() => mockDeviceAssetRepository.updateAll(any())) + .thenAnswer((_) async => true); + when(() => mockDeviceAssetRepository.deleteIds(any())) + .thenAnswer((_) async => true); + }); + + group("HashService: No DeviceAsset entry", () { + test("hash successfully", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + // No DB entries for this asset + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([mockAsset]); + + // Verify we stored the new hash in DB + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + expect( + result, + [AssetStub.image1.copyWith(checksum: base64.encode(hash))], + ); + }); + }); + + group("HashService: Has DeviceAsset entry", () { + test("when the asset is not modified", () async { + final hash = utf8.encode("image1-hash"); + + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer( + (_) async => [ + DeviceAsset( + assetId: AssetStub.image1.localId!, + hash: hash, + modifiedTime: AssetStub.image1.fileModifiedAt, + ), + ], + ); + final result = await sut.hashAssets([AssetStub.image1]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + + test("hashed successful when asset is modified", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + + final result = await sut.hashAssets([mockAsset]); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + }); + + group("HashService: Cleanup", () { + late Asset mockAsset; + late Uint8List hash; + late DeviceAsset deviceAsset; + late File file; + + setUp(() async { + (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + }); + + test("cleanups DeviceAsset when local file cannot be obtained", () async { + when(() => mockAsset.local).thenThrow(Exception("File not found")); + final result = await sut.hashAssets([mockAsset]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verify( + () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + + expect(result, isEmpty); + }); + + test("cleanups DeviceAsset when hashing failed", () async { + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + + // Verify the callback inside the transaction because, doing it outside results + // in a small delay before the callback is invoked, resulting in other LOCs getting executed + // resulting in an incorrect state + // + // i.e, consider the following piece of code + // await _deviceAssetRepository.transaction(() async { + // await _deviceAssetRepository.updateAll(toBeAdded); + // await _deviceAssetRepository.deleteIds(toBeDeleted); + // }); + // toBeDeleted.clear(); + // since the transaction method is mocked, the callback is not invoked until it is captured + // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed + // immediately once the transaction stub is executed, resulting in the deleteIds method being + // called with an empty list. + // + // To avoid this, we capture the callback and execute it within the transaction stub itself + // and verify the results inside the transaction stub + verify(() => mockDeviceAssetRepository.updateAll([])).called(1); + verify( + () => + mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + }); + + when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer( + // Invalid hash, length != 20 + (_) async => [Uint8List.fromList(hash.slice(2).toList())], + ); + + final result = await sut.hashAssets([mockAsset]); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + expect(result, isEmpty); + }); + }); + + group("HashService: Batch processing", () { + test("processes assets in batches when size limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + // Setup for multiple batch processing calls + when(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .thenAnswer((_) async => [hash1, hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + final size = await file1.length() + await file2.length(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchSizeLimit: size, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("processes assets in batches when file limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + when(() => mockBackgroundService.digestFiles([file1.path])) + .thenAnswer((_) async => [hash1]); + when(() => mockBackgroundService.digestFiles([file2.path])) + .thenAnswer((_) async => [hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchFileLimit: 1, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file2.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("HashService: Sort & Process different states", () async { + final (asset1, file1, deviceAsset1, hash1) = + await _createAssetMock(AssetStub.image1); // Will need rehashing + final (asset2, file2, deviceAsset2, hash2) = + await _createAssetMock(AssetStub.image2); // Will have matching hash + final (asset3, file3, deviceAsset3, hash3) = + await _createAssetMock(AssetStub.image3); // No DB entry + final asset4 = + AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed + + when(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .thenAnswer((_) async => [hash1, hash3]); + // DB entries are not sorted and a dummy entry added + when( + () => mockDeviceAssetRepository.getByIds([ + AssetStub.image1.localId!, + AssetStub.image2.localId!, + AssetStub.image3.localId!, + asset4.localId!, + ]), + ).thenAnswer( + (_) async => [ + // Same timestamp to reuse deviceAsset + deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt), + deviceAsset1, + deviceAsset3.copyWith(assetId: asset4.localId!), + ], + ); + + final result = await sut.hashAssets([asset1, asset2, asset3, asset4]); + + // Verify correct processing of all assets + verify(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .called(1); + expect(result.length, 3); + expect(result, [ + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ]); + }); + + group("HashService: Edge cases", () { + test("handles empty list of assets", () async { + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + final result = await sut.hashAssets([]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, isEmpty); + }); + + test("handles all file access failures", () async { + // No DB entries + when( + () => mockDeviceAssetRepository.getByIds( + [AssetStub.image1.localId!, AssetStub.image2.localId!], + ), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([ + AssetStub.image1, + AssetStub.image2, + ]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + expect(result, isEmpty); + }); + }); + }); +} + +Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock( + Asset asset, +) async { + final random = Random(); + final hash = + Uint8List.fromList(List.generate(20, (i) => random.nextInt(255))); + final mockAsset = MockAsset(); + final mockAssetEntity = MockAssetEntity(); + final fs = MemoryFileSystem(); + final deviceAsset = DeviceAsset( + assetId: asset.localId!, + hash: Uint8List.fromList(hash), + modifiedTime: DateTime.now(), + ); + final tmp = await fs.systemTempDirectory.createTemp(); + final file = tmp.childFile("${asset.fileName}-path"); + await file.writeAsString("${asset.fileName}-content"); + + when(() => mockAsset.localId).thenReturn(asset.localId); + when(() => mockAsset.fileName).thenReturn(asset.fileName); + when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt); + when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt); + when(() => mockAsset.copyWith(checksum: any(named: "checksum"))) + .thenReturn(asset.copyWith(checksum: base64.encode(hash))); + when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity); + when(() => mockAssetEntity.originFile).thenAnswer((_) async => file); + + return (mockAsset, file, deviceAsset, hash); +}