diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql new file mode 100644 index 00000000000..0e98416117f --- /dev/null +++ b/server/src/queries/asset.job.repository.sql @@ -0,0 +1,60 @@ +-- NOTE: This file is auto generated by ./sql-generator + +-- AssetJobRepository.getForSearchDuplicatesJob +select + "id", + "type", + "ownerId", + "duplicateId", + "stackId", + "isVisible", + "smart_search"."embedding", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "asset_files"."id", + "asset_files"."path", + "asset_files"."type" + from + "asset_files" + where + "asset_files"."assetId" = "assets"."id" + and "asset_files"."type" = $1 + ) as agg + ) as "files" +from + "assets" + left join "smart_search" on "assets"."id" = "smart_search"."assetId" +where + "assets"."id" = $2::uuid +limit + $3 + +-- AssetJobRepository.getForSidecarWriteJob +select + "id", + "sidecarPath", + "originalPath", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "tags"."value" + from + "tags" + inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId" + where + "assets"."id" = "tag_asset"."assetsId" + ) as agg + ) as "tags" +from + "assets" +where + "assets"."id" = $1::uuid +limit + $2 diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index b59cee58643..fa89814b8b5 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -181,65 +181,6 @@ from where "livePhotoVideoId" = $1::uuid --- AssetRepository.getAssetForSearchDuplicatesJob -select - "id", - "type", - "ownerId", - "duplicateId", - "stackId", - "isVisible", - "smart_search"."embedding", - ( - select - coalesce(json_agg(agg), '[]') - from - ( - select - "asset_files"."id", - "asset_files"."path", - "asset_files"."type" - from - "asset_files" - where - "asset_files"."assetId" = "assets"."id" - and "asset_files"."type" = $1 - ) as agg - ) as "files" -from - "assets" - left join "smart_search" on "assets"."id" = "smart_search"."assetId" -where - "assets"."id" = $2::uuid -limit - $3 - --- AssetRepository.getAssetForSidecarWriteJob -select - "id", - "sidecarPath", - "originalPath", - ( - select - coalesce(json_agg(agg), '[]') - from - ( - select - "tags"."value" - from - "tags" - inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId" - where - "assets"."id" = "tag_asset"."assetsId" - ) as agg - ) as "tags" -from - "assets" -where - "assets"."id" = $1::uuid -limit - $2 - -- AssetRepository.getById select "assets".* diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts new file mode 100644 index 00000000000..cc5e15d4215 --- /dev/null +++ b/server/src/repositories/asset-job.repository.ts @@ -0,0 +1,95 @@ +import { Injectable } from '@nestjs/common'; +import { Kysely } from 'kysely'; +import { jsonArrayFrom } from 'kysely/helpers/postgres'; +import { InjectKysely } from 'nestjs-kysely'; +import { DB } from 'src/db'; +import { DummyValue, GenerateSql } from 'src/decorators'; +import { withFiles } from 'src/entities/asset.entity'; +import { AssetFileType } from 'src/enum'; +import { StorageAsset } from 'src/types'; +import { asUuid } from 'src/utils/database'; + +@Injectable() +export class AssetJobRepository { + constructor(@InjectKysely() private db: Kysely) {} + + @GenerateSql({ params: [DummyValue.UUID] }) + getForSearchDuplicatesJob(id: string) { + return this.db + .selectFrom('assets') + .where('assets.id', '=', asUuid(id)) + .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') + .select((eb) => [ + 'id', + 'type', + 'ownerId', + 'duplicateId', + 'stackId', + 'isVisible', + 'smart_search.embedding', + withFiles(eb, AssetFileType.PREVIEW), + ]) + .limit(1) + .executeTakeFirst(); + } + + @GenerateSql({ params: [DummyValue.UUID] }) + getForSidecarWriteJob(id: string) { + return this.db + .selectFrom('assets') + .where('assets.id', '=', asUuid(id)) + .select((eb) => [ + 'id', + 'sidecarPath', + 'originalPath', + jsonArrayFrom( + eb + .selectFrom('tags') + .select(['tags.value']) + .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') + .whereRef('assets.id', '=', 'tag_asset.assetsId'), + ).as('tags'), + ]) + .limit(1) + .executeTakeFirst(); + } + + private storageTemplateAssetQuery() { + return this.db + .selectFrom('assets') + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select([ + 'assets.id', + 'assets.ownerId', + 'assets.type', + 'assets.checksum', + 'assets.originalPath', + 'assets.isExternal', + 'assets.sidecarPath', + 'assets.originalFileName', + 'assets.livePhotoVideoId', + 'assets.fileCreatedAt', + 'exif.timeZone', + 'exif.fileSizeInByte', + ]) + .where('assets.deletedAt', 'is', null); + } + + getForStorageTemplateJob(id: string): Promise { + return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise< + StorageAsset | undefined + >; + } + + streamForStorageTemplateJob() { + return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator; + } + + streamForDeletedJob(trashedBefore: Date) { + return this.db + .selectFrom('assets') + .select(['id', 'isOffline']) + .where('assets.deletedAt', '<=', trashedBefore) + .stream(); + } +} diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 8b31f47fd13..9871fa711c9 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely'; -import { jsonArrayFrom } from 'kysely/helpers/postgres'; import { isEmpty, isUndefined, omitBy } from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; @@ -24,7 +23,6 @@ import { } from 'src/entities/asset.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; -import { StorageAsset } from 'src/types'; import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; @@ -473,47 +471,6 @@ export class AssetRepository { return count; } - @GenerateSql({ params: [DummyValue.UUID] }) - getAssetForSearchDuplicatesJob(id: string) { - return this.db - .selectFrom('assets') - .where('assets.id', '=', asUuid(id)) - .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => [ - 'id', - 'type', - 'ownerId', - 'duplicateId', - 'stackId', - 'isVisible', - 'smart_search.embedding', - withFiles(eb, AssetFileType.PREVIEW), - ]) - .limit(1) - .executeTakeFirst(); - } - - @GenerateSql({ params: [DummyValue.UUID] }) - getAssetForSidecarWriteJob(id: string) { - return this.db - .selectFrom('assets') - .where('assets.id', '=', asUuid(id)) - .select((eb) => [ - 'id', - 'sidecarPath', - 'originalPath', - jsonArrayFrom( - eb - .selectFrom('tags') - .select(['tags.value']) - .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') - .whereRef('assets.id', '=', 'tag_asset.assetsId'), - ).as('tags'), - ]) - .limit(1) - .executeTakeFirst(); - } - @GenerateSql({ params: [DummyValue.UUID] }) getById( id: string, @@ -653,45 +610,6 @@ export class AssetRepository { .executeTakeFirst() as Promise; } - private storageTemplateAssetQuery() { - return this.db - .selectFrom('assets') - .innerJoin('exif', 'assets.id', 'exif.assetId') - .select([ - 'assets.id', - 'assets.ownerId', - 'assets.type', - 'assets.checksum', - 'assets.originalPath', - 'assets.isExternal', - 'assets.sidecarPath', - 'assets.originalFileName', - 'assets.livePhotoVideoId', - 'assets.fileCreatedAt', - 'exif.timeZone', - 'exif.fileSizeInByte', - ]) - .where('assets.deletedAt', 'is', null); - } - - getStorageTemplateAsset(id: string): Promise { - return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise< - StorageAsset | undefined - >; - } - - streamStorageTemplateAssets() { - return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator; - } - - streamDeletedAssets(trashedBefore: Date) { - return this.db - .selectFrom('assets') - .select(['id', 'isOffline']) - .where('assets.deletedAt', '<=', trashedBefore) - .stream(); - } - @GenerateSql( ...Object.values(WithProperty).map((property) => ({ name: property, diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index 8c262edcdeb..ef36a2b3f86 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository'; import { AlbumUserRepository } from 'src/repositories/album-user.repository'; import { AlbumRepository } from 'src/repositories/album.repository'; import { ApiKeyRepository } from 'src/repositories/api-key.repository'; +import { AssetJobRepository } from 'src/repositories/asset-job.repository'; import { AssetRepository } from 'src/repositories/asset.repository'; import { AuditRepository } from 'src/repositories/audit.repository'; import { ConfigRepository } from 'src/repositories/config.repository'; @@ -48,6 +49,7 @@ export const repositories = [ AuditRepository, ApiKeyRepository, AssetRepository, + AssetJobRepository, ConfigRepository, CronRepository, CryptoRepository, diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index a02324a3ae6..5fc4984b62e 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -536,12 +536,12 @@ describe(AssetService.name, () => { it('should immediately queue assets for deletion if trash is disabled', async () => { const asset = factory.asset({ isOffline: false }); - mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset])); mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: false } }); await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(new Date()); + expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(new Date()); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } }, ]); @@ -550,12 +550,12 @@ describe(AssetService.name, () => { it('should queue assets for deletion after trash duration', async () => { const asset = factory.asset({ isOffline: false }); - mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset])); mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: true, days: 7 } }); await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate()); + expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate()); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } }, ]); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 6ab3a76f28d..1ded79680bf 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -172,7 +172,7 @@ export class AssetService extends BaseService { } }; - const assets = this.assetRepository.streamDeletedAssets(trashedBefore); + const assets = this.assetJobRepository.streamForDeletedJob(trashedBefore); for await (const asset of assets) { chunk.push(asset); if (chunk.length >= JOBS_ASSET_PAGINATION_SIZE) { diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index b985ef8352d..2fbdd6e4c0e 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository'; import { AlbumUserRepository } from 'src/repositories/album-user.repository'; import { AlbumRepository } from 'src/repositories/album.repository'; import { ApiKeyRepository } from 'src/repositories/api-key.repository'; +import { AssetJobRepository } from 'src/repositories/asset-job.repository'; import { AssetRepository } from 'src/repositories/asset.repository'; import { AuditRepository } from 'src/repositories/audit.repository'; import { ConfigRepository } from 'src/repositories/config.repository'; @@ -62,6 +63,7 @@ export class BaseService { protected albumUserRepository: AlbumUserRepository, protected apiKeyRepository: ApiKeyRepository, protected assetRepository: AssetRepository, + protected assetJobRepository: AssetJobRepository, protected auditRepository: AuditRepository, protected configRepository: ConfigRepository, protected cronRepository: CronRepository, diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.ts index adb884c24f4..57ab955514e 100644 --- a/server/src/services/duplicate.service.spec.ts +++ b/server/src/services/duplicate.service.spec.ts @@ -194,6 +194,8 @@ describe(SearchService.name, () => { }); it('should fail if asset is not found', async () => { + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(void 0); + const result = await sut.handleSearchDuplicates({ id: assetStub.image.id }); expect(result).toBe(JobStatus.FAILED); @@ -202,7 +204,7 @@ describe(SearchService.name, () => { it('should skip if asset is part of stack', async () => { const id = assetStub.primaryImage.id; - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' }); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' }); const result = await sut.handleSearchDuplicates({ id }); @@ -212,7 +214,7 @@ describe(SearchService.name, () => { it('should skip if asset is not visible', async () => { const id = assetStub.livePhotoMotionAsset.id; - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false }); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false }); const result = await sut.handleSearchDuplicates({ id }); @@ -221,7 +223,7 @@ describe(SearchService.name, () => { }); it('should fail if asset is missing preview image', async () => { - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] }); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] }); const result = await sut.handleSearchDuplicates({ id: assetStub.noResizePath.id }); @@ -230,7 +232,7 @@ describe(SearchService.name, () => { }); it('should fail if asset is missing embedding', async () => { - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null }); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null }); const result = await sut.handleSearchDuplicates({ id: assetStub.image.id }); @@ -239,7 +241,7 @@ describe(SearchService.name, () => { }); it('should search for duplicates and update asset with duplicateId', async () => { - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); mocks.search.searchDuplicates.mockResolvedValue([ { assetId: assetStub.image.id, distance: 0.01, duplicateId: null }, ]); @@ -267,7 +269,7 @@ describe(SearchService.name, () => { it('should use existing duplicate ID among matched duplicates', async () => { const duplicateId = hasDupe.duplicateId; - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]); const expectedAssetIds = [hasEmbedding.id]; @@ -292,7 +294,7 @@ describe(SearchService.name, () => { }); it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => { - mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasDupe); + mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe); mocks.search.searchDuplicates.mockResolvedValue([]); const result = await sut.handleSearchDuplicates({ id: hasDupe.id }); diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 10adb645d38..9f1ac3d4ced 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -52,7 +52,7 @@ export class DuplicateService extends BaseService { return JobStatus.SKIPPED; } - const asset = await this.assetRepository.getAssetForSearchDuplicatesJob(id); + const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id); if (!asset) { this.logger.error(`Asset ${id} not found`); return JobStatus.FAILED; diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index f275477d935..358c0eea5ef 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -1486,14 +1486,14 @@ describe(MetadataService.name, () => { describe('handleSidecarWrite', () => { it('should skip assets that do not exist anymore', async () => { - mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(void 0); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0); await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.FAILED); expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); }); it('should skip jobs with no metadata', async () => { const asset = factory.jobAssets.sidecarWrite(); - mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset); await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.SKIPPED); expect(mocks.metadata.writeTags).not.toHaveBeenCalled(); }); @@ -1504,7 +1504,7 @@ describe(MetadataService.name, () => { const gps = 12; const date = '2023-11-22T04:56:12.196Z'; - mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset); + mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset); await expect( sut.handleSidecarWrite({ id: asset.id, diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 5277ea2d684..e460170968f 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -329,7 +329,7 @@ export class MetadataService extends BaseService { @OnJob({ name: JobName.SIDECAR_WRITE, queue: QueueName.SIDECAR }) async handleSidecarWrite(job: JobOf): Promise { const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job; - const asset = await this.assetRepository.getAssetForSidecarWriteJob(id); + const asset = await this.assetJobRepository.getForSidecarWriteJob(id); if (!asset) { return JobStatus.FAILED; } diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.ts index 46603cdbced..971a9e83026 100644 --- a/server/src/services/storage-template.service.spec.ts +++ b/server/src/services/storage-template.service.spec.ts @@ -112,8 +112,8 @@ describe(StorageTemplateService.name, () => { const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`; const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`; - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset); - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(stillAsset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(motionAsset); mocks.move.create.mockResolvedValueOnce({ id: '123', @@ -148,7 +148,7 @@ describe(StorageTemplateService.name, () => { sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset); mocks.album.getByAssetId.mockResolvedValueOnce([album]); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); @@ -169,7 +169,7 @@ describe(StorageTemplateService.name, () => { sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); @@ -197,7 +197,7 @@ describe(StorageTemplateService.name, () => { oldPath: asset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset); mocks.move.update.mockResolvedValue({ id: '123', entityId: asset.id, @@ -208,7 +208,7 @@ describe(StorageTemplateService.name, () => { await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id); + expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath); expect(mocks.move.update).toHaveBeenCalledWith('123', { @@ -239,7 +239,7 @@ describe(StorageTemplateService.name, () => { oldPath: asset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset); mocks.move.update.mockResolvedValue({ id: '123', entityId: asset.id, @@ -250,7 +250,7 @@ describe(StorageTemplateService.name, () => { await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id); + expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath); @@ -266,7 +266,7 @@ describe(StorageTemplateService.name, () => { mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats); mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8')); - mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset); mocks.move.create.mockResolvedValue({ id: '123', entityId: testAsset.id, @@ -277,7 +277,7 @@ describe(StorageTemplateService.name, () => { await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id); + expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1); expect(mocks.storage.stat).toHaveBeenCalledWith(newPath); expect(mocks.move.create).toHaveBeenCalledWith({ @@ -316,7 +316,7 @@ describe(StorageTemplateService.name, () => { oldPath: testAsset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset); mocks.move.update.mockResolvedValue({ id: '123', entityId: testAsset.id, @@ -327,7 +327,7 @@ describe(StorageTemplateService.name, () => { await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id); + expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.rename).not.toHaveBeenCalled(); @@ -340,12 +340,12 @@ describe(StorageTemplateService.name, () => { describe('handle template migration', () => { it('should handle no assets', async () => { - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([])); mocks.user.getList.mockResolvedValue([]); await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); }); it('should handle an asset with a duplicate destination', async () => { @@ -354,7 +354,7 @@ describe(StorageTemplateService.name, () => { const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; const newPath2 = newPath.replace('.jpg', '+1.jpg'); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', @@ -369,7 +369,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2); expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 }); expect(mocks.user.getList).toHaveBeenCalled(); @@ -378,12 +378,12 @@ describe(StorageTemplateService.name, () => { it('should skip when an asset already matches the template', async () => { const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); @@ -393,12 +393,12 @@ describe(StorageTemplateService.name, () => { it('should skip when an asset is probably a duplicate', async () => { const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); @@ -409,7 +409,7 @@ describe(StorageTemplateService.name, () => { const asset = assetStub.storageAsset(); const oldPath = asset.originalPath; const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', @@ -421,7 +421,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath); expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath }); }); @@ -429,7 +429,7 @@ describe(StorageTemplateService.name, () => { it('should use the user storage label', async () => { const user = factory.userAdmin({ storageLabel: 'label-1' }); const asset = assetStub.storageAsset({ ownerId: user.id }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([user]); mocks.move.create.mockResolvedValue({ id: '123', @@ -441,7 +441,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', `upload/library/${user.storageLabel}/2022/2022-06-19/${asset.originalFileName}`, @@ -456,7 +456,7 @@ describe(StorageTemplateService.name, () => { const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 }); const oldPath = asset.originalPath; const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ @@ -482,7 +482,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath); expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath); expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath); @@ -495,7 +495,7 @@ describe(StorageTemplateService.name, () => { it('should not update the database if the move fails due to incorrect newPath filesize', async () => { const asset = assetStub.storageAsset(); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ @@ -511,7 +511,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, @@ -528,7 +528,7 @@ describe(StorageTemplateService.name, () => { it('should not update the database if the move fails', async () => { const asset = assetStub.storageAsset(); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue(new Error('Read only system')); mocks.storage.copyFile.mockRejectedValue(new Error('Read only system')); mocks.move.create.mockResolvedValue({ @@ -542,7 +542,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, @@ -559,7 +559,7 @@ describe(StorageTemplateService.name, () => { originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`, originalFileName: 'IMG_7065.HEIC', }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([user]); mocks.move.create.mockResolvedValue({ id: '123', @@ -571,7 +571,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`, `upload/library/${user.storageLabel}/2022/2022-06-19/IMG_7065.heic`, @@ -585,7 +585,7 @@ describe(StorageTemplateService.name, () => { originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`, originalFileName: 'IMG_7065.HEIC', }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([user]); mocks.move.create.mockResolvedValue({ id: '123', @@ -597,7 +597,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`, `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`, @@ -611,7 +611,7 @@ describe(StorageTemplateService.name, () => { originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`, originalFileName: 'IMG_7065.JPEG', }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([user]); mocks.move.create.mockResolvedValue({ id: '123', @@ -623,7 +623,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`, `upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`, @@ -637,7 +637,7 @@ describe(StorageTemplateService.name, () => { originalPath: 'upload/library/user-id/2022/2022-06-19/IMG_7065.JPG', originalFileName: 'IMG_7065.JPG', }); - mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); + mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([user]); mocks.move.create.mockResolvedValue({ id: '123', @@ -649,7 +649,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPG`, `upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`, diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.ts index 07bac9839a6..71a0160ee24 100644 --- a/server/src/services/storage-template.service.ts +++ b/server/src/services/storage-template.service.ts @@ -118,7 +118,7 @@ export class StorageTemplateService extends BaseService { return JobStatus.SKIPPED; } - const asset = await this.assetRepository.getStorageTemplateAsset(id); + const asset = await this.assetJobRepository.getForStorageTemplateJob(id); if (!asset) { return JobStatus.FAILED; } @@ -130,7 +130,7 @@ export class StorageTemplateService extends BaseService { // move motion part of live photo if (asset.livePhotoVideoId) { - const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId); + const livePhotoVideo = await this.assetJobRepository.getForStorageTemplateJob(asset.livePhotoVideoId); if (!livePhotoVideo) { return JobStatus.FAILED; } @@ -152,7 +152,7 @@ export class StorageTemplateService extends BaseService { await this.moveRepository.cleanMoveHistory(); - const assets = this.assetRepository.streamStorageTemplateAssets(); + const assets = this.assetJobRepository.streamForStorageTemplateJob(); const users = await this.userRepository.getList(); for await (const asset of assets) { diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index a4674dafc90..a1d05ccd1c8 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -6,6 +6,7 @@ import { AssetJobStatus, Assets, DB } from 'src/db'; import { AssetType } from 'src/enum'; import { ActivityRepository } from 'src/repositories/activity.repository'; import { AlbumRepository } from 'src/repositories/album.repository'; +import { AssetJobRepository } from 'src/repositories/asset-job.repository'; import { AssetRepository } from 'src/repositories/asset.repository'; import { ConfigRepository } from 'src/repositories/config.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; @@ -21,7 +22,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.repos import { UserTable } from 'src/schema/tables/user.table'; import { BaseService } from 'src/services/base.service'; import { RepositoryInterface } from 'src/types'; -import { newUuid } from 'test/small.factory'; +import { newDate, newUuid } from 'test/small.factory'; import { automock, ServiceOverrides } from 'test/utils'; import { Mocked } from 'vitest'; @@ -30,6 +31,7 @@ type Repositories = { activity: ActivityRepository; album: AlbumRepository; asset: AssetRepository; + assetJob: AssetJobRepository; config: ConfigRepository; crypto: CryptoRepository; database: DatabaseRepository; @@ -113,6 +115,10 @@ export const getRepository = (key: K, db: Kysely(key: K) => { return automock(AssetRepository); } + case 'assetJob': { + return automock(AssetJobRepository); + } + case 'config': { return automock(ConfigRepository); } @@ -237,6 +247,7 @@ export const asDeps = (repositories: ServiceOverrides) => { repositories.albumUser, repositories.apiKey, repositories.asset || getRepositoryMock('asset'), + repositories.assetJob || getRepositoryMock('assetJob'), repositories.audit, repositories.config || getRepositoryMock('config'), repositories.cron, @@ -276,6 +287,7 @@ export const asDeps = (repositories: ServiceOverrides) => { const assetInsert = (asset: Partial> = {}) => { const id = asset.id || newUuid(); + const now = newDate(); const defaults: Insertable = { deviceAssetId: '', deviceId: '', @@ -285,6 +297,9 @@ const assetInsert = (asset: Partial> = {}) => { originalPath: '/path/to/something.jpg', ownerId: '@immich.cloud', isVisible: true, + fileCreatedAt: now, + fileModifiedAt: now, + localDateTime: now, }; return { diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 36fb298f7f3..f33b4878889 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -11,8 +11,6 @@ export const newAssetRepositoryMock = (): Mocked( album: automock(AlbumRepository, { strict: false }), albumUser: automock(AlbumUserRepository), asset: newAssetRepositoryMock(), + assetJob: automock(AssetJobRepository), config: newConfigRepositoryMock(), database: newDatabaseRepositoryMock(), downloadRepository: automock(DownloadRepository, { strict: false }), @@ -227,6 +230,7 @@ export const newTestService = ( overrides.albumUser || (mocks.albumUser as As), overrides.apiKey || (mocks.apiKey as As), overrides.asset || (mocks.asset as As), + overrides.assetJob || (mocks.assetJob as As), overrides.audit || (mocks.audit as As), overrides.config || (mocks.config as As as ConfigRepository), overrides.cron || (mocks.cron as As),