diff --git a/mobile/openapi/doc/AssetResponseDto.md b/mobile/openapi/doc/AssetResponseDto.md index f64e7144f62..986ca1b53d0 100644 --- a/mobile/openapi/doc/AssetResponseDto.md +++ b/mobile/openapi/doc/AssetResponseDto.md @@ -16,6 +16,7 @@ Name | Type | Description | Notes **originalPath** | **String** | | **originalFileName** | **String** | | **resized** | **bool** | | +**thumbhash** | **String** | base64 encoded thumbhash | **fileCreatedAt** | [**DateTime**](DateTime.md) | | **fileModifiedAt** | [**DateTime**](DateTime.md) | | **updatedAt** | [**DateTime**](DateTime.md) | | diff --git a/mobile/openapi/lib/model/asset_response_dto.dart b/mobile/openapi/lib/model/asset_response_dto.dart index 22f6f5f5d13..3b9264034dc 100644 --- a/mobile/openapi/lib/model/asset_response_dto.dart +++ b/mobile/openapi/lib/model/asset_response_dto.dart @@ -21,6 +21,7 @@ class AssetResponseDto { required this.originalPath, required this.originalFileName, required this.resized, + required this.thumbhash, required this.fileCreatedAt, required this.fileModifiedAt, required this.updatedAt, @@ -52,6 +53,9 @@ class AssetResponseDto { bool resized; + /// base64 encoded thumbhash + String? thumbhash; + DateTime fileCreatedAt; DateTime fileModifiedAt; @@ -101,6 +105,7 @@ class AssetResponseDto { other.originalPath == originalPath && other.originalFileName == originalFileName && other.resized == resized && + other.thumbhash == thumbhash && other.fileCreatedAt == fileCreatedAt && other.fileModifiedAt == fileModifiedAt && other.updatedAt == updatedAt && @@ -126,6 +131,7 @@ class AssetResponseDto { (originalPath.hashCode) + (originalFileName.hashCode) + (resized.hashCode) + + (thumbhash == null ? 0 : thumbhash!.hashCode) + (fileCreatedAt.hashCode) + (fileModifiedAt.hashCode) + (updatedAt.hashCode) + @@ -141,7 +147,7 @@ class AssetResponseDto { (checksum.hashCode); @override - String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resized=$resized, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people, checksum=$checksum]'; + String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resized=$resized, thumbhash=$thumbhash, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people, checksum=$checksum]'; Map toJson() { final json = {}; @@ -153,6 +159,11 @@ class AssetResponseDto { json[r'originalPath'] = this.originalPath; json[r'originalFileName'] = this.originalFileName; json[r'resized'] = this.resized; + if (this.thumbhash != null) { + json[r'thumbhash'] = this.thumbhash; + } else { + // json[r'thumbhash'] = null; + } json[r'fileCreatedAt'] = this.fileCreatedAt.toUtc().toIso8601String(); json[r'fileModifiedAt'] = this.fileModifiedAt.toUtc().toIso8601String(); json[r'updatedAt'] = this.updatedAt.toUtc().toIso8601String(); @@ -212,6 +223,7 @@ class AssetResponseDto { originalPath: mapValueOfType(json, r'originalPath')!, originalFileName: mapValueOfType(json, r'originalFileName')!, resized: mapValueOfType(json, r'resized')!, + thumbhash: mapValueOfType(json, r'thumbhash'), fileCreatedAt: mapDateTime(json, r'fileCreatedAt', '')!, fileModifiedAt: mapDateTime(json, r'fileModifiedAt', '')!, updatedAt: mapDateTime(json, r'updatedAt', '')!, @@ -280,6 +292,7 @@ class AssetResponseDto { 'originalPath', 'originalFileName', 'resized', + 'thumbhash', 'fileCreatedAt', 'fileModifiedAt', 'updatedAt', diff --git a/mobile/openapi/test/asset_response_dto_test.dart b/mobile/openapi/test/asset_response_dto_test.dart index 34b31d510e3..0bbcde25711 100644 --- a/mobile/openapi/test/asset_response_dto_test.dart +++ b/mobile/openapi/test/asset_response_dto_test.dart @@ -56,6 +56,12 @@ void main() { // TODO }); + // base64 encoded thumbhash + // String thumbhash + test('to test the property `thumbhash`', () async { + // TODO + }); + // DateTime fileCreatedAt test('to test the property `fileCreatedAt`', () async { // TODO diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index ab6e5d00dc9..c64c1c19ba0 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -4865,6 +4865,11 @@ "resized": { "type": "boolean" }, + "thumbhash": { + "type": "string", + "nullable": true, + "description": "base64 encoded thumbhash" + }, "fileCreatedAt": { "format": "date-time", "type": "string" @@ -4926,6 +4931,7 @@ "originalPath", "originalFileName", "resized", + "thumbhash", "fileCreatedAt", "fileModifiedAt", "updatedAt", diff --git a/server/package-lock.json b/server/package-lock.json index 7819b42c1ca..6b0fd842bc6 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -46,6 +46,7 @@ "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3", "sharp": "^0.31.3", + "thumbhash": "^0.1.1", "typeorm": "^0.3.11", "typesense": "^1.5.3", "ua-parser-js": "^1.0.35" @@ -4234,9 +4235,9 @@ } }, "node_modules/bullmq": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz", - "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==", + "version": "3.15.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.4.tgz", + "integrity": "sha512-jig63/PWODJEsAuswiCVUHaDWMv5fGpU36SjI0watAdXZMmy9K/iMKQAfPXmfZeK98bY/+co/efaDWVh3eVImw==", "dependencies": { "cron-parser": "^4.6.0", "glob": "^8.0.3", @@ -10806,6 +10807,11 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, + "node_modules/thumbhash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz", + "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -15241,9 +15247,9 @@ "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" }, "bullmq": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz", - "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==", + "version": "3.15.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.4.tgz", + "integrity": "sha512-jig63/PWODJEsAuswiCVUHaDWMv5fGpU36SjI0watAdXZMmy9K/iMKQAfPXmfZeK98bY/+co/efaDWVh3eVImw==", "requires": { "cron-parser": "^4.6.0", "glob": "^8.0.3", @@ -20185,6 +20191,11 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, + "thumbhash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz", + "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/server/package.json b/server/package.json index 1cce78e0163..b5f9687c9db 100644 --- a/server/package.json +++ b/server/package.json @@ -75,6 +75,7 @@ "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3", "sharp": "^0.31.3", + "thumbhash": "^0.1.1", "typeorm": "^0.3.11", "typesense": "^1.5.3", "ua-parser-js": "^1.0.35" diff --git a/server/src/domain/asset/response-dto/asset-response.dto.ts b/server/src/domain/asset/response-dto/asset-response.dto.ts index 4ece34bcd66..fa4d057d343 100644 --- a/server/src/domain/asset/response-dto/asset-response.dto.ts +++ b/server/src/domain/asset/response-dto/asset-response.dto.ts @@ -16,6 +16,8 @@ export class AssetResponseDto { originalPath!: string; originalFileName!: string; resized!: boolean; + /**base64 encoded thumbhash */ + thumbhash!: string | null; fileCreatedAt!: Date; fileModifiedAt!: Date; updatedAt!: Date; @@ -42,6 +44,7 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto { originalPath: entity.originalPath, originalFileName: entity.originalFileName, resized: !!entity.resizePath, + thumbhash: entity.thumbhash?.toString('base64') ?? null, fileCreatedAt: entity.fileCreatedAt, fileModifiedAt: entity.fileModifiedAt, updatedAt: entity.updatedAt, @@ -68,6 +71,7 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto { originalPath: entity.originalPath, originalFileName: entity.originalFileName, resized: !!entity.resizePath, + thumbhash: entity.thumbhash?.toString('base64') || null, fileCreatedAt: entity.fileCreatedAt, fileModifiedAt: entity.fileModifiedAt, updatedAt: entity.updatedAt, diff --git a/server/src/domain/job/job.constants.ts b/server/src/domain/job/job.constants.ts index 1d3b55b5362..a02248b3062 100644 --- a/server/src/domain/job/job.constants.ts +++ b/server/src/domain/job/job.constants.ts @@ -27,6 +27,7 @@ export enum JobName { QUEUE_GENERATE_THUMBNAILS = 'queue-generate-thumbnails', GENERATE_JPEG_THUMBNAIL = 'generate-jpeg-thumbnail', GENERATE_WEBP_THUMBNAIL = 'generate-webp-thumbnail', + GENERATE_THUMBHASH_THUMBNAIL = 'generate-thumbhash-thumbnail', // metadata QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction', @@ -92,6 +93,7 @@ export const JOBS_TO_QUEUE: Record = { [JobName.QUEUE_GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION, [JobName.GENERATE_JPEG_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION, [JobName.GENERATE_WEBP_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION, + [JobName.GENERATE_THUMBHASH_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION, // metadata [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION, diff --git a/server/src/domain/job/job.repository.ts b/server/src/domain/job/job.repository.ts index bd1d0025439..c088a2eedf8 100644 --- a/server/src/domain/job/job.repository.ts +++ b/server/src/domain/job/job.repository.ts @@ -31,6 +31,7 @@ export type JobItem = | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob } | { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IEntityJob } | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob } + | { name: JobName.GENERATE_THUMBHASH_THUMBNAIL; data: IEntityJob } // User Deletion | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob } diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index 391a40d9291..7da2f8e6708 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -261,7 +261,13 @@ describe(JobService.name, () => { }, { item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } }, - jobs: [JobName.GENERATE_WEBP_THUMBNAIL, JobName.CLASSIFY_IMAGE, JobName.ENCODE_CLIP, JobName.RECOGNIZE_FACES], + jobs: [ + JobName.GENERATE_WEBP_THUMBNAIL, + JobName.CLASSIFY_IMAGE, + JobName.ENCODE_CLIP, + JobName.RECOGNIZE_FACES, + JobName.GENERATE_THUMBHASH_THUMBNAIL, + ], }, { item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } }, diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index 4d819191d91..edc6aacd0ec 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -160,6 +160,7 @@ export class JobService { case JobName.GENERATE_JPEG_THUMBNAIL: { await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data }); + await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data }); await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: item.data }); await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data }); await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data }); diff --git a/server/src/domain/media/media.repository.ts b/server/src/domain/media/media.repository.ts index 6cd13b2e503..c3d7dc0e417 100644 --- a/server/src/domain/media/media.repository.ts +++ b/server/src/domain/media/media.repository.ts @@ -47,6 +47,7 @@ export interface IMediaRepository { // image resize(input: string | Buffer, output: string, options: ResizeOptions): Promise; crop(input: string, options: CropOptions): Promise; + generateThumbhash(imagePath: string): Promise; // video extractVideoThumbnail(input: string, output: string, size: number): Promise; diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 97081ab9044..4189a773feb 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -54,9 +54,9 @@ describe(MediaService.name, () => { }); }); - it('should queue all assets with missing thumbnails', async () => { + it('should queue all assets with missing resize path', async () => { assetMock.getWithout.mockResolvedValue({ - items: [assetEntityStub.image], + items: [assetEntityStub.noResizePath], hasNextPage: false, }); @@ -69,6 +69,38 @@ describe(MediaService.name, () => { data: { id: assetEntityStub.image.id }, }); }); + + it('should queue all assets with missing webp path', async () => { + assetMock.getWithout.mockResolvedValue({ + items: [assetEntityStub.noWebpPath], + hasNextPage: false, + }); + + await sut.handleQueueGenerateThumbnails({ force: false }); + + expect(assetMock.getAll).not.toHaveBeenCalled(); + expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.GENERATE_WEBP_THUMBNAIL, + data: { id: assetEntityStub.image.id }, + }); + }); + + it('should queue all assets with missing thumbhash', async () => { + assetMock.getWithout.mockResolvedValue({ + items: [assetEntityStub.noThumbhash], + hasNextPage: false, + }); + + await sut.handleQueueGenerateThumbnails({ force: false }); + + expect(assetMock.getAll).not.toHaveBeenCalled(); + expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.GENERATE_THUMBHASH_THUMBNAIL, + data: { id: assetEntityStub.image.id }, + }); + }); }); describe('handleGenerateJpegThumbnail', () => { @@ -129,6 +161,25 @@ describe(MediaService.name, () => { }); }); + describe('handleGenerateThumbhashThumbnail', () => { + it('should skip thumbhash generation if resize path is missing', async () => { + assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]); + await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.noResizePath.id }); + expect(mediaMock.generateThumbhash).not.toHaveBeenCalled(); + }); + + it('should generate a thumbhash', async () => { + const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8'); + assetMock.getByIds.mockResolvedValue([assetEntityStub.image]); + mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer); + + await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.image.id }); + + expect(mediaMock.generateThumbhash).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext'); + expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer }); + }); + }); + describe('handleQueueVideoConversion', () => { it('should queue all video assets', async () => { assetMock.getAll.mockResolvedValue({ diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index e1b8ee448c4..d11d4582ef2 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -37,7 +37,16 @@ export class MediaService { for await (const assets of assetPagination) { for (const asset of assets) { - await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } }); + if (!asset.resizePath || force) { + await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } }); + continue; + } + if (!asset.webpPath) { + await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } }); + } + if (!asset.thumbhash) { + await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } }); + } } } @@ -87,6 +96,18 @@ export class MediaService { return true; } + async handleGenerateThumbhashThumbnail({ id }: IEntityJob): Promise { + const [asset] = await this.assetRepository.getByIds([id]); + if (!asset?.resizePath) { + return false; + } + + const thumbhash = await this.mediaRepository.generateThumbhash(asset.resizePath); + await this.assetRepository.save({ id: asset.id, thumbhash }); + + return true; + } + async handleQueueVideoConversion(job: IBaseJob) { const { force } = job; diff --git a/server/src/immich/api-v1/asset/asset.core.ts b/server/src/immich/api-v1/asset/asset.core.ts index d0f48657a52..031ab58d437 100644 --- a/server/src/immich/api-v1/asset/asset.core.ts +++ b/server/src/immich/api-v1/asset/asset.core.ts @@ -35,6 +35,7 @@ export class AssetCore { livePhotoVideo: livePhotoAssetId != null ? ({ id: livePhotoAssetId } as AssetEntity) : null, resizePath: null, webpPath: null, + thumbhash: null, encodedVideoPath: null, tags: [], sharedLinks: [], diff --git a/server/src/infra/entities/asset.entity.ts b/server/src/infra/entities/asset.entity.ts index 202e393d22c..82172bcfaca 100644 --- a/server/src/infra/entities/asset.entity.ts +++ b/server/src/infra/entities/asset.entity.ts @@ -51,6 +51,9 @@ export class AssetEntity { @Column({ type: 'varchar', nullable: true, default: '' }) webpPath!: string | null; + @Column({ type: 'bytea', nullable: true }) + thumbhash!: Buffer | null; + @Column({ type: 'varchar', nullable: true, default: '' }) encodedVideoPath!: string | null; diff --git a/server/src/infra/migrations/1686762895180-AddThumbhashColumn.ts b/server/src/infra/migrations/1686762895180-AddThumbhashColumn.ts new file mode 100644 index 00000000000..4ad73163db0 --- /dev/null +++ b/server/src/infra/migrations/1686762895180-AddThumbhashColumn.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddThumbhashColumn1685546571785 implements MigrationInterface { + name = 'AddThumbhashColumn1686762895180'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "assets" ADD "thumbhash" bytea NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "thumbhash"`); + } +} diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index 0dc37047467..766819d3763 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -135,6 +135,7 @@ export class AssetRepository implements IAssetRepository { { resizePath: '', isVisible: true }, { webpPath: IsNull(), isVisible: true }, { webpPath: '', isVisible: true }, + { thumbhash: IsNull(), isVisible: true }, ]; break; diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index e404a026f7f..b73b61aaeb4 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -119,4 +119,17 @@ export class MediaRepository implements IMediaRepository { .run(); }); } + + async generateThumbhash(imagePath: string): Promise { + const maxSize = 100; + + const { data, info } = await sharp(imagePath) + .resize(maxSize, maxSize, { fit: 'inside', withoutEnlargement: true }) + .raw() + .ensureAlpha() + .toBuffer({ resolveWithObject: true }); + + const thumbhash = await import('thumbhash'); + return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data)); + } } diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 92b0f237c61..079fd40d305 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -61,6 +61,7 @@ export class AppService { [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data), [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data), [JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data), + [JobName.GENERATE_THUMBHASH_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbhashThumbnail(data), [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data), [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data), diff --git a/server/test/fixtures.ts b/server/test/fixtures.ts index eaa22a466d4..f60a0743a4c 100644 --- a/server/test/fixtures.ts +++ b/server/test/fixtures.ts @@ -196,7 +196,8 @@ export const assetEntityStub = { resizePath: null, checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, - webpPath: null, + webpPath: '/uploads/user-id/webp/path.ext', + thumbhash: Buffer.from('blablabla', 'base64'), encodedVideoPath: null, createdAt: new Date('2023-02-23T05:06:29.716Z'), updatedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -212,7 +213,7 @@ export const assetEntityStub = { faces: [], sidecarPath: null, }), - image: Object.freeze({ + noWebpPath: Object.freeze({ id: 'asset-id', deviceAssetId: 'device-asset-id', fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -225,6 +226,67 @@ export const assetEntityStub = { checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, + thumbhash: Buffer.from('blablabla', 'base64'), + encodedVideoPath: null, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), + mimeType: null, + isFavorite: true, + isArchived: false, + duration: null, + isVisible: true, + livePhotoVideo: null, + livePhotoVideoId: null, + tags: [], + sharedLinks: [], + originalFileName: 'asset-id.ext', + faces: [], + sidecarPath: null, + }), + noThumbhash: Object.freeze({ + id: 'asset-id', + deviceAssetId: 'device-asset-id', + fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), + fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'), + owner: userEntityStub.user1, + ownerId: 'user-id', + deviceId: 'device-id', + originalPath: '/original/path.ext', + resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), + type: AssetType.IMAGE, + webpPath: '/uploads/user-id/webp/path.ext', + thumbhash: null, + encodedVideoPath: null, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), + mimeType: null, + isFavorite: true, + isArchived: false, + duration: null, + isVisible: true, + livePhotoVideo: null, + livePhotoVideoId: null, + tags: [], + sharedLinks: [], + originalFileName: 'asset-id.ext', + faces: [], + sidecarPath: null, + }), + image: Object.freeze({ + id: 'asset-id', + deviceAssetId: 'device-asset-id', + fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), + fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'), + owner: userEntityStub.user1, + ownerId: 'user-id', + deviceId: 'device-id', + originalPath: '/original/path.ext', + resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), + type: AssetType.IMAGE, + webpPath: '/uploads/user-id/webp/path.ext', + thumbhash: Buffer.from('blablabla', 'base64'), encodedVideoPath: null, createdAt: new Date('2023-02-23T05:06:29.716Z'), updatedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -255,6 +317,7 @@ export const assetEntityStub = { checksum: Buffer.from('file hash', 'utf8'), type: AssetType.VIDEO, webpPath: null, + thumbhash: null, encodedVideoPath: null, createdAt: new Date('2023-02-23T05:06:29.716Z'), updatedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -305,6 +368,7 @@ export const assetEntityStub = { sidecarPath: null, type: AssetType.IMAGE, webpPath: null, + thumbhash: null, encodedVideoPath: null, createdAt: new Date('2023-02-23T05:06:29.716Z'), updatedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -334,6 +398,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', + thumbhash: null, checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, @@ -507,6 +572,7 @@ const assetResponse: AssetResponseDto = { originalPath: 'fake_path/jpeg', originalFileName: 'asset_1.jpeg', resized: false, + thumbhash: null, fileModifiedAt: today, fileCreatedAt: today, updatedAt: today, @@ -787,6 +853,7 @@ export const sharedLinkStub = { clipEmbedding: [0.12, 0.13, 0.14], }, webpPath: '', + thumbhash: null, encodedVideoPath: '', duration: null, isVisible: true, diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts index 78141df1060..487ddb4800b 100644 --- a/server/test/repositories/media.repository.mock.ts +++ b/server/test/repositories/media.repository.mock.ts @@ -3,6 +3,7 @@ import { IMediaRepository } from '@app/domain'; export const newMediaRepositoryMock = (): jest.Mocked => { return { extractVideoThumbnail: jest.fn(), + generateThumbhash: jest.fn(), resize: jest.fn(), crop: jest.fn(), probe: jest.fn(), diff --git a/server/tsconfig.json b/server/tsconfig.json index 743d4a24155..55d9e7f2eb4 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -9,6 +9,7 @@ "allowSyntheticDefaultImports": true, "resolveJsonModule": true, "target": "es2017", + "moduleResolution": "node16", "sourceMap": true, "outDir": "./dist", "incremental": true, diff --git a/web/package-lock.json b/web/package-lock.json index b4301467763..dbbc2e323e0 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -20,7 +20,8 @@ "rxjs": "^7.8.0", "socket.io-client": "^4.6.1", "svelte-local-storage-store": "^0.5.0", - "svelte-material-icons": "^3.0.4" + "svelte-material-icons": "^3.0.4", + "unlazy": "^0.8.9" }, "devDependencies": { "@babel/preset-env": "^7.20.2", @@ -4134,6 +4135,15 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@unlazy/core": { + "version": "0.8.9", + "resolved": "https://registry.npmjs.org/@unlazy/core/-/core-0.8.9.tgz", + "integrity": "sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA==", + "dependencies": { + "fast-blurhash": "^1.1.2", + "thumbhash": "^0.1.1" + } + }, "node_modules/@zoom-image/core": { "version": "0.18.2", "resolved": "https://registry.npmjs.org/@zoom-image/core/-/core-0.18.2.tgz", @@ -5945,6 +5955,11 @@ "node": ">= 14" } }, + "node_modules/fast-blurhash": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fast-blurhash/-/fast-blurhash-1.1.2.tgz", + "integrity": "sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -11217,6 +11232,11 @@ "node": ">=0.8" } }, + "node_modules/thumbhash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz", + "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==" + }, "node_modules/tiny-glob": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", @@ -11441,6 +11461,18 @@ "node": ">= 4.0.0" } }, + "node_modules/unlazy": { + "version": "0.8.9", + "resolved": "https://registry.npmjs.org/unlazy/-/unlazy-0.8.9.tgz", + "integrity": "sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg==", + "dependencies": { + "@unlazy/core": "0.8.9" + }, + "peerDependencies": { + "fast-blurhash": "^1.1.2", + "thumbhash": "^0.1.1" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", @@ -14739,6 +14771,15 @@ "eslint-visitor-keys": "^3.3.0" } }, + "@unlazy/core": { + "version": "0.8.9", + "resolved": "https://registry.npmjs.org/@unlazy/core/-/core-0.8.9.tgz", + "integrity": "sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA==", + "requires": { + "fast-blurhash": "^1.1.2", + "thumbhash": "^0.1.1" + } + }, "@zoom-image/core": { "version": "0.18.2", "resolved": "https://registry.npmjs.org/@zoom-image/core/-/core-0.18.2.tgz", @@ -16053,6 +16094,11 @@ "source-map-support": "^0.5.21" } }, + "fast-blurhash": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fast-blurhash/-/fast-blurhash-1.1.2.tgz", + "integrity": "sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A==" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -19861,6 +19907,11 @@ "thenify": ">= 3.1.0 < 4" } }, + "thumbhash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz", + "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==" + }, "tiny-glob": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", @@ -20023,6 +20074,14 @@ "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", "dev": true }, + "unlazy": { + "version": "0.8.9", + "resolved": "https://registry.npmjs.org/unlazy/-/unlazy-0.8.9.tgz", + "integrity": "sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg==", + "requires": { + "@unlazy/core": "0.8.9" + } + }, "update-browserslist-db": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", diff --git a/web/package.json b/web/package.json index b80ed77b4b4..b3287e0008a 100644 --- a/web/package.json +++ b/web/package.json @@ -70,6 +70,7 @@ "rxjs": "^7.8.0", "socket.io-client": "^4.6.1", "svelte-local-storage-store": "^0.5.0", - "svelte-material-icons": "^3.0.4" + "svelte-material-icons": "^3.0.4", + "unlazy": "^0.8.9" } } diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index fd832c4047f..c85c767fef1 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -637,6 +637,12 @@ export interface AssetResponseDto { * @memberof AssetResponseDto */ 'resized': boolean; + /** + * base64 encoded thumbhash + * @type {string} + * @memberof AssetResponseDto + */ + 'thumbhash': string | null; /** * * @type {string} diff --git a/web/src/lib/components/asset-viewer/detail-panel.svelte b/web/src/lib/components/asset-viewer/detail-panel.svelte index c2257fb2e51..1a15e11d100 100644 --- a/web/src/lib/components/asset-viewer/detail-panel.svelte +++ b/web/src/lib/components/asset-viewer/detail-panel.svelte @@ -120,6 +120,7 @@ altText={person.name} widthStyle="90px" heightStyle="90px" + thumbhash={null} />

{person.name}

diff --git a/web/src/lib/components/assets/thumbnail/image-thumbnail.svelte b/web/src/lib/components/assets/thumbnail/image-thumbnail.svelte index 08a6176d7f5..e4163b8b6c1 100644 --- a/web/src/lib/components/assets/thumbnail/image-thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/image-thumbnail.svelte @@ -1,27 +1,58 @@ -{altText} (loading = false)} -/> +{#if thumbhash} + {altText} + + +{:else} + {altText} (loading = false)} + /> +{/if} diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index dd8662172ff..e4b078490d2 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -129,6 +129,7 @@ altText={asset.originalFileName} widthStyle="{width}px" heightStyle="{height}px" + thumbhash={asset.thumbhash} /> {:else}