From 3925445de8206946de8c7f9b7a6742b2df5e110d Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Fri, 21 Feb 2025 12:20:25 +0300 Subject: [PATCH] feat(web): use wasm for justified layout calculation (#15524) * working * use wrapper class * update import * simplify * it works without changing `optimizeDeps` * inline layout options * update gallery view * use es2022 * fix import * fix vitest * empty geometry * bump version * Update web/src/lib/stores/assets.store.ts Co-authored-by: Jason Rasmussen * fix: typo --------- Co-authored-by: Jason Rasmussen Co-authored-by: Alex Tran --- web/package-lock.json | 31 +++++++------ web/package.json | 4 +- .../photos-page/asset-date-group.svelte | 30 ++++++++----- .../gallery-viewer/gallery-viewer.svelte | 41 +++++++---------- web/src/lib/stores/assets.store.ts | 36 ++++++--------- web/src/lib/utils/asset-utils.ts | 11 +++++ web/src/lib/utils/timeline-util.ts | 44 +++++-------------- web/tsconfig.json | 2 +- web/vite.config.js | 9 ++++ 9 files changed, 98 insertions(+), 110 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 782971cffbd..e97731061e3 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -10,6 +10,7 @@ "license": "GNU Affero General Public License version 3", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.9.8", + "@immich/justified-layout-wasm": "^0.1.2", "@immich/sdk": "file:../open-api/typescript-sdk", "@immich/ui": "^0.16.0", "@mapbox/mapbox-gl-rtl-text": "0.2.3", @@ -23,7 +24,6 @@ "dom-to-image": "^2.6.0", "handlebars": "^4.7.8", "intl-messageformat": "^10.7.11", - "justified-layout": "^4.1.0", "lodash-es": "^4.17.21", "luxon": "^3.4.4", "socket.io-client": "~4.8.0", @@ -45,7 +45,6 @@ "@testing-library/svelte": "^5.2.6", "@testing-library/user-event": "^14.5.2", "@types/dom-to-image": "^2.6.7", - "@types/justified-layout": "^4.1.4", "@types/lodash-es": "^4.17.12", "@types/luxon": "^3.4.2", "@typescript-eslint/eslint-plugin": "^8.20.0", @@ -71,6 +70,7 @@ "tslib": "^2.6.2", "typescript": "^5.7.3", "vite": "^6.0.0", + "vite-plugin-wasm": "^3.4.1", "vitest": "^3.0.0" } }, @@ -1339,6 +1339,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@immich/justified-layout-wasm": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.1.2.tgz", + "integrity": "sha512-6AmzYJhLedzIXSkhO/0tfBbHAeUeLmG1c4yTzJmtuSGyn7JAzVCFp0dp4T8Wh1tfIDx0Y0pAYB9tm2xlJHdEPA==", + "license": "AGPL-3" + }, "node_modules/@immich/sdk": { "resolved": "../open-api/typescript-sdk", "link": true @@ -2413,12 +2419,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/justified-layout": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/@types/justified-layout/-/justified-layout-4.1.4.tgz", - "integrity": "sha512-q2ybP0u0NVj87oMnGZOGxY2iUN8ddr48zPOBHBdbOLpsMTA/keGj+93ou+OMCnJk0xewzlNIaVEkxM6VBD3E2w==", - "dev": true - }, "node_modules/@types/leaflet": { "version": "1.9.8", "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.8.tgz", @@ -5382,11 +5382,6 @@ "resolved": "https://registry.npmjs.org/just-flush/-/just-flush-2.3.0.tgz", "integrity": "sha512-fBuxQ1gJ61BurmhwKS5LYTzhkbrT5j/2U7ax+UbLm9aRvCTh2h6AfzLteOckE4KKomqOf0Y3zIG3Xu57sRsKUg==" }, - "node_modules/justified-layout": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/justified-layout/-/justified-layout-4.1.0.tgz", - "integrity": "sha512-M5FimNMXgiOYerVRGsXZ2YK9YNCaTtwtYp7Hb2308U1Q9TXXHx5G0p08mcVR5O53qf8bWY4NJcPBxE6zuayXSg==" - }, "node_modules/kdbush": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", @@ -8625,6 +8620,16 @@ } } }, + "node_modules/vite-plugin-wasm": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/vite-plugin-wasm/-/vite-plugin-wasm-3.4.1.tgz", + "integrity": "sha512-ja3nSo2UCkVeitltJGkS3pfQHAanHv/DqGatdI39ja6McgABlpsZ5hVgl6wuR8Qx5etY3T5qgDQhOWzc5RReZA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "vite": "^2 || ^3 || ^4 || ^5 || ^6" + } + }, "node_modules/vitefu": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.0.5.tgz", diff --git a/web/package.json b/web/package.json index 3fd4396e211..2ca8a71848b 100644 --- a/web/package.json +++ b/web/package.json @@ -35,7 +35,6 @@ "@testing-library/svelte": "^5.2.6", "@testing-library/user-event": "^14.5.2", "@types/dom-to-image": "^2.6.7", - "@types/justified-layout": "^4.1.4", "@types/lodash-es": "^4.17.12", "@types/luxon": "^3.4.2", "@typescript-eslint/eslint-plugin": "^8.20.0", @@ -61,11 +60,13 @@ "tslib": "^2.6.2", "typescript": "^5.7.3", "vite": "^6.0.0", + "vite-plugin-wasm": "^3.4.1", "vitest": "^3.0.0" }, "type": "module", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.9.8", + "@immich/justified-layout-wasm": "^0.1.2", "@immich/sdk": "file:../open-api/typescript-sdk", "@immich/ui": "^0.16.0", "@mapbox/mapbox-gl-rtl-text": "0.2.3", @@ -79,7 +80,6 @@ "dom-to-image": "^2.6.0", "handlebars": "^4.7.8", "intl-messageformat": "^10.7.11", - "justified-layout": "^4.1.0", "lodash-es": "^4.17.21", "luxon": "^3.4.4", "socket.io-client": "~4.8.0", diff --git a/web/src/lib/components/photos-page/asset-date-group.svelte b/web/src/lib/components/photos-page/asset-date-group.svelte index 5d1497c3a99..566477e7433 100644 --- a/web/src/lib/components/photos-page/asset-date-group.svelte +++ b/web/src/lib/components/photos-page/asset-date-group.svelte @@ -97,6 +97,7 @@ {#each dateGroups as dateGroup, groupIndex (dateGroup.date)} {@const display = dateGroup.intersecting || !!dateGroup.assets.some((asset) => asset.id === $assetStore.pendingScrollAssetId)} + {@const geometry = dateGroup.geometry}
{#if !display} @@ -149,7 +150,7 @@
{#if !singleSelect && ((hoveredDateGroup == dateGroup.groupTitle && isMouseOverGroup) || assetInteraction.selectedGroup.has(dateGroup.groupTitle))}
- {#each dateGroup.assets as asset, index (asset.id)} - {@const box = dateGroup.geometry.boxes[index]} + {#each dateGroup.assets as asset, i (asset.id)} {@const isSmallGroup = dateGroup.assets.length <= SMALL_GROUP_THRESHOLD} + + {@const top = geometry.getTop(i)} + {@const left = geometry.getLeft(i)} + {@const width = geometry.getWidth(i)} + {@const height = geometry.getHeight(i)} +
diff --git a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte index 4c3c35aecab..df54295f04c 100644 --- a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte +++ b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte @@ -8,13 +8,11 @@ import type { Viewport } from '$lib/stores/assets.store'; import { showDeleteModal } from '$lib/stores/preferences.store'; import { deleteAssets } from '$lib/utils/actions'; - import { archiveAssets, cancelMultiselect, getAssetRatio } from '$lib/utils/asset-utils'; + import { archiveAssets, cancelMultiselect, getJustifiedLayoutFromAssets } from '$lib/utils/asset-utils'; import { featureFlags } from '$lib/stores/server-config.store'; import { handleError } from '$lib/utils/handle-error'; import { navigate } from '$lib/utils/navigation'; - import { calculateWidth } from '$lib/utils/timeline-util'; import { type AssetResponseDto } from '@immich/sdk'; - import justifiedLayout from 'justified-layout'; import { t } from 'svelte-i18n'; import AssetViewer from '../../asset-viewer/asset-viewer.svelte'; import ShowShortcuts from '../show-shortcuts.svelte'; @@ -310,23 +308,12 @@ let idsSelectedAssets = $derived(assetInteraction.selectedAssetsArray.map(({ id }) => id)); let geometry = $derived( - (() => { - const justifiedLayoutResult = justifiedLayout( - assets.map((asset) => getAssetRatio(asset)), - { - boxSpacing: 2, - containerWidth: Math.floor(viewport.width), - containerPadding: 0, - targetRowHeightTolerance: 0.15, - targetRowHeight: 235, - }, - ); - - return { - ...justifiedLayoutResult, - containerWidth: calculateWidth(justifiedLayoutResult.boxes), - }; - })(), + getJustifiedLayoutFromAssets(assets, { + spacing: 2, + rowWidth: Math.floor(viewport.width), + heightTolerance: 0.15, + rowHeight: 235, + }), ); $effect(() => { @@ -364,11 +351,15 @@ {#if assets.length > 0}
- {#each assets as asset, i (i)} + {#each assets as asset, i} + {@const top = geometry.getTop(i)} + {@const left = geometry.getLeft(i)} + {@const width = geometry.getWidth(i)} + {@const height = geometry.getHeight(i)} +
{#if showAssetName}
[0]; export type AssetStoreOptions = Omit; -const LAYOUT_OPTIONS = { - boxSpacing: 2, - containerPadding: 0, - targetRowHeightTolerance: 0.15, - targetRowHeight: 235, -}; - export interface Viewport { width: number; height: number; @@ -470,32 +462,30 @@ export class AssetStore { assetGroup.heightActual = false; } } + + const viewportWidth = this.viewport.width; if (!bucket.isBucketHeightActual) { const unwrappedWidth = (3 / 2) * bucket.bucketCount * THUMBNAIL_HEIGHT * (7 / 10); - const rows = Math.ceil(unwrappedWidth / this.viewport.width); + const rows = Math.ceil(unwrappedWidth / viewportWidth); const height = 51 + rows * THUMBNAIL_HEIGHT; bucket.bucketHeight = height; } + const layoutOptions = { + spacing: 2, + heightTolerance: 0.15, + rowHeight: 235, + rowWidth: Math.floor(viewportWidth), + }; for (const assetGroup of bucket.dateGroups) { if (!assetGroup.heightActual) { const unwrappedWidth = (3 / 2) * assetGroup.assets.length * THUMBNAIL_HEIGHT * (7 / 10); - const rows = Math.ceil(unwrappedWidth / this.viewport.width); + const rows = Math.ceil(unwrappedWidth / viewportWidth); const height = rows * THUMBNAIL_HEIGHT; assetGroup.height = height; } - const layoutResult = createJustifiedLayout( - assetGroup.assets.map((g) => getAssetRatio(g)), - { - ...LAYOUT_OPTIONS, - containerWidth: Math.floor(this.viewport.width), - }, - ); - assetGroup.geometry = { - ...layoutResult, - containerWidth: calculateWidth(layoutResult.boxes), - }; + assetGroup.geometry = getJustifiedLayoutFromAssets(assetGroup.assets, layoutOptions); } } diff --git a/web/src/lib/utils/asset-utils.ts b/web/src/lib/utils/asset-utils.ts index 70f5c5f8f26..d3090bf066e 100644 --- a/web/src/lib/utils/asset-utils.ts +++ b/web/src/lib/utils/asset-utils.ts @@ -12,6 +12,7 @@ import { downloadRequest, getKey, withError } from '$lib/utils'; import { createAlbum } from '$lib/utils/album-utils'; import { getByteUnitString } from '$lib/utils/byte-units'; import { getFormatter } from '$lib/utils/i18n'; +import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm'; import { addAssetsToAlbum as addAssets, createStack, @@ -587,3 +588,13 @@ export const copyImageToClipboard = async (source: HTMLImageElement | string) => const blob = source instanceof HTMLImageElement ? await imgToBlob(source) : await urlToBlob(source); await navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]); }; + +export function getJustifiedLayoutFromAssets(assets: AssetResponseDto[], options: LayoutOptions) { + const aspectRatios = new Float32Array(assets.length); + // eslint-disable-next-line unicorn/no-for-loop + for (let i = 0; i < assets.length; i++) { + const { width, height } = getAssetRatio(assets[i]); + aspectRatios[i] = width / height; + } + return new JustifiedLayout(aspectRatios, options); +} diff --git a/web/src/lib/utils/timeline-util.ts b/web/src/lib/utils/timeline-util.ts index 7e65dcdb997..44baffb1354 100644 --- a/web/src/lib/utils/timeline-util.ts +++ b/web/src/lib/utils/timeline-util.ts @@ -1,7 +1,7 @@ import type { AssetBucket } from '$lib/stores/assets.store'; import { locale } from '$lib/stores/preferences.store'; +import { JustifiedLayout } from '@immich/justified-layout-wasm'; import type { AssetResponseDto } from '@immich/sdk'; -import type createJustifiedLayout from 'justified-layout'; import { groupBy, memoize, sortBy } from 'lodash-es'; import { DateTime } from 'luxon'; import { get } from 'svelte/store'; @@ -13,7 +13,7 @@ export type DateGroup = { height: number; heightActual: boolean; intersecting: boolean; - geometry: Geometry; + geometry: JustifiedLayout; bucket: AssetBucket; }; export type ScrubberListener = ( @@ -80,18 +80,12 @@ export function formatGroupTitle(_date: DateTime): string { return date.toLocaleString(groupDateFormat); } -type Geometry = ReturnType & { - containerWidth: number; -}; - -function emptyGeometry() { - return { - containerWidth: 0, - containerHeight: 0, - widowCount: 0, - boxes: [], - }; -} +const emptyGeometry = new JustifiedLayout(Float32Array.from([]), { + rowHeight: 1, + heightTolerance: 0, + rowWidth: 1, + spacing: 0, +}); const formatDateGroupTitle = memoize(formatGroupTitle); @@ -100,6 +94,7 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string | fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat, { locale }), ); const sorted = sortBy(grouped, (group) => bucket.assets.indexOf(group[0])); + return sorted.map((group) => { const date = fromLocalDateTime(group[0].localDateTime).startOf('day'); return { @@ -109,31 +104,12 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string | height: 0, heightActual: false, intersecting: false, - geometry: emptyGeometry(), + geometry: emptyGeometry, bucket, }; }); } -export type LayoutBox = { - aspectRatio: number; - top: number; - width: number; - height: number; - left: number; - forcedAspectRatio?: boolean; -}; - -export function calculateWidth(boxes: LayoutBox[]): number { - let width = 0; - for (const box of boxes) { - if (box.top < 100) { - width = box.left + box.width; - } - } - return width; -} - export function findTotalOffset(element: HTMLElement, stop: HTMLElement) { let offset = 0; while (element.offsetParent && element !== stop) { diff --git a/web/tsconfig.json b/web/tsconfig.json index 31aef23e31b..c7bc16f52bd 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -4,7 +4,7 @@ "checkJs": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, - "module": "es2020", + "module": "es2022", "moduleResolution": "bundler", "resolveJsonModule": true, "skipLibCheck": true, diff --git a/web/vite.config.js b/web/vite.config.js index 5d134beab08..9ab90e4873d 100644 --- a/web/vite.config.js +++ b/web/vite.config.js @@ -4,6 +4,7 @@ import { svelteTesting } from '@testing-library/svelte/vite'; import path from 'node:path'; import { visualizer } from 'rollup-plugin-visualizer'; import { defineConfig } from 'vite'; +import wasm from 'vite-plugin-wasm'; const upstream = { target: process.env.IMMICH_SERVER_URL || 'http://immich-server:2283/', @@ -14,6 +15,9 @@ const upstream = { }; export default defineConfig({ + build: { + target: 'es2022', + }, resolve: { alias: { 'xmlhttprequest-ssl': './node_modules/engine.io-client/lib/xmlhttprequest.js', @@ -40,6 +44,7 @@ export default defineConfig({ : undefined, enhancedImages(), svelteTesting(), + wasm(), ], optimizeDeps: { entries: ['src/**/*.{svelte,ts,html}'], @@ -52,5 +57,9 @@ export default defineConfig({ sequence: { hooks: 'list', }, + deps: { + // workaround for https://github.com/vitest-dev/vitest/issues/2150 + inline: ['@immich/justified-layout-wasm'], + }, }, });