feat(web): use wasm for justified layout calculation (#15524)

* working

* use wrapper class

* update import

* simplify

* it works without changing `optimizeDeps`

* inline layout options

* update gallery view

* use es2022

* fix import

* fix vitest

* empty geometry

* bump version

* Update web/src/lib/stores/assets.store.ts

Co-authored-by: Jason Rasmussen <jason@rasm.me>

* fix: typo

---------

Co-authored-by: Jason Rasmussen <jason@rasm.me>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

web/package-lock.json

@@ -10,6 +10,7 @@
       "license": "GNU Affero General Public License version 3",
       "dependencies": {
         "@formatjs/icu-messageformat-parser": "^2.9.8",
+        "@immich/justified-layout-wasm": "^0.1.2",
         "@immich/sdk": "file:../open-api/typescript-sdk",
         "@immich/ui": "^0.16.0",
         "@mapbox/mapbox-gl-rtl-text": "0.2.3",
@@ -23,7 +24,6 @@
         "dom-to-image": "^2.6.0",
         "handlebars": "^4.7.8",
         "intl-messageformat": "^10.7.11",
-        "justified-layout": "^4.1.0",
         "lodash-es": "^4.17.21",
         "luxon": "^3.4.4",
         "socket.io-client": "~4.8.0",
@@ -45,7 +45,6 @@
         "@testing-library/svelte": "^5.2.6",
         "@testing-library/user-event": "^14.5.2",
         "@types/dom-to-image": "^2.6.7",
-        "@types/justified-layout": "^4.1.4",
         "@types/lodash-es": "^4.17.12",
         "@types/luxon": "^3.4.2",
         "@typescript-eslint/eslint-plugin": "^8.20.0",
@@ -71,6 +70,7 @@
         "tslib": "^2.6.2",
         "typescript": "^5.7.3",
         "vite": "^6.0.0",
+        "vite-plugin-wasm": "^3.4.1",
         "vitest": "^3.0.0"
       }
     },
@@ -1339,6 +1339,12 @@
         "url": "https://opencollective.com/libvips"
       }
     },
+    "node_modules/@immich/justified-layout-wasm": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.1.2.tgz",
+      "integrity": "sha512-6AmzYJhLedzIXSkhO/0tfBbHAeUeLmG1c4yTzJmtuSGyn7JAzVCFp0dp4T8Wh1tfIDx0Y0pAYB9tm2xlJHdEPA==",
+      "license": "AGPL-3"
+    },
     "node_modules/@immich/sdk": {
       "resolved": "../open-api/typescript-sdk",
       "link": true
@@ -2413,12 +2419,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/@types/justified-layout": {
-      "version": "4.1.4",
-      "resolved": "https://registry.npmjs.org/@types/justified-layout/-/justified-layout-4.1.4.tgz",
-      "integrity": "sha512-q2ybP0u0NVj87oMnGZOGxY2iUN8ddr48zPOBHBdbOLpsMTA/keGj+93ou+OMCnJk0xewzlNIaVEkxM6VBD3E2w==",
-      "dev": true
-    },
     "node_modules/@types/leaflet": {
       "version": "1.9.8",
       "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.8.tgz",
@@ -5382,11 +5382,6 @@
       "resolved": "https://registry.npmjs.org/just-flush/-/just-flush-2.3.0.tgz",
       "integrity": "sha512-fBuxQ1gJ61BurmhwKS5LYTzhkbrT5j/2U7ax+UbLm9aRvCTh2h6AfzLteOckE4KKomqOf0Y3zIG3Xu57sRsKUg=="
     },
-    "node_modules/justified-layout": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/justified-layout/-/justified-layout-4.1.0.tgz",
-      "integrity": "sha512-M5FimNMXgiOYerVRGsXZ2YK9YNCaTtwtYp7Hb2308U1Q9TXXHx5G0p08mcVR5O53qf8bWY4NJcPBxE6zuayXSg=="
-    },
     "node_modules/kdbush": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz",
@@ -8625,6 +8620,16 @@
         }
       }
     },
+    "node_modules/vite-plugin-wasm": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/vite-plugin-wasm/-/vite-plugin-wasm-3.4.1.tgz",
+      "integrity": "sha512-ja3nSo2UCkVeitltJGkS3pfQHAanHv/DqGatdI39ja6McgABlpsZ5hVgl6wuR8Qx5etY3T5qgDQhOWzc5RReZA==",
+      "dev": true,
+      "license": "MIT",
+      "peerDependencies": {
+        "vite": "^2 || ^3 || ^4 || ^5 || ^6"
+      }
+    },
     "node_modules/vitefu": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.0.5.tgz",

web/package.json

@@ -35,7 +35,6 @@
     "@testing-library/svelte": "^5.2.6",
     "@testing-library/user-event": "^14.5.2",
     "@types/dom-to-image": "^2.6.7",
-    "@types/justified-layout": "^4.1.4",
     "@types/lodash-es": "^4.17.12",
     "@types/luxon": "^3.4.2",
     "@typescript-eslint/eslint-plugin": "^8.20.0",
@@ -61,11 +60,13 @@
     "tslib": "^2.6.2",
     "typescript": "^5.7.3",
     "vite": "^6.0.0",
+    "vite-plugin-wasm": "^3.4.1",
     "vitest": "^3.0.0"
   },
   "type": "module",
   "dependencies": {
     "@formatjs/icu-messageformat-parser": "^2.9.8",
+    "@immich/justified-layout-wasm": "^0.1.2",
     "@immich/sdk": "file:../open-api/typescript-sdk",
     "@immich/ui": "^0.16.0",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",
@@ -79,7 +80,6 @@
     "dom-to-image": "^2.6.0",
     "handlebars": "^4.7.8",
     "intl-messageformat": "^10.7.11",
-    "justified-layout": "^4.1.0",
     "lodash-es": "^4.17.21",
     "luxon": "^3.4.4",
     "socket.io-client": "~4.8.0",

web/src/lib/components/photos-page/asset-date-group.svelte

@@ -97,6 +97,7 @@
   {#each dateGroups as dateGroup, groupIndex (dateGroup.date)}
     {@const display =
       dateGroup.intersecting || !!dateGroup.assets.some((asset) => asset.id === $assetStore.pendingScrollAssetId)}
+    {@const geometry = dateGroup.geometry}
 
     <div
       id="date-group"
@@ -118,7 +119,7 @@
       data-display={display}
       data-date-group={dateGroup.date}
       style:height={dateGroup.height + 'px'}
-      style:width={dateGroup.geometry.containerWidth + 'px'}
+      style:width={geometry.containerWidth + 'px'}
       style:overflow={'clip'}
     >
       {#if !display}
@@ -149,7 +150,7 @@
           <!-- Date group title -->
           <div
             class="flex z-[100] sticky top-[-1px] pt-[calc(1.75rem+1px)] pb-5 h-6 place-items-center text-xs font-medium text-immich-fg bg-immich-bg dark:bg-immich-dark-bg dark:text-immich-dark-fg md:text-sm"
-            style:width={dateGroup.geometry.containerWidth + 'px'}
+            style:width={geometry.containerWidth + 'px'}
           >
             {#if !singleSelect && ((hoveredDateGroup == dateGroup.groupTitle && isMouseOverGroup) || assetInteraction.selectedGroup.has(dateGroup.groupTitle))}
               <div
@@ -174,12 +175,17 @@
           <!-- Image grid -->
           <div
             class="relative overflow-clip"
-            style:height={dateGroup.geometry.containerHeight + 'px'}
+            style:height={geometry.containerHeight + 'px'}
-            style:width={dateGroup.geometry.containerWidth + 'px'}
+            style:width={geometry.containerWidth + 'px'}
           >
-            {#each dateGroup.assets as asset, index (asset.id)}
+            {#each dateGroup.assets as asset, i (asset.id)}
-              {@const box = dateGroup.geometry.boxes[index]}
               {@const isSmallGroup = dateGroup.assets.length <= SMALL_GROUP_THRESHOLD}
+              <!-- getting these together here in this order is very cache-efficient -->
+              {@const top = geometry.getTop(i)}
+              {@const left = geometry.getLeft(i)}
+              {@const width = geometry.getWidth(i)}
+              {@const height = geometry.getHeight(i)}
+
               <!-- update ASSET_GRID_PADDING-->
               <div
                 use:intersectionObserver={{
@@ -191,10 +197,10 @@
                 }}
                 data-asset-id={asset.id}
                 class="absolute"
-                style:width={box.width + 'px'}
+                style:top={top + 'px'}
-                style:height={box.height + 'px'}
+                style:left={left + 'px'}
-                style:top={box.top + 'px'}
+                style:width={width + 'px'}
-                style:left={box.left + 'px'}
+                style:height={height + 'px'}
               >
                 <Thumbnail
                   {dateGroup}
@@ -216,8 +222,8 @@
                   selected={assetInteraction.selectedAssets.has(asset) || $assetStore.albumAssets.has(asset.id)}
                   selectionCandidate={assetInteraction.assetSelectionCandidates.has(asset)}
                   disabled={$assetStore.albumAssets.has(asset.id)}
-                  thumbnailWidth={box.width}
+                  thumbnailWidth={width}
-                  thumbnailHeight={box.height}
+                  thumbnailHeight={height}
                   eagerThumbhash={isSmallGroup}
                 />
               </div>

web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte

@@ -8,13 +8,11 @@
   import type { Viewport } from '$lib/stores/assets.store';
   import { showDeleteModal } from '$lib/stores/preferences.store';
   import { deleteAssets } from '$lib/utils/actions';
-  import { archiveAssets, cancelMultiselect, getAssetRatio } from '$lib/utils/asset-utils';
+  import { archiveAssets, cancelMultiselect, getJustifiedLayoutFromAssets } from '$lib/utils/asset-utils';
   import { featureFlags } from '$lib/stores/server-config.store';
   import { handleError } from '$lib/utils/handle-error';
   import { navigate } from '$lib/utils/navigation';
-  import { calculateWidth } from '$lib/utils/timeline-util';
   import { type AssetResponseDto } from '@immich/sdk';
-  import justifiedLayout from 'justified-layout';
   import { t } from 'svelte-i18n';
   import AssetViewer from '../../asset-viewer/asset-viewer.svelte';
   import ShowShortcuts from '../show-shortcuts.svelte';
@@ -310,23 +308,12 @@
   let idsSelectedAssets = $derived(assetInteraction.selectedAssetsArray.map(({ id }) => id));
 
   let geometry = $derived(
-    (() => {
+    getJustifiedLayoutFromAssets(assets, {
-      const justifiedLayoutResult = justifiedLayout(
+      spacing: 2,
-        assets.map((asset) => getAssetRatio(asset)),
+      rowWidth: Math.floor(viewport.width),
-        {
+      heightTolerance: 0.15,
-          boxSpacing: 2,
+      rowHeight: 235,
-          containerWidth: Math.floor(viewport.width),
+    }),
-          containerPadding: 0,
-          targetRowHeightTolerance: 0.15,
-          targetRowHeight: 235,
-        },
-      );
-
-      return {
-        ...justifiedLayoutResult,
-        containerWidth: calculateWidth(justifiedLayoutResult.boxes),
-      };
-    })(),
   );
 
   $effect(() => {
@@ -364,11 +351,15 @@
 
 {#if assets.length > 0}
   <div class="relative" style="height: {geometry.containerHeight}px;width: {geometry.containerWidth}px ">
-    {#each assets as asset, i (i)}
+    {#each assets as asset, i}
+      {@const top = geometry.getTop(i)}
+      {@const left = geometry.getLeft(i)}
+      {@const width = geometry.getWidth(i)}
+      {@const height = geometry.getHeight(i)}
+
       <div
         class="absolute"
-        style="width: {geometry.boxes[i].width}px; height: {geometry.boxes[i].height}px; top: {geometry.boxes[i]
+        style="width: {width}px; height: {height}px; top: {top}px; left: {left}px"
-          .top}px; left: {geometry.boxes[i].left}px"
         title={showAssetName ? asset.originalFileName : ''}
       >
         <Thumbnail
@@ -387,8 +378,8 @@
           {asset}
           selected={assetInteraction.selectedAssets.has(asset)}
           selectionCandidate={assetInteraction.assetSelectionCandidates.has(asset)}
-          thumbnailWidth={geometry.boxes[i].width}
+          thumbnailWidth={width}
-          thumbnailHeight={geometry.boxes[i].height}
+          thumbnailHeight={height}
         />
         {#if showAssetName}
           <div

web/src/lib/stores/assets.store.ts

@@ -1,12 +1,11 @@
 import { locale } from '$lib/stores/preferences.store';
 import { getKey } from '$lib/utils';
 import { AssetGridTaskManager } from '$lib/utils/asset-store-task-manager';
-import { getAssetRatio } from '$lib/utils/asset-utils';
+import { getJustifiedLayoutFromAssets } from '$lib/utils/asset-utils';
 import { generateId } from '$lib/utils/generate-id';
 import type { AssetGridRouteSearchParams } from '$lib/utils/navigation';
-import { calculateWidth, fromLocalDateTime, splitBucketIntoDateGroups, type DateGroup } from '$lib/utils/timeline-util';
+import { fromLocalDateTime, splitBucketIntoDateGroups, type DateGroup } from '$lib/utils/timeline-util';
 import { TimeBucketSize, getAssetInfo, getTimeBucket, getTimeBuckets, type AssetResponseDto } from '@immich/sdk';
-import createJustifiedLayout from 'justified-layout';
 import { throttle } from 'lodash-es';
 import { DateTime } from 'luxon';
 import { t } from 'svelte-i18n';
@@ -16,13 +15,6 @@ import { websocketEvents } from './websocket';
 type AssetApiGetTimeBucketsRequest = Parameters<typeof getTimeBuckets>[0];
 export type AssetStoreOptions = Omit<AssetApiGetTimeBucketsRequest, 'size'>;
 
-const LAYOUT_OPTIONS = {
-  boxSpacing: 2,
-  containerPadding: 0,
-  targetRowHeightTolerance: 0.15,
-  targetRowHeight: 235,
-};
-
 export interface Viewport {
   width: number;
   height: number;
@@ -470,32 +462,30 @@ export class AssetStore {
         assetGroup.heightActual = false;
       }
     }
+
+    const viewportWidth = this.viewport.width;
     if (!bucket.isBucketHeightActual) {
       const unwrappedWidth = (3 / 2) * bucket.bucketCount * THUMBNAIL_HEIGHT * (7 / 10);
-      const rows = Math.ceil(unwrappedWidth / this.viewport.width);
+      const rows = Math.ceil(unwrappedWidth / viewportWidth);
       const height = 51 + rows * THUMBNAIL_HEIGHT;
       bucket.bucketHeight = height;
     }
 
+    const layoutOptions = {
+      spacing: 2,
+      heightTolerance: 0.15,
+      rowHeight: 235,
+      rowWidth: Math.floor(viewportWidth),
+    };
     for (const assetGroup of bucket.dateGroups) {
       if (!assetGroup.heightActual) {
         const unwrappedWidth = (3 / 2) * assetGroup.assets.length * THUMBNAIL_HEIGHT * (7 / 10);
-        const rows = Math.ceil(unwrappedWidth / this.viewport.width);
+        const rows = Math.ceil(unwrappedWidth / viewportWidth);
         const height = rows * THUMBNAIL_HEIGHT;
         assetGroup.height = height;
       }
 
-      const layoutResult = createJustifiedLayout(
+      assetGroup.geometry = getJustifiedLayoutFromAssets(assetGroup.assets, layoutOptions);
-        assetGroup.assets.map((g) => getAssetRatio(g)),
-        {
-          ...LAYOUT_OPTIONS,
-          containerWidth: Math.floor(this.viewport.width),
-        },
-      );
-      assetGroup.geometry = {
-        ...layoutResult,
-        containerWidth: calculateWidth(layoutResult.boxes),
-      };
     }
   }
 

web/src/lib/utils/asset-utils.ts

@@ -12,6 +12,7 @@ import { downloadRequest, getKey, withError } from '$lib/utils';
 import { createAlbum } from '$lib/utils/album-utils';
 import { getByteUnitString } from '$lib/utils/byte-units';
 import { getFormatter } from '$lib/utils/i18n';
+import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
 import {
   addAssetsToAlbum as addAssets,
   createStack,
@@ -587,3 +588,13 @@ export const copyImageToClipboard = async (source: HTMLImageElement | string) =>
   const blob = source instanceof HTMLImageElement ? await imgToBlob(source) : await urlToBlob(source);
   await navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]);
 };
+
+export function getJustifiedLayoutFromAssets(assets: AssetResponseDto[], options: LayoutOptions) {
+  const aspectRatios = new Float32Array(assets.length);
+  // eslint-disable-next-line unicorn/no-for-loop
+  for (let i = 0; i < assets.length; i++) {
+    const { width, height } = getAssetRatio(assets[i]);
+    aspectRatios[i] = width / height;
+  }
+  return new JustifiedLayout(aspectRatios, options);
+}

web/src/lib/utils/timeline-util.ts

@@ -1,7 +1,7 @@
 import type { AssetBucket } from '$lib/stores/assets.store';
 import { locale } from '$lib/stores/preferences.store';
+import { JustifiedLayout } from '@immich/justified-layout-wasm';
 import type { AssetResponseDto } from '@immich/sdk';
-import type createJustifiedLayout from 'justified-layout';
 import { groupBy, memoize, sortBy } from 'lodash-es';
 import { DateTime } from 'luxon';
 import { get } from 'svelte/store';
@@ -13,7 +13,7 @@ export type DateGroup = {
   height: number;
   heightActual: boolean;
   intersecting: boolean;
-  geometry: Geometry;
+  geometry: JustifiedLayout;
   bucket: AssetBucket;
 };
 export type ScrubberListener = (
@@ -80,18 +80,12 @@ export function formatGroupTitle(_date: DateTime): string {
   return date.toLocaleString(groupDateFormat);
 }
 
-type Geometry = ReturnType<typeof createJustifiedLayout> & {
+const emptyGeometry = new JustifiedLayout(Float32Array.from([]), {
-  containerWidth: number;
+  rowHeight: 1,
-};
+  heightTolerance: 0,
-
+  rowWidth: 1,
-function emptyGeometry() {
+  spacing: 0,
-  return {
+});
-    containerWidth: 0,
-    containerHeight: 0,
-    widowCount: 0,
-    boxes: [],
-  };
-}
 
 const formatDateGroupTitle = memoize(formatGroupTitle);
 
@@ -100,6 +94,7 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string |
     fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat, { locale }),
   );
   const sorted = sortBy(grouped, (group) => bucket.assets.indexOf(group[0]));
+
   return sorted.map((group) => {
     const date = fromLocalDateTime(group[0].localDateTime).startOf('day');
     return {
@@ -109,31 +104,12 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string |
       height: 0,
       heightActual: false,
       intersecting: false,
-      geometry: emptyGeometry(),
+      geometry: emptyGeometry,
       bucket,
     };
   });
 }
 
-export type LayoutBox = {
-  aspectRatio: number;
-  top: number;
-  width: number;
-  height: number;
-  left: number;
-  forcedAspectRatio?: boolean;
-};
-
-export function calculateWidth(boxes: LayoutBox[]): number {
-  let width = 0;
-  for (const box of boxes) {
-    if (box.top < 100) {
-      width = box.left + box.width;
-    }
-  }
-  return width;
-}
-
 export function findTotalOffset(element: HTMLElement, stop: HTMLElement) {
   let offset = 0;
   while (element.offsetParent && element !== stop) {

web/tsconfig.json

@@ -4,7 +4,7 @@
     "checkJs": true,
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
-    "module": "es2020",
+    "module": "es2022",
     "moduleResolution": "bundler",
     "resolveJsonModule": true,
     "skipLibCheck": true,

web/vite.config.js

@@ -4,6 +4,7 @@ import { svelteTesting } from '@testing-library/svelte/vite';
 import path from 'node:path';
 import { visualizer } from 'rollup-plugin-visualizer';
 import { defineConfig } from 'vite';
+import wasm from 'vite-plugin-wasm';
 
 const upstream = {
   target: process.env.IMMICH_SERVER_URL || 'http://immich-server:2283/',
@@ -14,6 +15,9 @@ const upstream = {
 };
 
 export default defineConfig({
+  build: {
+    target: 'es2022',
+  },
   resolve: {
     alias: {
       'xmlhttprequest-ssl': './node_modules/engine.io-client/lib/xmlhttprequest.js',
@@ -40,6 +44,7 @@ export default defineConfig({
       : undefined,
     enhancedImages(),
     svelteTesting(),
+    wasm(),
   ],
   optimizeDeps: {
     entries: ['src/**/*.{svelte,ts,html}'],
@@ -52,5 +57,9 @@ export default defineConfig({
     sequence: {
       hooks: 'list',
     },
+    deps: {
+      // workaround for https://github.com/vitest-dev/vitest/issues/2150
+      inline: ['@immich/justified-layout-wasm'],
+    },
   },
 });