diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d98e99a6ae..a0e8f51b067 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -648,7 +648,7 @@ jobs: contents: read services: postgres: - image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0@sha256:1076bb152a3000df23911fdec83f14ea83f0dd0c42bc7d4e14b854e9bda1b0c9 + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.1 env: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 11a9ee7f261..35b98a35f39 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -122,7 +122,7 @@ services: database: container_name: immich_postgres - image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0@sha256:fa4f6e0971f454cd95fec5a9aaed2ed93d8f46725cc6bc61e0698e97dba96da1 + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.1-pgvectors0.2.0 env_file: - .env environment: diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 7fd28d49202..412d9cccdd8 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -63,7 +63,7 @@ services: database: container_name: immich_postgres - image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0@sha256:fa4f6e0971f454cd95fec5a9aaed2ed93d8f46725cc6bc61e0698e97dba96da1 + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.1-pgvectors0.2.0 env_file: - .env environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6bc4bea6029..aec55fe9200 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -56,7 +56,7 @@ services: database: container_name: immich_postgres - image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0@sha256:fa4f6e0971f454cd95fec5a9aaed2ed93d8f46725cc6bc61e0698e97dba96da1 + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.1-pgvectors0.2.0 environment: POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_USER: ${DB_USERNAME} diff --git a/docs/docs/administration/postgres-standalone.md b/docs/docs/administration/postgres-standalone.md index 9f0776a3637..d9ad3318106 100644 --- a/docs/docs/administration/postgres-standalone.md +++ b/docs/docs/administration/postgres-standalone.md @@ -19,7 +19,7 @@ You must install VectorChord into your instance of Postgres using their [instruc :::note Immich is known to work with Postgres versions `>= 14, < 18`. -Make sure the installed version of VectorChord is compatible with your version of Immich. The current accepted range for VectorChord is `>= 0.3.0, < 0.4.0`. +Make sure the installed version of VectorChord is compatible with your version of Immich. The current accepted range for VectorChord is `>= 0.3.0, < 0.5.0`. ::: ## Specifying the connection URL diff --git a/server/src/constants.ts b/server/src/constants.ts index 5a5984ab6ea..2e257979384 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -4,7 +4,7 @@ import { SemVer } from 'semver'; import { DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; export const POSTGRES_VERSION_RANGE = '>=14.0.0'; -export const VECTORCHORD_VERSION_RANGE = '>=0.3 <0.4'; +export const VECTORCHORD_VERSION_RANGE = '>=0.3 <0.5'; export const VECTORS_VERSION_RANGE = '>=0.2 <0.4'; export const VECTOR_VERSION_RANGE = '>=0.5 <1'; diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index 1aff2d1a1e6..1e8e147c430 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -144,20 +144,22 @@ export class DatabaseRepository { const isVectors = extension === DatabaseExtension.VECTORS; let restartRequired = false; + const diff = semver.diff(installedVersion, targetVersion); await this.db.transaction().execute(async (tx) => { await this.setSearchPath(tx); await sql`ALTER EXTENSION ${sql.raw(extension)} UPDATE TO ${sql.lit(targetVersion)}`.execute(tx); - const diff = semver.diff(installedVersion, targetVersion); if (isVectors && (diff === 'major' || diff === 'minor')) { await sql`SELECT pgvectors_upgrade()`.execute(tx); restartRequired = true; - } else if (diff) { - await Promise.all([this.reindexVectors(VectorIndex.CLIP), this.reindexVectors(VectorIndex.FACE)]); } }); + if (diff && !restartRequired) { + await Promise.all([this.reindexVectors(VectorIndex.CLIP), this.reindexVectors(VectorIndex.FACE)]); + } + return { restartRequired }; } @@ -204,24 +206,20 @@ export class DatabaseRepository { const matches = row.indexdef.match(/(?<=lists = \[)\d+/g); const lists = matches && matches.length > 0 ? Number(matches[0]) : 1; promises.push( - this.db - .selectFrom(this.db.dynamic.table(table).as('t')) - .select((eb) => eb.fn.countAll().as('count')) - .executeTakeFirstOrThrow() - .then(({ count }) => { - const targetLists = this.targetListCount(count); - this.logger.log(`targetLists=${targetLists}, current=${lists} for ${indexName} of ${count} rows`); - if ( - !row.indexdef.toLowerCase().includes('using vchordrq') || - // slack factor is to avoid frequent reindexing if the count is borderline - (lists !== targetLists && lists !== this.targetListCount(count * VECTORCHORD_LIST_SLACK_FACTOR)) - ) { - probes[indexName] = this.targetProbeCount(targetLists); - return this.reindexVectors(indexName, { lists: targetLists }); - } else { - probes[indexName] = this.targetProbeCount(lists); - } - }), + this.getRowCount(table).then((count) => { + const targetLists = this.targetListCount(count); + this.logger.log(`targetLists=${targetLists}, current=${lists} for ${indexName} of ${count} rows`); + if ( + !row.indexdef.toLowerCase().includes('using vchordrq') || + // slack factor is to avoid frequent reindexing if the count is borderline + (lists !== targetLists && lists !== this.targetListCount(count * VECTORCHORD_LIST_SLACK_FACTOR)) + ) { + probes[indexName] = this.targetProbeCount(targetLists); + return this.reindexVectors(indexName, { lists: targetLists }); + } else { + probes[indexName] = this.targetProbeCount(lists); + } + }), ); break; } @@ -237,6 +235,7 @@ export class DatabaseRepository { this.logger.log(`Reindexing ${indexName}`); const table = VECTOR_INDEX_TABLES[indexName]; const vectorExtension = await getVectorExtension(this.db); + const { rows } = await sql<{ columnName: string; }>`SELECT column_name as "columnName" FROM information_schema.columns WHERE table_name = ${table}`.execute(this.db); @@ -263,6 +262,7 @@ export class DatabaseRepository { ALTER TABLE ${sql.raw(table)} ALTER COLUMN embedding SET DATA TYPE ${sql.raw(schema)}vector(${sql.raw(String(dimSize))})`.execute(tx); + lists ||= this.targetListCount(await this.getRowCount(table)); await sql.raw(vectorIndexQuery({ vectorExtension, table, indexName, lists })).execute(tx); }); try { @@ -350,6 +350,14 @@ export class DatabaseRepository { return Math.ceil(lists / 8); } + private async getRowCount(table: keyof DB): Promise { + const { count } = await this.db + .selectFrom(this.db.dynamic.table(table).as('t')) + .select((eb) => eb.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + return count; + } + async runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise { const { database } = this.configRepository.getEnv(); diff --git a/server/test/medium/globalSetup.ts b/server/test/medium/globalSetup.ts index 91f47a8ca71..c1bc755a0e1 100644 --- a/server/test/medium/globalSetup.ts +++ b/server/test/medium/globalSetup.ts @@ -7,7 +7,7 @@ import { getKyselyConfig } from 'src/utils/database'; import { GenericContainer, Wait } from 'testcontainers'; const globalSetup = async () => { - const postgresContainer = await new GenericContainer('ghcr.io/immich-app/postgres:14-vectorchord0.3.0') + const postgresContainer = await new GenericContainer('ghcr.io/immich-app/postgres:14-vectorchord0.4.1') .withExposedPorts(5432) .withEnvironment({ POSTGRES_PASSWORD: 'postgres',