diff --git a/app/models/assistant/function/search_family_files.rb b/app/models/assistant/function/search_family_files.rb index c9c917f0a..3f5f914ff 100644 --- a/app/models/assistant/function/search_family_files.rb +++ b/app/models/assistant/function/search_family_files.rb @@ -71,7 +71,10 @@ class Assistant::Function::SearchFamilyFiles < Assistant::Function return { success: false, error: "provider_not_configured", - message: "No vector store is configured. Set VECTOR_STORE_PROVIDER or configure OpenAI." + message: "No vector store is configured. Set VECTOR_STORE_PROVIDER " \ + "(openai | pgvector | qdrant), configure OpenAI, or — for " \ + "Anthropic-only installs — enable the pgvector adapter and " \ + "point EMBEDDING_URI_BASE at an embeddings endpoint." } end diff --git a/app/models/vector_store/pgvector.rb b/app/models/vector_store/pgvector.rb index a434ec1f5..2331bd335 100644 --- a/app/models/vector_store/pgvector.rb +++ b/app/models/vector_store/pgvector.rb @@ -15,6 +15,25 @@ class VectorStore::Pgvector < VectorStore::Base PGVECTOR_SUPPORTED_EXTENSIONS = (VectorStore::Embeddable::TEXT_EXTENSIONS + [ ".pdf" ]).uniq.freeze + TABLE_NAME = "vector_store_chunks" + + # True when this adapter can actually operate: the chunks table already + # exists, or the server has the pgvector extension available so + # ensure_schema! can provision it on first use. The Registry consults this + # before building the adapter, so an install without pgvector degrades to + # the assistant's friendly "provider_not_configured" message instead of + # raising raw PG errors mid-chat. + def self.available? + conn = ActiveRecord::Base.connection + return true if conn.table_exists?(TABLE_NAME) + + conn.select_value( + "SELECT 1 FROM pg_available_extensions WHERE name = 'vector' LIMIT 1" + ).present? + rescue StandardError + false + end + def supported_extensions PGVECTOR_SUPPORTED_EXTENSIONS end @@ -27,6 +46,7 @@ class VectorStore::Pgvector < VectorStore::Base def delete_store(store_id:) with_response do + ensure_schema! connection.exec_delete( "DELETE FROM vector_store_chunks WHERE store_id = $1", "VectorStore::Pgvector DeleteStore", @@ -37,6 +57,7 @@ class VectorStore::Pgvector < VectorStore::Base def upload_file(store_id:, file_content:, filename:) with_response do + ensure_schema! text = extract_text(file_content, filename) raise VectorStore::Error, "Could not extract text from #{filename}" if text.blank? @@ -81,6 +102,7 @@ class VectorStore::Pgvector < VectorStore::Base def remove_file(store_id:, file_id:) with_response do + ensure_schema! connection.exec_delete( "DELETE FROM vector_store_chunks WHERE store_id = $1 AND file_id = $2", "VectorStore::Pgvector RemoveFile", @@ -94,6 +116,7 @@ class VectorStore::Pgvector < VectorStore::Base def search(store_id:, query:, max_results: 10) with_response do + ensure_schema! query_vector = embed(query) vector_literal = "[#{query_vector.join(',')}]" @@ -127,6 +150,44 @@ class VectorStore::Pgvector < VectorStore::Base private + # Provisions the chunks table on first use, mirroring the + # CreateVectorStoreChunks migration. Migrations cover db:migrate + # upgrades, but fresh installs go through db:prepare → schema:load + # (bin/docker-entrypoint), which marks conditional migrations as applied + # without running them — and the table can't live in schema.rb because it + # requires the vector extension. Idempotent; memoized per instance. + def ensure_schema! + return if @schema_ensured + if connection.table_exists?(TABLE_NAME) + @schema_ensured = true + return + end + + connection.enable_extension("vector") unless connection.extension_enabled?("vector") + # if_not_exists on the DDL (not a Mutex) is the right concurrency guard + # here: adapter instances are built per call and never shared across + # threads, so the realistic race is two *processes* (e.g. web + Sidekiq) + # provisioning at once. IF NOT EXISTS makes the loser's DDL a no-op + # instead of a duplicate-relation error. + connection.create_table(TABLE_NAME, id: :uuid, if_not_exists: true) do |t| + t.string :store_id, null: false + t.string :file_id, null: false + t.string :filename + t.integer :chunk_index, null: false, default: 0 + t.text :content, null: false + t.column :embedding, "vector(#{ENV.fetch('EMBEDDING_DIMENSIONS', '1024')})", null: false + t.jsonb :metadata, null: false, default: {} + t.timestamps null: false + end + connection.add_index TABLE_NAME, :store_id, if_not_exists: true + connection.add_index TABLE_NAME, :file_id, if_not_exists: true + connection.add_index TABLE_NAME, [ :store_id, :file_id, :chunk_index ], unique: true, + name: "index_vector_store_chunks_on_store_file_chunk", if_not_exists: true + @schema_ensured = true + rescue StandardError => e + raise VectorStore::Error, "pgvector store unavailable: #{e.message}" + end + def connection ActiveRecord::Base.connection end diff --git a/app/models/vector_store/registry.rb b/app/models/vector_store/registry.rb index 10c73d770..fcc470661 100644 --- a/app/models/vector_store/registry.rb +++ b/app/models/vector_store/registry.rb @@ -7,8 +7,10 @@ class VectorStore::Registry class << self # Returns the configured adapter instance. - # Reads from VECTOR_STORE_PROVIDER env var, falling back to :openai - # when OpenAI credentials are present. + # Reads from VECTOR_STORE_PROVIDER env var; without an explicit override, + # Anthropic installs (Setting.llm_provider == "anthropic") default to + # :pgvector, and anything else falls back to :openai when OpenAI + # credentials are present. def adapter name = adapter_name return nil unless name @@ -24,10 +26,27 @@ class VectorStore::Registry explicit = ENV["VECTOR_STORE_PROVIDER"].presence return explicit.to_sym if explicit && ADAPTERS.key?(explicit.to_sym) - # Default: use OpenAI when credentials are available + # Default routing: + # - When the configured LLM provider is Anthropic (which has no hosted + # vector store), fall back to the local pgvector adapter. The + # Embeddable concern still pulls embeddings from EMBEDDING_URI_BASE / + # OPENAI_ACCESS_TOKEN — Anthropic users typically point this at + # Voyage AI, a local Ollama instance, or OpenAI embeddings. + # - Otherwise, use OpenAI when credentials are available. + return :pgvector if Setting.llm_provider == "anthropic" :openai if openai_access_token.present? end + # True when pgvector is the effective vector store — whether set explicitly + # via VECTOR_STORE_PROVIDER or selected by the Anthropic default above. + # Single source of truth shared with the migration that provisions + # `vector_store_chunks`, so the table is created exactly when pgvector is in + # use (an Anthropic-default install would otherwise skip it and fail on the + # missing table). + def pgvector_effective? + adapter_name == :pgvector + end + private def build_adapter(name) @@ -53,6 +72,12 @@ class VectorStore::Registry end def build_pgvector + # Gate on availability (extension present, or table already created) + # so an Anthropic-default install on a Postgres without pgvector + # degrades to the assistant's "provider_not_configured" message + # instead of raising raw PG errors mid-chat. + return nil unless VectorStore::Pgvector.available? + VectorStore::Pgvector.new end diff --git a/db/migrate/20260316120000_create_vector_store_chunks.rb b/db/migrate/20260316120000_create_vector_store_chunks.rb index 216768486..c65e33c1a 100644 --- a/db/migrate/20260316120000_create_vector_store_chunks.rb +++ b/db/migrate/20260316120000_create_vector_store_chunks.rb @@ -28,14 +28,20 @@ class CreateVectorStoreChunks < ActiveRecord::Migration[7.2] private - # Only run this migration when pgvector is explicitly configured as the - # vector store provider AND the extension is actually available on the - # PostgreSQL server. Previously we only checked server availability, - # which caused failures in production Docker environments where the - # extension may be present but the DB user lacks superuser privileges + # Only run this migration when pgvector is the effective vector store AND + # the extension is actually available on the PostgreSQL server. + # + # Provider selection goes through VectorStore::Registry.pgvector_effective? + # (the single source of truth) rather than a raw VECTOR_STORE_PROVIDER check, + # so an Anthropic-default install — which selects pgvector implicitly via + # Setting.llm_provider without setting VECTOR_STORE_PROVIDER — still + # provisions the table instead of failing later on a missing relation. + # + # The server-availability check stays: production Docker environments may + # have the extension present but the DB user may lack superuser privileges # to enable it. def pgvector_available? - return false unless ENV["VECTOR_STORE_PROVIDER"].to_s.downcase == "pgvector" + return false unless VectorStore::Registry.pgvector_effective? result = ActiveRecord::Base.connection.execute( "SELECT 1 FROM pg_available_extensions WHERE name = 'vector' LIMIT 1" diff --git a/db/migrate/20260601120000_ensure_vector_store_chunks_for_default_pgvector.rb b/db/migrate/20260601120000_ensure_vector_store_chunks_for_default_pgvector.rb new file mode 100644 index 000000000..452717d7b --- /dev/null +++ b/db/migrate/20260601120000_ensure_vector_store_chunks_for_default_pgvector.rb @@ -0,0 +1,55 @@ +class EnsureVectorStoreChunksForDefaultPgvector < ActiveRecord::Migration[7.2] + # CreateVectorStoreChunks only provisions the table when + # VECTOR_STORE_PROVIDER == "pgvector" is set explicitly. Since #1986 makes + # pgvector the *default* vector store for Anthropic installs (no + # VECTOR_STORE_PROVIDER needed), a fresh Anthropic-only install would migrate + # without the table and then fail on uploads/searches. Backfill it whenever + # pgvector is the effective store, idempotently, so fresh and already-migrated + # installs converge. Gating uses the same VectorStore::Registry predicate as + # the runtime adapter selection, so the two can't drift again. + def up + return unless pgvector_effective? + return unless pgvector_extension_available? + return if table_exists?(:vector_store_chunks) + + enable_extension "vector" unless extension_enabled?("vector") + + create_table :vector_store_chunks, id: :uuid do |t| + t.string :store_id, null: false + t.string :file_id, null: false + t.string :filename + t.integer :chunk_index, null: false, default: 0 + t.text :content, null: false + t.column :embedding, "vector(#{ENV.fetch('EMBEDDING_DIMENSIONS', '1024')})", null: false + t.jsonb :metadata, null: false, default: {} + t.timestamps null: false + end + + add_index :vector_store_chunks, :store_id + add_index :vector_store_chunks, :file_id + add_index :vector_store_chunks, [ :store_id, :file_id, :chunk_index ], unique: true, + name: "index_vector_store_chunks_on_store_file_chunk" + end + + def down + # No-op: the table's lifecycle is owned by CreateVectorStoreChunks. This + # migration only backfills it for the pgvector-by-default case, so reverting + # must not drop a table other installs rely on. + end + + private + + def pgvector_effective? + VectorStore::Registry.pgvector_effective? + rescue StandardError + false + end + + def pgvector_extension_available? + ActiveRecord::Base.connection.execute( + "SELECT 1 FROM pg_available_extensions WHERE name = 'vector' LIMIT 1" + ).any? + rescue StandardError + false + end +end diff --git a/db/schema.rb b/db/schema.rb index aead1cb06..a300b338c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_05_31_213000) do +ActiveRecord::Schema[7.2].define(version: 2026_06_01_120000) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" diff --git a/test/models/vector_store/pgvector_test.rb b/test/models/vector_store/pgvector_test.rb index e0393f139..64212861e 100644 --- a/test/models/vector_store/pgvector_test.rb +++ b/test/models/vector_store/pgvector_test.rb @@ -3,6 +3,9 @@ require "test_helper" class VectorStore::PgvectorTest < ActiveSupport::TestCase setup do @adapter = VectorStore::Pgvector.new + # Schema provisioning is exercised by its own tests below; the operation + # tests stub it out so their mock connections only see the op's SQL. + @adapter.stubs(:ensure_schema!) end test "create_store returns a UUID" do @@ -138,4 +141,69 @@ class VectorStore::PgvectorTest < ActiveSupport::TestCase assert_not_includes @adapter.supported_extensions, ".zip" assert_not_includes @adapter.supported_extensions, ".docx" end + + test "ensure_schema! is a no-op when the table already exists" do + adapter = VectorStore::Pgvector.new + + mock_conn = mock("connection") + mock_conn.expects(:table_exists?).with(VectorStore::Pgvector::TABLE_NAME).returns(true) + mock_conn.expects(:create_table).never + adapter.stubs(:connection).returns(mock_conn) + + adapter.send(:ensure_schema!) + # Memoized: a second call must not hit the connection again. + adapter.send(:ensure_schema!) + end + + test "ensure_schema! provisions extension, table, and indexes when missing" do + adapter = VectorStore::Pgvector.new + + mock_conn = mock("connection") + mock_conn.expects(:table_exists?).with(VectorStore::Pgvector::TABLE_NAME).returns(false) + mock_conn.expects(:extension_enabled?).with("vector").returns(false) + mock_conn.expects(:enable_extension).with("vector") + mock_conn.expects(:create_table).with(VectorStore::Pgvector::TABLE_NAME, id: :uuid, if_not_exists: true) + mock_conn.expects(:add_index).times(3) + adapter.stubs(:connection).returns(mock_conn) + + adapter.send(:ensure_schema!) + end + + test "ensure_schema! wraps provisioning failures in VectorStore::Error" do + adapter = VectorStore::Pgvector.new + + mock_conn = mock("connection") + mock_conn.expects(:table_exists?).returns(false) + mock_conn.expects(:extension_enabled?).raises(ActiveRecord::StatementInvalid.new("permission denied")) + adapter.stubs(:connection).returns(mock_conn) + + error = assert_raises(VectorStore::Error) { adapter.send(:ensure_schema!) } + assert_match(/pgvector store unavailable/, error.message) + end + + test "available? is true when the chunks table exists" do + conn = mock("connection") + conn.expects(:table_exists?).with(VectorStore::Pgvector::TABLE_NAME).returns(true) + ActiveRecord::Base.stubs(:connection).returns(conn) + + assert VectorStore::Pgvector.available? + end + + test "available? is true when the extension is available but the table is missing" do + conn = mock("connection") + conn.expects(:table_exists?).returns(false) + conn.expects(:select_value).returns(1) + ActiveRecord::Base.stubs(:connection).returns(conn) + + assert VectorStore::Pgvector.available? + end + + test "available? is false when neither table nor extension is available" do + conn = mock("connection") + conn.expects(:table_exists?).returns(false) + conn.expects(:select_value).returns(nil) + ActiveRecord::Base.stubs(:connection).returns(conn) + + assert_not VectorStore::Pgvector.available? + end end diff --git a/test/models/vector_store/registry_test.rb b/test/models/vector_store/registry_test.rb index 514b30b24..0bd544ed0 100644 --- a/test/models/vector_store/registry_test.rb +++ b/test/models/vector_store/registry_test.rb @@ -43,13 +43,62 @@ class VectorStore::RegistryTest < ActiveSupport::TestCase end end - test "adapter returns VectorStore::Pgvector instance when pgvector configured" do + test "adapter returns VectorStore::Pgvector instance when pgvector configured and available" do + VectorStore::Pgvector.stubs(:available?).returns(true) ClimateControl.modify(VECTOR_STORE_PROVIDER: "pgvector") do adapter = VectorStore::Registry.adapter assert_instance_of VectorStore::Pgvector, adapter end end + test "adapter is nil when pgvector is selected but unavailable" do + VectorStore::Pgvector.stubs(:available?).returns(false) + ClimateControl.modify(VECTOR_STORE_PROVIDER: "pgvector") do + assert_nil VectorStore::Registry.adapter + assert_not VectorStore.configured? + end + end + + test "adapter is nil for the anthropic default when pgvector is unavailable" do + Setting.stubs(:llm_provider).returns("anthropic") + VectorStore::Pgvector.stubs(:available?).returns(false) + VectorStore::Registry.stubs(:openai_access_token).returns(nil) + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert_nil VectorStore::Registry.adapter + end + end + + test "adapter builds pgvector for the anthropic default when available" do + Setting.stubs(:llm_provider).returns("anthropic") + VectorStore::Pgvector.stubs(:available?).returns(true) + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert_instance_of VectorStore::Pgvector, VectorStore::Registry.adapter + end + end + + test "adapter_name defaults to pgvector when LLM_PROVIDER is anthropic" do + Setting.stubs(:llm_provider).returns("anthropic") + VectorStore::Registry.stubs(:openai_access_token).returns(nil) + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert_equal :pgvector, VectorStore::Registry.adapter_name + end + end + + test "adapter_name routes anthropic installs to pgvector even when OpenAI key is present" do + Setting.stubs(:llm_provider).returns("anthropic") + VectorStore::Registry.stubs(:openai_access_token).returns("sk-test") + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert_equal :pgvector, VectorStore::Registry.adapter_name + end + end + + test "explicit VECTOR_STORE_PROVIDER overrides anthropic default" do + Setting.stubs(:llm_provider).returns("anthropic") + ClimateControl.modify(VECTOR_STORE_PROVIDER: "qdrant") do + assert_equal :qdrant, VectorStore::Registry.adapter_name + end + end + test "configured? delegates to adapter presence" do VectorStore::Registry.stubs(:adapter).returns(nil) assert_not VectorStore.configured? @@ -57,4 +106,26 @@ class VectorStore::RegistryTest < ActiveSupport::TestCase VectorStore::Registry.stubs(:adapter).returns(VectorStore::Openai.new(access_token: "sk-test")) assert VectorStore.configured? end + + test "pgvector_effective? is true when pgvector is explicit" do + ClimateControl.modify(VECTOR_STORE_PROVIDER: "pgvector") do + assert VectorStore::Registry.pgvector_effective? + end + end + + test "pgvector_effective? is true for the anthropic default (no explicit provider)" do + Setting.stubs(:llm_provider).returns("anthropic") + VectorStore::Registry.stubs(:openai_access_token).returns(nil) + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert VectorStore::Registry.pgvector_effective? + end + end + + test "pgvector_effective? is false for the openai default" do + Setting.stubs(:llm_provider).returns("openai") + VectorStore::Registry.stubs(:openai_access_token).returns("sk-test") + ClimateControl.modify(VECTOR_STORE_PROVIDER: nil) do + assert_not VectorStore::Registry.pgvector_effective? + end + end end