Add live AI checks to system health (#3155)

* Add live AI checks to system health

Give super admins a dedicated AI status view with bounded liveness probes for LLMs, vector stores, pgvector, and embedding endpoints. Record sanitized failures in both the system debug log and Rails logger, and document the recommended local configuration.\n\nCloses #3145

* Fix AI health CI checks

* Address AI health review feedback

* Correct Ollama model preload guidance

* Distinguish OpenAI-compatible providers

* Make Ollama startup readiness explicit

* Recognize Cloudflare AI endpoints

test/models/ai_health/probe_test.rb

@@ -0,0 +1,198 @@
+require "test_helper"
+
+class AiHealth::ProbeTest < ActiveSupport::TestCase
+  setup do
+    @cache = ActiveSupport::Cache::MemoryStore.new
+    @probe = AiHealth::Probe.new(cache: @cache)
+  end
+
+  test "OpenAI LLM probe calls the models endpoint and verifies the configured model" do
+    request = stub_request(:get, "http://ollama.example.test:11434/v1/models")
+              .with(headers: { "Authorization" => "Bearer local-token" })
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [ { id: "qwen3:8b" } ] }.to_json
+              )
+
+    result = @probe.llm(
+      provider: :openai,
+      endpoint: "http://ollama.example.test:11434/v1",
+      access_token: "local-token",
+      model: "qwen3:8b"
+    )
+
+    assert result.passing?
+    assert result.checked_at
+    assert_requested request
+  end
+
+  test "Anthropic LLM probe calls the models endpoint and verifies the configured model" do
+    model_info = Struct.new(:id).new("claude-sonnet-4-6")
+    models = mock("models")
+    models.expects(:retrieve).with("claude-sonnet-4-6").returns(model_info)
+    client = mock("anthropic_client")
+    client.expects(:models).returns(models)
+    @probe.stubs(:anthropic_client).returns(client)
+
+    result = @probe.llm(
+      provider: :anthropic,
+      endpoint: "https://api.anthropic.com",
+      access_token: "anthropic-token",
+      model: "claude-sonnet-4-6"
+    )
+
+    assert result.passing?
+  end
+
+  test "failed LLM probe writes a system-wide debug entry and Rails log without secrets" do
+    models = stub(list: { "data" => [ { "id" => "another-model" } ] })
+    @probe.stubs(:openai_client).returns(stub(models: models))
+    endpoint = URI::HTTP.build(
+      userinfo: "operator:uri-secret",
+      host: "ollama",
+      port: 11_434,
+      path: "/v1",
+      query: "api_key=query-secret"
+    ).to_s
+    Rails.logger.expects(:error).with do |message|
+      message.include?("AI health llm liveness probe failed") &&
+        !message.include?("secret-token") &&
+        !message.include?("uri-secret") &&
+        !message.include?("query-secret")
+    end
+
+    assert_difference -> { DebugLogEntry.where(category: "ai_health").count }, 1 do
+      @result = @probe.llm(
+        provider: :openai,
+        endpoint: endpoint,
+        access_token: "secret-token",
+        model: "missing-model"
+      )
+    end
+
+    assert @result.failing?
+    assert_equal :model_not_available, @result.failure_code
+
+    entry = DebugLogEntry.where(category: "ai_health")
+                         .where("metadata ->> 'model' = ?", "missing-model")
+                         .order(:id)
+                         .last
+    assert_equal "error", entry.level
+    assert_equal "AiHealth::Probe", entry.source
+    assert_equal "openai", entry.provider_key
+    assert_nil entry.family
+    assert_nil entry.account
+    assert_equal "http://ollama:11434/v1", entry.metadata.fetch("endpoint")
+    assert_equal "model_not_available", entry.metadata.fetch("failure_code")
+    assert_no_match(/secret-token|uri-secret|query-secret/, entry.metadata.to_json)
+  end
+
+  test "probe results are cached to avoid repeated requests and failure logs" do
+    models = mock("models")
+    models.expects(:list).once.returns({ "data" => [ { "id" => "gpt-4.1" } ] })
+    client = stub(models: models)
+    @probe.stubs(:openai_client).returns(client)
+
+    2.times do
+      result = @probe.llm(
+        provider: :openai,
+        endpoint: "https://api.openai.com/v1",
+        access_token: "token",
+        model: "gpt-4.1"
+      )
+      assert result.passing?
+    end
+  end
+
+  test "forced probe bypasses the cached result" do
+    models = mock("models")
+    models.expects(:list).twice.returns({ "data" => [ { "id" => "gpt-4.1" } ] })
+    client = stub(models: models)
+    @probe.stubs(:openai_client).returns(client)
+
+    arguments = {
+      provider: :openai,
+      endpoint: "https://api.openai.com/v1",
+      access_token: "token",
+      model: "gpt-4.1"
+    }
+    @probe.llm(**arguments)
+
+    forced_probe = AiHealth::Probe.new(cache: @cache, force: true)
+    forced_probe.stubs(:openai_client).returns(client)
+    assert forced_probe.llm(**arguments).passing?
+  end
+
+  test "hosted vector-store probe calls the non-destructive list endpoint" do
+    request = stub_request(:get, "https://api.openai.example.test/v1/vector_stores")
+              .with(query: { limit: 1 }, headers: { "Authorization" => "Bearer token" })
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [] }.to_json
+              )
+
+    result = @probe.openai_vector_store(
+      endpoint: "https://api.openai.example.test/v1",
+      access_token: "token"
+    )
+
+    assert result.passing?
+    assert_requested request
+  end
+
+  test "pgvector probe verifies the extension, table, and a real query" do
+    connection = mock("connection")
+    connection.expects(:extension_enabled?).with("vector").returns(true)
+    connection.expects(:table_exists?).with("vector_store_chunks").returns(true)
+    connection.expects(:quote_table_name).with("vector_store_chunks").returns(%("vector_store_chunks"))
+    connection.expects(:select_value).with('SELECT 1 FROM "vector_store_chunks" LIMIT 1').returns(nil)
+
+    assert @probe.pgvector(connection: connection).passing?
+  end
+
+  test "embedding probe sends a small request and verifies dimensions" do
+    request = stub_request(:post, "http://ollama.example.test:11434/v1/embeddings")
+              .with(
+                body: {
+                  model: "nomic-embed-text",
+                  input: AiHealth::Probe::EMBEDDING_TEST_INPUT
+                }
+              )
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [ { embedding: [ 0.1, 0.2, 0.3 ] } ] }.to_json
+              )
+
+    result = @probe.embedding(
+      endpoint: "http://ollama.example.test:11434/v1",
+      access_token: nil,
+      model: "nomic-embed-text",
+      dimensions: 3
+    )
+
+    assert result.passing?
+    assert_requested request
+  end
+
+  test "embedding probe fails when returned dimensions do not match configuration" do
+    response = Struct.new(:body).new({ "data" => [ { "embedding" => [ 0.1, 0.2 ] } ] })
+    client = stub
+    client.stubs(:post).yields(Struct.new(:body).new).returns(response)
+    @probe.stubs(:embedding_client).returns(client)
+    Rails.logger.stubs(:error)
+    DebugLogEntry.stubs(:capture)
+
+    result = @probe.embedding(
+      endpoint: "http://ollama:11434/v1",
+      access_token: nil,
+      model: "nomic-embed-text",
+      dimensions: 3
+    )
+
+    assert result.failing?
+    assert_equal :dimensions_mismatch, result.failure_code
+  end
+end

test/models/ai_health_test.rb

@@ -0,0 +1,58 @@
+require "test_helper"
+
+class AiHealthTest < ActiveSupport::TestCase
+  AI_ENVIRONMENT = %w[
+    OPENAI_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_MODEL
+    ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY VECTOR_STORE_PROVIDER
+  ].index_with(nil).freeze
+
+  setup do
+    Setting.stubs(:llm_provider).returns("openai")
+    Setting.stubs(:openai_access_token).returns(nil)
+    Setting.stubs(:openai_uri_base).returns(nil)
+    Setting.stubs(:openai_model).returns(nil)
+    Setting.stubs(:anthropic_access_token).returns(nil)
+  end
+
+  test "native OpenAI remains distinct from OpenAI-compatible providers" do
+    with_openai_endpoint(nil) do |health|
+      assert_equal :openai, health.selected_llm_provider
+      assert_equal :openai, health.effective_llm_provider
+      assert_not health.openai_compatible_endpoint?
+    end
+  end
+
+  test "identifies known OpenAI-compatible providers from their endpoints" do
+    {
+      "http://ollama:11434/v1" => :ollama,
+      "http://127.0.0.1:11434/v1" => :ollama,
+      "https://openrouter.ai/api/v1" => :openrouter,
+      "https://api.together.ai/v1" => :together,
+      "https://api.kilo.ai/api/gateway" => :kilo,
+      "https://api.cloudflare.com/client/v4/accounts/account-id/ai/v1" => :cloudflare,
+      "https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat" => :cloudflare,
+      "https://models.example.test/v1" => :custom_openai_compatible
+    }.each do |endpoint, provider|
+      with_openai_endpoint(endpoint) do |health|
+        assert_equal :openai_compatible, health.selected_llm_provider, endpoint
+        assert_equal provider, health.effective_llm_provider, endpoint
+        assert health.openai_compatible_endpoint?, endpoint
+        assert_not health.llm_fallback?, endpoint
+      end
+    end
+  end
+
+  private
+    def with_openai_endpoint(endpoint)
+      ClimateControl.modify(
+        AI_ENVIRONMENT.merge(
+          "OPENAI_ACCESS_TOKEN" => "test-token",
+          "OPENAI_URI_BASE" => endpoint,
+          "OPENAI_MODEL" => endpoint.present? ? "test-model" : nil,
+          "VECTOR_STORE_PROVIDER" => "qdrant"
+        )
+      ) do
+        yield AiHealth.new(run_probes: false)
+      end
+    end
+end

test/models/vector_store/embeddable_test.rb

@@ -4,7 +4,9 @@ class VectorStore::EmbeddableTest < ActiveSupport::TestCase
   class EmbeddableHost
     include VectorStore::Embeddable
     # Expose private methods for testing
-    public :extract_text, :chunk_text, :embed, :embed_batch
+    public :extract_text, :chunk_text, :embed, :embed_batch,
+           :embedding_model, :embedding_dimensions, :embedding_uri_base,
+           :embedding_access_token
   end
 
   setup do
@@ -138,6 +140,18 @@ class VectorStore::EmbeddableTest < ActiveSupport::TestCase
     assert_raises(VectorStore::Error) { @host.embed("test text") }
   end
 
+  test "embedding configuration delegates to the shared runtime configuration" do
+    VectorStore.expects(:embedding_model).returns("model")
+    VectorStore.expects(:embedding_dimensions).returns(3)
+    VectorStore.expects(:embedding_uri_base).returns("https://embeddings.example.test/v1")
+    VectorStore.expects(:embedding_access_token).returns("token")
+
+    assert_equal "model", @host.embedding_model
+    assert_equal 3, @host.embedding_dimensions
+    assert_equal "https://embeddings.example.test/v1", @host.embedding_uri_base
+    assert_equal "token", @host.embedding_access_token
+  end
+
   # --- embed_batch ---
 
   test "embed_batch processes texts and returns ordered vectors" do

test/models/vector_store_test.rb

@@ -0,0 +1,34 @@
+require "test_helper"
+
+class VectorStoreTest < ActiveSupport::TestCase
+  EMBEDDING_ENVIRONMENT = %w[
+    EMBEDDING_MODEL EMBEDDING_DIMENSIONS EMBEDDING_URI_BASE
+    EMBEDDING_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_ACCESS_TOKEN
+  ].index_with(nil).freeze
+
+  test "embedding defaults use a matching model and vector width" do
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT) do
+      assert_equal "mxbai-embed-large", VectorStore.embedding_model
+      assert_equal 1024, VectorStore.embedding_dimensions
+    end
+  end
+
+  test "embedding credentials match runtime environment precedence" do
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT.merge(
+      "EMBEDDING_ACCESS_TOKEN" => "embedding-token",
+      "OPENAI_ACCESS_TOKEN" => "openai-token"
+    )) do
+      assert_equal "embedding-token", VectorStore.embedding_access_token
+    end
+
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT.merge(
+      "OPENAI_ACCESS_TOKEN" => "openai-token"
+    )) do
+      assert_equal "openai-token", VectorStore.embedding_access_token
+    end
+
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT) do
+      assert_nil VectorStore.embedding_access_token
+    end
+  end
+end