Add live AI checks to system health (#3155)

* Add live AI checks to system health

Give super admins a dedicated AI status view with bounded liveness probes for LLMs, vector stores, pgvector, and embedding endpoints. Record sanitized failures in both the system debug log and Rails logger, and document the recommended local configuration.\n\nCloses #3145

* Fix AI health CI checks

* Address AI health review feedback

* Correct Ollama model preload guidance

* Distinguish OpenAI-compatible providers

* Make Ollama startup readiness explicit

* Recognize Cloudflare AI endpoints

test/components/DS/tabs_test.rb

@@ -0,0 +1,21 @@
+require "test_helper"
+
+class DS::TabsTest < ViewComponent::TestCase
+  test "can navigate to the selected tab for server-backed content" do
+    render_inline(DS::Tabs.new(
+      active_tab: "background_jobs",
+      url_param_key: "tab",
+      navigate_on_change: true
+    )) do |tabs|
+      tabs.with_nav do |nav|
+        nav.with_btn(id: "background_jobs", label: "Background jobs")
+        nav.with_btn(id: "ai", label: "AI status")
+      end
+      tabs.with_panel(tab_id: "background_jobs") { "Jobs" }
+      tabs.with_panel(tab_id: "ai") { "AI" }
+    end
+
+    assert_selector "[data-ds--tabs-url-param-key-value='tab']"
+    assert_selector "[data-ds--tabs-navigate-on-change-value='true']"
+  end
+end

test/controllers/admin/system_health_controller_test.rb

@@ -1,6 +1,29 @@
 require "test_helper"
 
 class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest
+  AI_ENVIRONMENT = %w[
+    OPENAI_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_MODEL OPENAI_REQUEST_TIMEOUT
+    OPENAI_SUPPORTS_PDF_PROCESSING ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY
+    ANTHROPIC_BASE_URL ANTHROPIC_MODEL ANTHROPIC_REQUEST_TIMEOUT
+    VECTOR_STORE_PROVIDER EMBEDDING_URI_BASE EMBEDDING_MODEL
+    EMBEDDING_DIMENSIONS EMBEDDING_ACCESS_TOKEN QDRANT_URL QDRANT_API_KEY
+    AI_HEALTH_PROBE_TIMEOUT AI_HEALTH_PROBE_CACHE_TTL
+  ].index_with(nil).freeze
+
+  setup do
+    Setting.stubs(:llm_provider).returns("openai")
+    Setting.stubs(:openai_access_token).returns(nil)
+    Setting.stubs(:openai_uri_base).returns(nil)
+    Setting.stubs(:openai_model).returns(nil)
+    Setting.stubs(:anthropic_access_token).returns(nil)
+    Setting.stubs(:anthropic_base_url).returns(nil)
+    Setting.stubs(:anthropic_model).returns(nil)
+    AiHealth::Probe.any_instance.stubs(:llm).returns(probe_result(:passing))
+    AiHealth::Probe.any_instance.stubs(:openai_vector_store).returns(probe_result(:passing))
+    AiHealth::Probe.any_instance.stubs(:pgvector).returns(probe_result(:passing))
+    AiHealth::Probe.any_instance.stubs(:embedding).returns(probe_result(:passing))
+  end
+
   test "super admin can view the system health page" do
     sign_in users(:sure_support_staff)
     SidekiqHealth.any_instance.stubs(:healthy?).returns(true)
@@ -18,6 +41,8 @@ class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest
     assert_response :success
     assert_match(/Sidekiq status/, response.body)
     assert_match(/Healthy/, response.body)
+    assert_select "button[role='tab']", text: "AI status"
+    assert_select "[data-ds--tabs-navigate-on-change-value='true']"
   end
 
   test "renders degraded state with reason when Sidekiq is unhealthy" do
@@ -53,4 +78,188 @@ class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest
 
     assert_redirected_to new_session_path
   end
+
+  test "AI status reports the default OpenAI LLM and hosted vector store" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+
+    with_ai_environment("OPENAI_ACCESS_TOKEN" => "sk-secret-openai") do
+      get admin_system_health_url(tab: "ai")
+    end
+
+    assert_response :success
+    assert_select "button[role='tab'][aria-selected='true']", text: "AI status"
+    assert_match(/LLM and PDF processing/, response.body)
+    assert_select "[data-testid='selected-llm-provider']", text: "OpenAI"
+    assert_select "[data-testid='effective-llm-provider']", text: "OpenAI"
+    assert_match(/gpt-4\.1/, response.body)
+    assert_match(%r{https://api\.openai\.com/v1}, response.body)
+    assert_match(/OpenAI hosted vector store/, response.body)
+    assert_match(/Live check passed/, response.body)
+    assert_match(/Live checks passed/, response.body)
+    assert_match(/Supported/, response.body)
+    assert_no_match(/sk-secret-openai/, response.body)
+  end
+
+  test "background jobs tab does not run AI probes" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+    AiHealth::Probe.any_instance.expects(:llm).never
+    AiHealth::Probe.any_instance.expects(:openai_vector_store).never
+
+    with_ai_environment("OPENAI_ACCESS_TOKEN" => "sk-secret-openai") do
+      get admin_system_health_url
+    end
+
+    assert_response :success
+    assert_match(/Not checked/, response.body)
+    assert_no_match(/sk-secret-openai/, response.body)
+  end
+
+  test "AI status warns when a custom OpenAI endpoint is paired with the hosted vector store" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+
+    with_ai_environment(
+      "OPENAI_ACCESS_TOKEN" => "local-token",
+      "OPENAI_URI_BASE" => credentialed_url(
+        scheme: "http",
+        host: "ollama",
+        port: 11_434,
+        path: "/v1",
+        user: "operator",
+        password: "uri-secret",
+        query: "api_key=query-secret"
+      ),
+      "OPENAI_MODEL" => "qwen3:8b"
+    ) do
+      AiHealth::Probe.any_instance.stubs(:openai_vector_store).returns(
+        probe_result(:failing, failure_code: :request_failed, http_status: 404)
+      )
+      get admin_system_health_url(tab: "ai")
+    end
+
+    assert_response :success
+    assert_select "[data-testid='selected-llm-provider']", text: "OpenAI-compatible"
+    assert_select "[data-testid='effective-llm-provider']", text: "Ollama"
+    assert_match(/OpenAI-compatible API credentials/, response.body)
+    assert_match(%r{http://ollama:11434/v1}, response.body)
+    assert_match(%r{did not pass the /v1/vector_stores liveness check}, response.body)
+    assert_match(/use pgvector with a separate embeddings endpoint/, response.body)
+    assert_match(/Live check failed/, response.body)
+    assert_no_match(/local-token|uri-secret|query-secret/, response.body)
+  end
+
+  test "AI status reports Anthropic with an available pgvector store" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+    Setting.stubs(:llm_provider).returns("anthropic")
+
+    connection = stub("connection")
+    connection.stubs(:table_exists?).with(VectorStore::Pgvector::TABLE_NAME).returns(true)
+    connection.stubs(:extension_enabled?).with("vector").returns(true)
+    ActiveRecord::Base.stubs(:connection).returns(connection)
+    VectorStore.expects(:embedding_access_token).returns("runtime-embedding-token")
+    AiHealth::Probe.any_instance.expects(:embedding).with(
+      endpoint: "http://ollama:11434/v1",
+      access_token: "runtime-embedding-token",
+      model: "mxbai-embed-large",
+      dimensions: 1024
+    ).returns(probe_result(:passing))
+
+    with_ai_environment(
+      "ANTHROPIC_ACCESS_TOKEN" => "anthropic-secret",
+      "ANTHROPIC_MODEL" => "claude-sonnet-4-6",
+      "EMBEDDING_URI_BASE" => "http://ollama:11434/v1",
+      "EMBEDDING_MODEL" => "mxbai-embed-large",
+      "EMBEDDING_DIMENSIONS" => "1024"
+    ) do
+      get admin_system_health_url(tab: "ai")
+    end
+
+    assert_response :success
+    assert_match(/Anthropic/, response.body)
+    assert_match(/pgvector/, response.body)
+    assert_match(/PostgreSQL vector extension/, response.body)
+    assert_match(/mxbai-embed-large/, response.body)
+    assert_match(%r{http://ollama:11434/v1}, response.body)
+    assert_match(/Live checks passed/, response.body)
+    assert_no_match(/anthropic-secret/, response.body)
+  end
+
+  test "AI status explains when no vector store is configured" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+
+    with_ai_environment do
+      get admin_system_health_url(tab: "ai")
+    end
+
+    assert_response :success
+    assert_match(/No vector store is configured/, response.body)
+    assert_match(/Uploaded documents cannot be indexed or searched/, response.body)
+  end
+
+  test "AI status marks Qdrant as scaffolded and redacts its URL" do
+    sign_in users(:sure_support_staff)
+    stub_healthy_sidekiq
+
+    with_ai_environment(
+      "VECTOR_STORE_PROVIDER" => "qdrant",
+      "QDRANT_URL" => credentialed_url(
+        scheme: "https",
+        host: "qdrant.example.test",
+        port: 6333,
+        user: "admin",
+        password: "qdrant-secret",
+        query: "api_key=query-secret"
+      ),
+      "QDRANT_API_KEY" => "header-secret"
+    ) do
+      get admin_system_health_url(tab: "ai")
+    end
+
+    assert_response :success
+    assert_match(/Qdrant support is not implemented yet/, response.body)
+    assert_match(/Scaffolded/, response.body)
+    assert_match(%r{https://qdrant\.example\.test:6333}, response.body)
+    assert_no_match(/qdrant-secret|query-secret|header-secret/, response.body)
+  end
+
+  private
+    def credentialed_url(scheme:, host:, port:, user:, password:, path: nil, query: nil)
+      URI::Generic.build(
+        scheme: scheme,
+        userinfo: "#{user}:#{password}",
+        host: host,
+        port: port,
+        path: path,
+        query: query
+      ).to_s
+    end
+
+    def probe_result(status, failure_code: nil, http_status: nil)
+      AiHealth::Probe::Result.new(
+        status: status,
+        checked_at: status.in?([ :passing, :failing ]) ? Time.current : nil,
+        failure_code: failure_code,
+        http_status: http_status
+      )
+    end
+
+    def with_ai_environment(overrides = {}, &block)
+      ClimateControl.modify(AI_ENVIRONMENT.merge(overrides), &block)
+    end
+
+    def stub_healthy_sidekiq
+      SidekiqHealth.any_instance.stubs(:healthy?).returns(true)
+      SidekiqHealth.any_instance.stubs(:processes_count).returns(1)
+      SidekiqHealth.any_instance.stubs(:last_heartbeat_at).returns(Time.current)
+      SidekiqHealth.any_instance.stubs(:max_queue_latency).returns(0.0)
+      SidekiqHealth.any_instance.stubs(:enqueued_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:retry_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:failed_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:processed_count).returns(42)
+      SidekiqHealth.any_instance.stubs(:queue_breakdown).returns([ [ "default", 0, 0.0 ] ])
+    end
 end

test/models/ai_health/probe_test.rb

@@ -0,0 +1,198 @@
+require "test_helper"
+
+class AiHealth::ProbeTest < ActiveSupport::TestCase
+  setup do
+    @cache = ActiveSupport::Cache::MemoryStore.new
+    @probe = AiHealth::Probe.new(cache: @cache)
+  end
+
+  test "OpenAI LLM probe calls the models endpoint and verifies the configured model" do
+    request = stub_request(:get, "http://ollama.example.test:11434/v1/models")
+              .with(headers: { "Authorization" => "Bearer local-token" })
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [ { id: "qwen3:8b" } ] }.to_json
+              )
+
+    result = @probe.llm(
+      provider: :openai,
+      endpoint: "http://ollama.example.test:11434/v1",
+      access_token: "local-token",
+      model: "qwen3:8b"
+    )
+
+    assert result.passing?
+    assert result.checked_at
+    assert_requested request
+  end
+
+  test "Anthropic LLM probe calls the models endpoint and verifies the configured model" do
+    model_info = Struct.new(:id).new("claude-sonnet-4-6")
+    models = mock("models")
+    models.expects(:retrieve).with("claude-sonnet-4-6").returns(model_info)
+    client = mock("anthropic_client")
+    client.expects(:models).returns(models)
+    @probe.stubs(:anthropic_client).returns(client)
+
+    result = @probe.llm(
+      provider: :anthropic,
+      endpoint: "https://api.anthropic.com",
+      access_token: "anthropic-token",
+      model: "claude-sonnet-4-6"
+    )
+
+    assert result.passing?
+  end
+
+  test "failed LLM probe writes a system-wide debug entry and Rails log without secrets" do
+    models = stub(list: { "data" => [ { "id" => "another-model" } ] })
+    @probe.stubs(:openai_client).returns(stub(models: models))
+    endpoint = URI::HTTP.build(
+      userinfo: "operator:uri-secret",
+      host: "ollama",
+      port: 11_434,
+      path: "/v1",
+      query: "api_key=query-secret"
+    ).to_s
+    Rails.logger.expects(:error).with do |message|
+      message.include?("AI health llm liveness probe failed") &&
+        !message.include?("secret-token") &&
+        !message.include?("uri-secret") &&
+        !message.include?("query-secret")
+    end
+
+    assert_difference -> { DebugLogEntry.where(category: "ai_health").count }, 1 do
+      @result = @probe.llm(
+        provider: :openai,
+        endpoint: endpoint,
+        access_token: "secret-token",
+        model: "missing-model"
+      )
+    end
+
+    assert @result.failing?
+    assert_equal :model_not_available, @result.failure_code
+
+    entry = DebugLogEntry.where(category: "ai_health")
+                         .where("metadata ->> 'model' = ?", "missing-model")
+                         .order(:id)
+                         .last
+    assert_equal "error", entry.level
+    assert_equal "AiHealth::Probe", entry.source
+    assert_equal "openai", entry.provider_key
+    assert_nil entry.family
+    assert_nil entry.account
+    assert_equal "http://ollama:11434/v1", entry.metadata.fetch("endpoint")
+    assert_equal "model_not_available", entry.metadata.fetch("failure_code")
+    assert_no_match(/secret-token|uri-secret|query-secret/, entry.metadata.to_json)
+  end
+
+  test "probe results are cached to avoid repeated requests and failure logs" do
+    models = mock("models")
+    models.expects(:list).once.returns({ "data" => [ { "id" => "gpt-4.1" } ] })
+    client = stub(models: models)
+    @probe.stubs(:openai_client).returns(client)
+
+    2.times do
+      result = @probe.llm(
+        provider: :openai,
+        endpoint: "https://api.openai.com/v1",
+        access_token: "token",
+        model: "gpt-4.1"
+      )
+      assert result.passing?
+    end
+  end
+
+  test "forced probe bypasses the cached result" do
+    models = mock("models")
+    models.expects(:list).twice.returns({ "data" => [ { "id" => "gpt-4.1" } ] })
+    client = stub(models: models)
+    @probe.stubs(:openai_client).returns(client)
+
+    arguments = {
+      provider: :openai,
+      endpoint: "https://api.openai.com/v1",
+      access_token: "token",
+      model: "gpt-4.1"
+    }
+    @probe.llm(**arguments)
+
+    forced_probe = AiHealth::Probe.new(cache: @cache, force: true)
+    forced_probe.stubs(:openai_client).returns(client)
+    assert forced_probe.llm(**arguments).passing?
+  end
+
+  test "hosted vector-store probe calls the non-destructive list endpoint" do
+    request = stub_request(:get, "https://api.openai.example.test/v1/vector_stores")
+              .with(query: { limit: 1 }, headers: { "Authorization" => "Bearer token" })
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [] }.to_json
+              )
+
+    result = @probe.openai_vector_store(
+      endpoint: "https://api.openai.example.test/v1",
+      access_token: "token"
+    )
+
+    assert result.passing?
+    assert_requested request
+  end
+
+  test "pgvector probe verifies the extension, table, and a real query" do
+    connection = mock("connection")
+    connection.expects(:extension_enabled?).with("vector").returns(true)
+    connection.expects(:table_exists?).with("vector_store_chunks").returns(true)
+    connection.expects(:quote_table_name).with("vector_store_chunks").returns(%("vector_store_chunks"))
+    connection.expects(:select_value).with('SELECT 1 FROM "vector_store_chunks" LIMIT 1').returns(nil)
+
+    assert @probe.pgvector(connection: connection).passing?
+  end
+
+  test "embedding probe sends a small request and verifies dimensions" do
+    request = stub_request(:post, "http://ollama.example.test:11434/v1/embeddings")
+              .with(
+                body: {
+                  model: "nomic-embed-text",
+                  input: AiHealth::Probe::EMBEDDING_TEST_INPUT
+                }
+              )
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { data: [ { embedding: [ 0.1, 0.2, 0.3 ] } ] }.to_json
+              )
+
+    result = @probe.embedding(
+      endpoint: "http://ollama.example.test:11434/v1",
+      access_token: nil,
+      model: "nomic-embed-text",
+      dimensions: 3
+    )
+
+    assert result.passing?
+    assert_requested request
+  end
+
+  test "embedding probe fails when returned dimensions do not match configuration" do
+    response = Struct.new(:body).new({ "data" => [ { "embedding" => [ 0.1, 0.2 ] } ] })
+    client = stub
+    client.stubs(:post).yields(Struct.new(:body).new).returns(response)
+    @probe.stubs(:embedding_client).returns(client)
+    Rails.logger.stubs(:error)
+    DebugLogEntry.stubs(:capture)
+
+    result = @probe.embedding(
+      endpoint: "http://ollama:11434/v1",
+      access_token: nil,
+      model: "nomic-embed-text",
+      dimensions: 3
+    )
+
+    assert result.failing?
+    assert_equal :dimensions_mismatch, result.failure_code
+  end
+end

test/models/ai_health_test.rb

@@ -0,0 +1,58 @@
+require "test_helper"
+
+class AiHealthTest < ActiveSupport::TestCase
+  AI_ENVIRONMENT = %w[
+    OPENAI_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_MODEL
+    ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY VECTOR_STORE_PROVIDER
+  ].index_with(nil).freeze
+
+  setup do
+    Setting.stubs(:llm_provider).returns("openai")
+    Setting.stubs(:openai_access_token).returns(nil)
+    Setting.stubs(:openai_uri_base).returns(nil)
+    Setting.stubs(:openai_model).returns(nil)
+    Setting.stubs(:anthropic_access_token).returns(nil)
+  end
+
+  test "native OpenAI remains distinct from OpenAI-compatible providers" do
+    with_openai_endpoint(nil) do |health|
+      assert_equal :openai, health.selected_llm_provider
+      assert_equal :openai, health.effective_llm_provider
+      assert_not health.openai_compatible_endpoint?
+    end
+  end
+
+  test "identifies known OpenAI-compatible providers from their endpoints" do
+    {
+      "http://ollama:11434/v1" => :ollama,
+      "http://127.0.0.1:11434/v1" => :ollama,
+      "https://openrouter.ai/api/v1" => :openrouter,
+      "https://api.together.ai/v1" => :together,
+      "https://api.kilo.ai/api/gateway" => :kilo,
+      "https://api.cloudflare.com/client/v4/accounts/account-id/ai/v1" => :cloudflare,
+      "https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat" => :cloudflare,
+      "https://models.example.test/v1" => :custom_openai_compatible
+    }.each do |endpoint, provider|
+      with_openai_endpoint(endpoint) do |health|
+        assert_equal :openai_compatible, health.selected_llm_provider, endpoint
+        assert_equal provider, health.effective_llm_provider, endpoint
+        assert health.openai_compatible_endpoint?, endpoint
+        assert_not health.llm_fallback?, endpoint
+      end
+    end
+  end
+
+  private
+    def with_openai_endpoint(endpoint)
+      ClimateControl.modify(
+        AI_ENVIRONMENT.merge(
+          "OPENAI_ACCESS_TOKEN" => "test-token",
+          "OPENAI_URI_BASE" => endpoint,
+          "OPENAI_MODEL" => endpoint.present? ? "test-model" : nil,
+          "VECTOR_STORE_PROVIDER" => "qdrant"
+        )
+      ) do
+        yield AiHealth.new(run_probes: false)
+      end
+    end
+end

test/models/vector_store/embeddable_test.rb

@@ -4,7 +4,9 @@ class VectorStore::EmbeddableTest < ActiveSupport::TestCase
   class EmbeddableHost
     include VectorStore::Embeddable
     # Expose private methods for testing
-    public :extract_text, :chunk_text, :embed, :embed_batch
+    public :extract_text, :chunk_text, :embed, :embed_batch,
+           :embedding_model, :embedding_dimensions, :embedding_uri_base,
+           :embedding_access_token
   end
 
   setup do
@@ -138,6 +140,18 @@ class VectorStore::EmbeddableTest < ActiveSupport::TestCase
     assert_raises(VectorStore::Error) { @host.embed("test text") }
   end
 
+  test "embedding configuration delegates to the shared runtime configuration" do
+    VectorStore.expects(:embedding_model).returns("model")
+    VectorStore.expects(:embedding_dimensions).returns(3)
+    VectorStore.expects(:embedding_uri_base).returns("https://embeddings.example.test/v1")
+    VectorStore.expects(:embedding_access_token).returns("token")
+
+    assert_equal "model", @host.embedding_model
+    assert_equal 3, @host.embedding_dimensions
+    assert_equal "https://embeddings.example.test/v1", @host.embedding_uri_base
+    assert_equal "token", @host.embedding_access_token
+  end
+
   # --- embed_batch ---
 
   test "embed_batch processes texts and returns ordered vectors" do

test/models/vector_store_test.rb

@@ -0,0 +1,34 @@
+require "test_helper"
+
+class VectorStoreTest < ActiveSupport::TestCase
+  EMBEDDING_ENVIRONMENT = %w[
+    EMBEDDING_MODEL EMBEDDING_DIMENSIONS EMBEDDING_URI_BASE
+    EMBEDDING_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_ACCESS_TOKEN
+  ].index_with(nil).freeze
+
+  test "embedding defaults use a matching model and vector width" do
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT) do
+      assert_equal "mxbai-embed-large", VectorStore.embedding_model
+      assert_equal 1024, VectorStore.embedding_dimensions
+    end
+  end
+
+  test "embedding credentials match runtime environment precedence" do
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT.merge(
+      "EMBEDDING_ACCESS_TOKEN" => "embedding-token",
+      "OPENAI_ACCESS_TOKEN" => "openai-token"
+    )) do
+      assert_equal "embedding-token", VectorStore.embedding_access_token
+    end
+
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT.merge(
+      "OPENAI_ACCESS_TOKEN" => "openai-token"
+    )) do
+      assert_equal "openai-token", VectorStore.embedding_access_token
+    end
+
+    ClimateControl.modify(EMBEDDING_ENVIRONMENT) do
+      assert_nil VectorStore.embedding_access_token
+    end
+  end
+end

test/system/admin/system_health_test.rb

@@ -0,0 +1,54 @@
+require "application_system_test_case"
+
+class Admin::SystemHealthTest < ApplicationSystemTestCase
+  setup do
+    sign_in users(:sure_support_staff)
+    Setting.stubs(:llm_provider).returns("openai")
+    Setting.stubs(:openai_access_token).returns(nil)
+    Setting.stubs(:openai_uri_base).returns(nil)
+    Setting.stubs(:openai_model).returns(nil)
+    stub_healthy_sidekiq
+    AiHealth::Probe.any_instance.stubs(:llm).returns(probe_result(:passing))
+    AiHealth::Probe.any_instance.stubs(:openai_vector_store).returns(probe_result(:passing))
+  end
+
+  test "selecting AI status runs live probes" do
+    ClimateControl.modify(
+      "OPENAI_ACCESS_TOKEN" => "test-token",
+      "OPENAI_URI_BASE" => nil,
+      "OPENAI_MODEL" => nil,
+      "VECTOR_STORE_PROVIDER" => nil
+    ) do
+      visit admin_system_health_path
+
+      click_button "AI status"
+
+      assert_current_path admin_system_health_path(tab: "ai")
+      assert_selector "button[role='tab'][aria-selected='true']", text: "AI status"
+      assert_text "Live check passed"
+      assert_text "Live checks passed"
+    end
+  end
+
+  private
+    def probe_result(status)
+      AiHealth::Probe::Result.new(
+        status: status,
+        checked_at: Time.current,
+        failure_code: nil,
+        http_status: nil
+      )
+    end
+
+    def stub_healthy_sidekiq
+      SidekiqHealth.any_instance.stubs(:healthy?).returns(true)
+      SidekiqHealth.any_instance.stubs(:processes_count).returns(1)
+      SidekiqHealth.any_instance.stubs(:last_heartbeat_at).returns(Time.current)
+      SidekiqHealth.any_instance.stubs(:max_queue_latency).returns(0.0)
+      SidekiqHealth.any_instance.stubs(:enqueued_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:retry_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:failed_count).returns(0)
+      SidekiqHealth.any_instance.stubs(:processed_count).returns(42)
+      SidekiqHealth.any_instance.stubs(:queue_breakdown).returns([ [ "default", 0, 0.0 ] ])
+    end
+end

test/system/settings_test.rb

@@ -52,6 +52,8 @@ class SettingsTest < ApplicationSystemTestCase
   test "can update self hosting settings" do
     sign_in users(:sure_support_staff)
     Rails.application.config.app_mode.stubs(:self_hosted?).returns(true)
+    Provider::Registry.stubs(:get_provider).with(:openai).returns(nil)
+    Provider::Registry.stubs(:get_provider).with(:anthropic).returns(nil)
     Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(nil)
     Provider::Registry.stubs(:get_provider).with(:yahoo_finance).returns(nil)
     Provider::Registry.stubs(:get_provider).with(:rentcast).returns(nil)