diff --git a/app/models/ai_health.rb b/app/models/ai_health.rb index a143df1c2..af8737adf 100644 --- a/app/models/ai_health.rb +++ b/app/models/ai_health.rb @@ -158,7 +158,8 @@ class AiHealth provider: @effective_llm_protocol, endpoint: @llm_raw_endpoint, access_token: @llm_access_token, - model: llm_model + model: llm_model, + openai_compatible: @effective_llm_protocol == :openai && openai_compatible_endpoint? ) end diff --git a/app/models/ai_health/probe.rb b/app/models/ai_health/probe.rb index ae3178a68..b4bb5c2a1 100644 --- a/app/models/ai_health/probe.rb +++ b/app/models/ai_health/probe.rb @@ -12,6 +12,7 @@ class AiHealth DEFAULT_CACHE_TTL = 60.seconds DEFAULT_TIMEOUT = 5 EMBEDDING_TEST_INPUT = "Sure AI health check" + CHAT_TEST_INPUT = "Reply with OK." Result = Data.define(:status, :checked_at, :failure_code, :http_status) do def passing? @@ -45,18 +46,25 @@ class AiHealth @cache = cache end - def llm(provider:, endpoint:, access_token:, model:) + def llm(provider:, endpoint:, access_token:, model:, openai_compatible: false) + verification = openai_compatible ? :chat_completion : :models + run( component: "llm", provider_key: provider, endpoint: endpoint, model: model, - credential: access_token + credential: access_token, + verification: verification ) do model_available = case provider when :openai - response = openai_client(access_token:, endpoint:).models.list - openai_model_ids(response).include?(model) + if openai_compatible + openai_chat_completion_available?(access_token:, endpoint:, model:) + else + response = openai_client(access_token:, endpoint:).models.list + openai_model_ids(response).include?(model) + end when :anthropic model_info = anthropic_client(access_token:, endpoint:).models.retrieve(model) model_info.respond_to?(:id) && model_info.id.present? @@ -112,8 +120,8 @@ class AiHealth private attr_reader :cache, :force - def run(component:, provider_key:, endpoint: nil, model: nil, credential: nil, dimensions: nil) - key = cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:) + def run(component:, provider_key:, endpoint: nil, model: nil, credential: nil, dimensions: nil, verification: nil) + key = cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:, verification:) cache.delete(key) if force result = nil @@ -168,9 +176,20 @@ class AiHealth response["data"].filter_map { |item| item["id"] || item[:id] } end - def cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:) + def openai_chat_completion_available?(access_token:, endpoint:, model:) + response = openai_client(access_token:, endpoint:).chat( + parameters: { + model: model, + messages: [ { role: "user", content: CHAT_TEST_INPUT } ] + } + ) + + response.is_a?(Hash) && response["choices"].is_a?(Array) && response["choices"].any? + end + + def cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:, verification:) fingerprint = Digest::SHA256.hexdigest( - [ component, provider_key, endpoint, model, credential, dimensions ].join("\0") + [ component, provider_key, endpoint, model, credential, dimensions, verification ].join("\0") ) "#{CACHE_NAMESPACE}/#{fingerprint}" end diff --git a/test/models/ai_health/probe_test.rb b/test/models/ai_health/probe_test.rb index 6e84971c4..07fb85a2e 100644 --- a/test/models/ai_health/probe_test.rb +++ b/test/models/ai_health/probe_test.rb @@ -7,19 +7,48 @@ class AiHealth::ProbeTest < ActiveSupport::TestCase end test "OpenAI LLM probe calls the models endpoint and verifies the configured model" do - request = stub_request(:get, "http://ollama.example.test:11434/v1/models") + request = stub_request(:get, "https://api.openai.example.test/v1/models") .with(headers: { "Authorization" => "Bearer local-token" }) .to_return( status: 200, headers: { "Content-Type" => "application/json" }, - body: { data: [ { id: "qwen3:8b" } ] }.to_json + body: { data: [ { id: "gpt-4.1" } ] }.to_json ) result = @probe.llm( provider: :openai, - endpoint: "http://ollama.example.test:11434/v1", + endpoint: "https://api.openai.example.test/v1", access_token: "local-token", - model: "qwen3:8b" + model: "gpt-4.1" + ) + + assert result.passing? + assert result.checked_at + assert_requested request + end + + test "OpenAI-compatible LLM probe calls chat completions instead of the models endpoint" do + endpoint = "https://api.cloudflare.com/client/v4/accounts/account-id/ai/v1" + request = stub_request(:post, "#{endpoint}/chat/completions") + .with( + headers: { "Authorization" => "Bearer cf-token" }, + body: { + model: "@cf/zai-org/glm-5.2", + messages: [ { role: "user", content: AiHealth::Probe::CHAT_TEST_INPUT } ] + } + ) + .to_return( + status: 200, + headers: { "Content-Type" => "application/json" }, + body: { choices: [ { message: { content: "OK" } } ] }.to_json + ) + + result = @probe.llm( + provider: :openai, + endpoint: endpoint, + access_token: "cf-token", + model: "@cf/zai-org/glm-5.2", + openai_compatible: true ) assert result.passing? diff --git a/test/system/accounts_sync_ui_test.rb b/test/system/accounts_sync_ui_test.rb index 668549391..248abd4a9 100644 --- a/test/system/accounts_sync_ui_test.rb +++ b/test/system/accounts_sync_ui_test.rb @@ -3,6 +3,7 @@ require "application_system_test_case" class AccountsSyncUiTest < ApplicationSystemTestCase setup do @user = users(:family_admin) + Sync.for_family(@user.family).incomplete.destroy_all sign_in @user end