Fix AI health check for OpenAI-compatible endpoints (#3184)

* Fix AI health probe for OpenAI-compatible endpoints

* Stabilize accounts sync system test

* Address AI health probe review feedback

app/models/ai_health.rb

@@ -158,7 +158,8 @@ class AiHealth
           provider: @effective_llm_protocol,
           endpoint: @llm_raw_endpoint,
           access_token: @llm_access_token,
-          model: llm_model
+          model: llm_model,
+          openai_compatible: @effective_llm_protocol == :openai && openai_compatible_endpoint?
         )
       end
 

app/models/ai_health/probe.rb

@@ -12,6 +12,7 @@ class AiHealth
     DEFAULT_CACHE_TTL = 60.seconds
     DEFAULT_TIMEOUT = 5
     EMBEDDING_TEST_INPUT = "Sure AI health check"
+    CHAT_TEST_INPUT = "Reply with OK."
 
     Result = Data.define(:status, :checked_at, :failure_code, :http_status) do
       def passing?
@@ -45,18 +46,25 @@ class AiHealth
       @cache = cache
     end
 
-    def llm(provider:, endpoint:, access_token:, model:)
+    def llm(provider:, endpoint:, access_token:, model:, openai_compatible: false)
+      verification = openai_compatible ? :chat_completion : :models
+
       run(
         component: "llm",
         provider_key: provider,
         endpoint: endpoint,
         model: model,
-        credential: access_token
+        credential: access_token,
+        verification: verification
       ) do
         model_available = case provider
         when :openai
-          response = openai_client(access_token:, endpoint:).models.list
-          openai_model_ids(response).include?(model)
+          if openai_compatible
+            openai_chat_completion_available?(access_token:, endpoint:, model:)
+          else
+            response = openai_client(access_token:, endpoint:).models.list
+            openai_model_ids(response).include?(model)
+          end
         when :anthropic
           model_info = anthropic_client(access_token:, endpoint:).models.retrieve(model)
           model_info.respond_to?(:id) && model_info.id.present?
@@ -112,8 +120,8 @@ class AiHealth
     private
       attr_reader :cache, :force
 
-      def run(component:, provider_key:, endpoint: nil, model: nil, credential: nil, dimensions: nil)
-        key = cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:)
+      def run(component:, provider_key:, endpoint: nil, model: nil, credential: nil, dimensions: nil, verification: nil)
+        key = cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:, verification:)
         cache.delete(key) if force
 
         result = nil
@@ -168,9 +176,20 @@ class AiHealth
         response["data"].filter_map { |item| item["id"] || item[:id] }
       end
 
-      def cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:)
+      def openai_chat_completion_available?(access_token:, endpoint:, model:)
+        response = openai_client(access_token:, endpoint:).chat(
+          parameters: {
+            model: model,
+            messages: [ { role: "user", content: CHAT_TEST_INPUT } ]
+          }
+        )
+
+        response.is_a?(Hash) && response["choices"].is_a?(Array) && response["choices"].any?
+      end
+
+      def cache_key(component:, provider_key:, endpoint:, model:, credential:, dimensions:, verification:)
         fingerprint = Digest::SHA256.hexdigest(
-          [ component, provider_key, endpoint, model, credential, dimensions ].join("\0")
+          [ component, provider_key, endpoint, model, credential, dimensions, verification ].join("\0")
         )
         "#{CACHE_NAMESPACE}/#{fingerprint}"
       end

test/models/ai_health/probe_test.rb

@@ -7,19 +7,48 @@ class AiHealth::ProbeTest < ActiveSupport::TestCase
   end
 
   test "OpenAI LLM probe calls the models endpoint and verifies the configured model" do
-    request = stub_request(:get, "http://ollama.example.test:11434/v1/models")
+    request = stub_request(:get, "https://api.openai.example.test/v1/models")
               .with(headers: { "Authorization" => "Bearer local-token" })
               .to_return(
                 status: 200,
                 headers: { "Content-Type" => "application/json" },
-                body: { data: [ { id: "qwen3:8b" } ] }.to_json
+                body: { data: [ { id: "gpt-4.1" } ] }.to_json
               )
 
     result = @probe.llm(
       provider: :openai,
-      endpoint: "http://ollama.example.test:11434/v1",
+      endpoint: "https://api.openai.example.test/v1",
       access_token: "local-token",
-      model: "qwen3:8b"
+      model: "gpt-4.1"
+    )
+
+    assert result.passing?
+    assert result.checked_at
+    assert_requested request
+  end
+
+  test "OpenAI-compatible LLM probe calls chat completions instead of the models endpoint" do
+    endpoint = "https://api.cloudflare.com/client/v4/accounts/account-id/ai/v1"
+    request = stub_request(:post, "#{endpoint}/chat/completions")
+              .with(
+                headers: { "Authorization" => "Bearer cf-token" },
+                body: {
+                  model: "@cf/zai-org/glm-5.2",
+                  messages: [ { role: "user", content: AiHealth::Probe::CHAT_TEST_INPUT } ]
+                }
+              )
+              .to_return(
+                status: 200,
+                headers: { "Content-Type" => "application/json" },
+                body: { choices: [ { message: { content: "OK" } } ] }.to_json
+              )
+
+    result = @probe.llm(
+      provider: :openai,
+      endpoint: endpoint,
+      access_token: "cf-token",
+      model: "@cf/zai-org/glm-5.2",
+      openai_compatible: true
     )
 
     assert result.passing?

test/system/accounts_sync_ui_test.rb

@@ -3,6 +3,7 @@ require "application_system_test_case"
 class AccountsSyncUiTest < ApplicationSystemTestCase
   setup do
     @user = users(:family_admin)
+    Sync.for_family(@user.family).incomplete.destroy_all
     sign_in @user
   end