+ <%= t(".model_function_calling_help") %> + <% if Current.user&.super_admin? %> + <%= link_to t(".model_function_calling_link"), admin_system_health_path(tab: "ai"), class: "underline" %> + <% end %> +
<%= form.select :openai_json_mode, options_for_select( diff --git a/config/locales/views/admin/system_health/en.yml b/config/locales/views/admin/system_health/en.yml index 939e97c69..d1d52faab 100644 --- a/config/locales/views/admin/system_health/en.yml +++ b/config/locales/views/admin/system_health/en.yml @@ -42,6 +42,12 @@ en: custom_openai: title: The hosted vector-store check failed at this custom endpoint message: The configured endpoint did not pass the /v1/vector_stores liveness check. Most local and OpenAI-compatible endpoints do not implement that API. Chat and PDF processing may still work; use pgvector with a separate embeddings endpoint for the usual local setup. The failure was recorded in Settings → Debug logs and Rails.logger. + function_calling_unsupported: + title: The model does not support function calling + message: The endpoint served a plain chat request but rejected the same request carrying the tools parameter. The AI assistant reads accounts, transactions, and holdings through function calls, so chat keeps failing — often with an unhelpful 404 — until the model is one that supports tools. Pick a model whose provider documents function-calling support. The failure was recorded in Settings → Debug logs and Rails.logger. + function_calling_not_used: + title: The model answered without calling the tool it was asked to call + message: The endpoint accepted the tools parameter, but the model replied with text instead of calling the health-check tool. The AI assistant cannot read financial data without tool calls, so chat may answer with nothing or with invented figures. Prefer a model whose provider documents function-calling support. llm_probe_failed: title: The LLM live check failed message: Sure could not verify both the configured endpoint and model. The failure was recorded in Settings → Debug logs and Rails.logger. @@ -65,7 +71,7 @@ en: message: Check the adapter requirements below. For pgvector, the database must provide the vector extension or an existing vector_store_chunks table. llm: title: LLM and PDF processing - description: The live checks verify the configured model, then separately exercise text extraction and vision/native processing with a one-page synthetic PDF containing no customer data. Credential values are never displayed. + description: The live checks verify the configured model, ask it for one trivial function call the way the assistant does, then separately exercise text extraction and vision/native processing with a one-page synthetic PDF containing no customer data. Credential values are never displayed. vector_store: title: Vector store and document search description: The live check queries the hosted vector-store API, or tests the pgvector table and creates one short embedding without storing it. @@ -74,6 +80,7 @@ en: effective_provider: Effective provider model: Model endpoint: Endpoint + function_calling: Function calling (tools) pdf_text_extraction: PDF text-extraction path pdf_vision_processing: PDF vision/native path request_timeout: Request timeout @@ -87,6 +94,7 @@ en: embedding_model: Embedding model embedding_endpoint: Embedding endpoint embedding_dimensions: Embedding dimensions + function_calling_failure_reason: Function-calling failure reason pdf_text_extraction_failure_reason: Text-extraction failure reason pdf_vision_processing_failure_reason: Vision/native failure reason storage_probe: pgvector storage check @@ -118,6 +126,13 @@ en: not_available: Not available available: Available not_found: Not found + function_calling_statuses: + supported: Live tool call succeeded + unsupported: Not supported by the effective provider/model + not_used: Tools accepted, but the model called none + failing: Live check failed + not_checked: Not checked + unavailable: Unavailable until an LLM provider is configured pdf_statuses: supported: Synthetic PDF check passed failing: Synthetic PDF check failed @@ -138,6 +153,8 @@ en: missing: Not configured failure_codes: model_not_available: The configured model was not returned by the provider + no_tool_call: The model answered without calling the health-check tool + tools_refused: The service served the same request without tools and refused it with them invalid_response: The service returned an unexpected response dimensions_mismatch: The embedding vector dimensions do not match the configured dimensions extension_not_enabled: The PostgreSQL vector extension is not enabled diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index 1b2068598..23af61131 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -127,6 +127,8 @@ en: uri_base_placeholder: "https://api.openai.com/v1 (default)" model_label: Model (Optional) model_placeholder: "gpt-4.1 (default)" + model_function_calling_help: "The AI assistant reads your data through function calls, so pick a model your provider documents as supporting tools/function calling. Models without it fail chat with an opaque provider error." + model_function_calling_link: Check the configured model json_mode_label: JSON Mode json_mode_auto: Auto (recommended) json_mode_strict: Strict (best for thinking models) diff --git a/docs/hosting/ai.md b/docs/hosting/ai.md index a8157ec3e..f32c44992 100644 --- a/docs/hosting/ai.md +++ b/docs/hosting/ai.md @@ -1082,6 +1082,30 @@ ollama list # See what's installed ollama pull model-name # Install a model ``` +### Chat Fails With a Bare "404" (Model Without Function Calling) + +**Symptom:** The assistant answers every message with an unexplained `404` (or +another opaque provider error), while the same endpoint and model work for +auto-categorization. + +**Cause:** The assistant reads accounts, transactions, and holdings through +function calls, so every chat request carries a `tools` payload. A model +without function-calling support rejects it — OpenRouter answers `404` for +models such as `tngtech/deepseek-r1t2-chimera:free`. + +**Confirm it:** Open **System health → AI status** +(`/admin/system_health?tab=ai`). **Function calling (tools)** reports *Not +supported by the effective provider/model* when the endpoint served the plain +chat check but rejected the same request carrying tools, and *Tools accepted, +but the model called none* when the model answered with text instead of +calling the tool. + +**Fix:** Set `OPENAI_MODEL` to a model your provider documents as supporting +tools/function calling — see [For Chat Assistant](#for-chat-assistant) above — +then run the checks again. Free tiers are a poor place to look: providers +commonly log their prompts and completions for training, and the assistant +sends your accounts, transactions, and holdings in every tool call. + ### "Fixed prompt tokens exceed context budget" **Symptom:** Auto-categorization or merchant detection fails immediately with an error like: @@ -1408,6 +1432,9 @@ live checks against the effective configuration: - OpenAI-compatible and Anthropic providers must return the configured model from their models API. +- The configured model must complete one trivial function call, sent the way + the assistant sends its own tools. A model that serves plain chat but rejects + or ignores the `tools` parameter cannot answer questions about your data. - The hosted OpenAI vector-store adapter must answer a list request without creating or changing a store. - The pgvector adapter must have its extension enabled, its chunks table diff --git a/test/controllers/admin/system_health_controller_test.rb b/test/controllers/admin/system_health_controller_test.rb index 5af9070db..2d6f81fa1 100644 --- a/test/controllers/admin/system_health_controller_test.rb +++ b/test/controllers/admin/system_health_controller_test.rb @@ -3,7 +3,8 @@ require "test_helper" class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest AI_ENVIRONMENT = %w[ OPENAI_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_MODEL OPENAI_REQUEST_TIMEOUT - OPENAI_SUPPORTS_PDF_PROCESSING ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY + OPENAI_SUPPORTS_PDF_PROCESSING OPENAI_SUPPORTS_RESPONSES_ENDPOINT + ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY ANTHROPIC_BASE_URL ANTHROPIC_MODEL ANTHROPIC_REQUEST_TIMEOUT VECTOR_STORE_PROVIDER EMBEDDING_URI_BASE EMBEDDING_MODEL EMBEDDING_DIMENSIONS EMBEDDING_ACCESS_TOKEN QDRANT_URL QDRANT_API_KEY @@ -19,6 +20,7 @@ class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest Setting.stubs(:anthropic_base_url).returns(nil) Setting.stubs(:anthropic_model).returns(nil) AiHealth::Probe.any_instance.stubs(:llm).returns(probe_result(:passing)) + AiHealth::Probe.any_instance.stubs(:function_calling).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:pdf_text_extraction).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:pdf_vision_processing).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:openai_vector_store).returns(probe_result(:passing)) @@ -109,6 +111,7 @@ class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest sign_in users(:sure_support_staff) stub_healthy_sidekiq AiHealth::Probe.any_instance.expects(:llm).never + AiHealth::Probe.any_instance.expects(:function_calling).never AiHealth::Probe.any_instance.expects(:pdf_text_extraction).never AiHealth::Probe.any_instance.expects(:pdf_vision_processing).never AiHealth::Probe.any_instance.expects(:openai_vector_store).never @@ -156,6 +159,45 @@ class Admin::SystemHealthControllerTest < ActionDispatch::IntegrationTest assert_no_match(/local-token|uri-secret|query-secret/, response.body) end + test "AI status names the missing function-calling support behind an unhelpful chat error" do + sign_in users(:sure_support_staff) + stub_healthy_sidekiq + AiHealth::Probe.any_instance.stubs(:function_calling).returns( + probe_result(:failing, failure_code: :tools_refused, http_status: 404) + ) + + with_ai_environment( + "OPENAI_ACCESS_TOKEN" => "router-secret", + "OPENAI_URI_BASE" => "https://openrouter.ai/api/v1", + "OPENAI_MODEL" => "tngtech/deepseek-r1t2-chimera:free" + ) do + get admin_system_health_url(tab: "ai") + end + + assert_response :success + assert_select "[data-testid='function-calling-status']", text: /Not supported by the effective provider/ + assert_match(/The model does not support function calling/, response.body) + assert_match(/Function-calling failure reason/, response.body) + assert_no_match(/router-secret/, response.body) + end + + test "AI status separates a model that ignores tools from one that cannot use them" do + sign_in users(:sure_support_staff) + stub_healthy_sidekiq + AiHealth::Probe.any_instance.stubs(:function_calling).returns( + probe_result(:failing, failure_code: :no_tool_call) + ) + + with_ai_environment("OPENAI_ACCESS_TOKEN" => "sk-secret-openai") do + get admin_system_health_url(tab: "ai") + end + + assert_response :success + assert_select "[data-testid='function-calling-status']", text: /Tools accepted, but the model called none/ + assert_match(/answered without calling the tool it was asked to call/, response.body) + assert_no_match(/The model does not support function calling/, response.body) + end + test "AI status reports text and vision PDF probes separately" do sign_in users(:sure_support_staff) stub_healthy_sidekiq diff --git a/test/models/ai_health/probe_test.rb b/test/models/ai_health/probe_test.rb index 5e7430cd0..7b5e28a0f 100644 --- a/test/models/ai_health/probe_test.rb +++ b/test/models/ai_health/probe_test.rb @@ -378,4 +378,188 @@ class AiHealth::ProbeTest < ActiveSupport::TestCase assert result.failing? assert_equal :dimensions_mismatch, result.failure_code end + + test "function-calling probe sends the assistant's tools payload and passes when the model calls one" do + endpoint = "https://openrouter.example.test/api/v1" + request = stub_request(:post, "#{endpoint}/chat/completions") + .with { |req| + body = JSON.parse(req.body) + tool = body.dig("tools", 0, "function") + + body["model"] == "tools-model" && + body.dig("messages", 0, "content") == AiHealth::Probe::FUNCTION_CALL_TEST_INPUT && + tool["name"] == AiHealth::Probe::FUNCTION_CALL_TEST_TOOL[:name] && + tool["strict"] == true + } + .to_return( + status: 200, + headers: { "Content-Type" => "application/json" }, + body: { + choices: [ + { + message: { + tool_calls: [ + { id: "call_1", type: "function", function: { name: "sure_health_check", arguments: "{}" } } + ] + } + } + ] + }.to_json + ) + + result = @probe.function_calling( + provider: :openai, + endpoint: endpoint, + access_token: "token", + model: "tools-model" + ) + + assert result.passing? + assert_requested request + end + + test "function-calling probe fails when the model answers without calling the tool" do + endpoint = "https://openrouter.example.test/api/v1" + stub_request(:post, "#{endpoint}/chat/completions").to_return( + status: 200, + headers: { "Content-Type" => "application/json" }, + body: { choices: [ { message: { content: "ok" } } ] }.to_json + ) + Rails.logger.stubs(:error) + DebugLogEntry.stubs(:capture) + + result = @probe.function_calling( + provider: :openai, + endpoint: endpoint, + access_token: "token", + model: "chat-only-model" + ) + + assert result.failing? + assert_equal :no_tool_call, result.failure_code + end + + test "function-calling probe confirms a refusal by re-asking without the tools" do + endpoint = "https://openrouter.example.test/api/v1" + stub_tools_request(endpoint).to_return( + status: 404, + headers: { "Content-Type" => "application/json" }, + body: { error: { message: "No endpoints found that support tool use." } }.to_json + ) + control = stub_control_request(endpoint).to_return( + status: 200, + headers: { "Content-Type" => "application/json" }, + body: { choices: [ { message: { content: "ok" } } ] }.to_json + ) + Rails.logger.stubs(:error) + DebugLogEntry.stubs(:capture) + + result = @probe.function_calling( + provider: :openai, + endpoint: endpoint, + access_token: "token", + model: "tngtech/deepseek-r1t2-chimera:free" + ) + + assert result.failing? + assert_equal :tools_refused, result.failure_code + assert_requested control + end + + test "function-calling probe does not blame the tools for a client error the request gets either way" do + endpoint = "https://models.example.test/v1" + stub_tools_request(endpoint).to_return(status: 422, body: "{}", headers: { "Content-Type" => "application/json" }) + control = stub_control_request(endpoint).to_return( + status: 422, + body: "{}", + headers: { "Content-Type" => "application/json" } + ) + Rails.logger.stubs(:error) + DebugLogEntry.stubs(:capture) + + result = @probe.function_calling( + provider: :openai, + endpoint: endpoint, + access_token: "token", + model: "picky-model" + ) + + assert result.failing? + assert_equal :request_failed, result.failure_code + assert_equal 422, result.http_status + assert_requested control + end + + test "function-calling probe does not re-ask when the service said not now" do + endpoint = "https://models.example.test/v1" + stub_tools_request(endpoint).to_return(status: 429, body: "{}", headers: { "Content-Type" => "application/json" }) + control = stub_control_request(endpoint) + Rails.logger.stubs(:error) + DebugLogEntry.stubs(:capture) + + result = @probe.function_calling( + provider: :openai, + endpoint: endpoint, + access_token: "token", + model: "busy-model" + ) + + assert result.failing? + assert_equal :request_failed, result.failure_code + assert_equal 429, result.http_status + assert_not_requested control + end + + test "function-calling probe uses the responses endpoint when the assistant would" do + request = stub_request(:post, "https://api.openai.example.test/v1/responses") + .with { |req| + tool = JSON.parse(req.body).dig("tools", 0) + + tool["type"] == "function" && tool["name"] == AiHealth::Probe::FUNCTION_CALL_TEST_TOOL[:name] + } + .to_return( + status: 200, + headers: { "Content-Type" => "application/json" }, + body: { output: [ { type: "function_call", name: "sure_health_check", arguments: "{}" } ] }.to_json + ) + + result = @probe.function_calling( + provider: :openai, + endpoint: "https://api.openai.example.test/v1", + access_token: "token", + model: "gpt-4.1", + use_responses_endpoint: true + ) + + assert result.passing? + assert_requested request + end + + test "function-calling probe reads an Anthropic tool_use block" do + response = Struct.new(:content).new([ Struct.new(:type, :name).new(:tool_use, "sure_health_check") ]) + messages = mock("anthropic_messages") + messages.expects(:create).with do |params| + params[:tools].first[:input_schema] == AiHealth::Probe::FUNCTION_CALL_TEST_TOOL[:schema] && + !params[:tools].first.key?(:strict) + end.returns(response) + @probe.stubs(:anthropic_client).returns(stub(messages: messages)) + + result = @probe.function_calling( + provider: :anthropic, + endpoint: "https://api.anthropic.com", + access_token: "token", + model: "claude-sonnet-4-6" + ) + + assert result.passing? + end + + private + def stub_tools_request(endpoint) + stub_request(:post, "#{endpoint}/chat/completions").with { |request| JSON.parse(request.body).key?("tools") } + end + + def stub_control_request(endpoint) + stub_request(:post, "#{endpoint}/chat/completions").with { |request| !JSON.parse(request.body).key?("tools") } + end end diff --git a/test/models/ai_health_test.rb b/test/models/ai_health_test.rb index 04763ff36..3c3b87235 100644 --- a/test/models/ai_health_test.rb +++ b/test/models/ai_health_test.rb @@ -4,6 +4,7 @@ class AiHealthTest < ActiveSupport::TestCase AI_ENVIRONMENT = %w[ OPENAI_ACCESS_TOKEN OPENAI_URI_BASE OPENAI_MODEL ANTHROPIC_ACCESS_TOKEN ANTHROPIC_API_KEY VECTOR_STORE_PROVIDER + OPENAI_SUPPORTS_RESPONSES_ENDPOINT ].index_with(nil).freeze setup do @@ -42,7 +43,103 @@ class AiHealthTest < ActiveSupport::TestCase end end + test "a confirmed tools refusal reads as missing function-calling support" do + health = probed_health(llm: :passing, function_calling: failing_result(:tools_refused, http_status: 404)) + + assert_equal :unsupported, health.function_calling_status + end + + test "a service that fell over is not read as a model without function calling" do + [ + failing_result(:request_failed, http_status: 422), + failing_result(:request_failed, http_status: 429), + failing_result(:request_failed, http_status: 500), + failing_result(:request_failed), + failing_result(:invalid_response), + failing_result(:timeout) + ].each do |probe_result| + health = probed_health(llm: :passing, function_calling: probe_result) + + assert_equal :failing, health.function_calling_status, + "#{probe_result.failure_code} #{probe_result.http_status} should not blame the model" + end + end + + test "a model that answers without calling the tool is reported separately" do + health = probed_health(llm: :passing, function_calling: failing_result(:no_tool_call)) + + assert_equal :not_used, health.function_calling_status + end + + test "a refusal still reads as unsupported when the plain LLM check is failing too" do + health = probed_health(llm: :failing, function_calling: failing_result(:tools_refused)) + + assert_equal :unsupported, health.function_calling_status + end + + test "function calling is probed through the API the assistant would use" do + { + { "OPENAI_URI_BASE" => nil } => true, + { "OPENAI_URI_BASE" => "https://openrouter.ai/api/v1" } => false, + { "OPENAI_URI_BASE" => "https://openrouter.ai/api/v1", "OPENAI_SUPPORTS_RESPONSES_ENDPOINT" => "true" } => true, + { "OPENAI_SUPPORTS_RESPONSES_ENDPOINT" => "false" } => false + }.each do |environment, use_responses_endpoint| + AiHealth::Probe.any_instance.stubs(:llm).returns(result(:passing)) + AiHealth::Probe.any_instance.stubs(:pdf_text_extraction).returns(result(:passing)) + AiHealth::Probe.any_instance.stubs(:pdf_vision_processing).returns(result(:passing)) + AiHealth::Probe.any_instance.expects(:function_calling) + .with(has_entry(use_responses_endpoint: use_responses_endpoint)) + .returns(result(:passing)) + + ClimateControl.modify( + AI_ENVIRONMENT.merge( + "OPENAI_ACCESS_TOKEN" => "test-token", + "OPENAI_MODEL" => "test-model" + ).merge(environment) + ) { AiHealth.new } + end + end + + test "function calling is only checked alongside the other live probes" do + with_openai_endpoint("https://openrouter.ai/api/v1") do |health| + assert_equal :not_checked, health.function_calling_status + end + end + private + def probed_health(llm:, function_calling:) + AiHealth::Probe.any_instance.stubs(:llm).returns(result(llm)) + AiHealth::Probe.any_instance.stubs(:function_calling).returns(function_calling) + AiHealth::Probe.any_instance.stubs(:pdf_text_extraction).returns(result(:passing)) + AiHealth::Probe.any_instance.stubs(:pdf_vision_processing).returns(result(:passing)) + + ClimateControl.modify( + AI_ENVIRONMENT.merge( + "OPENAI_ACCESS_TOKEN" => "test-token", + "OPENAI_URI_BASE" => "https://openrouter.ai/api/v1", + "OPENAI_MODEL" => "test-model" + ) + ) { AiHealth.new } + end + + def result(status) + AiHealth::Probe::Result.new( + status: status, + checked_at: Time.current, + failure_code: nil, + http_status: nil + ) + end + + def failing_result(failure_code, http_status: nil) + AiHealth::Probe::Result.new( + status: :failing, + checked_at: Time.current, + failure_code: failure_code, + http_status: http_status + ) + end + def with_openai_endpoint(endpoint) ClimateControl.modify( AI_ENVIRONMENT.merge( diff --git a/test/system/admin/system_health_test.rb b/test/system/admin/system_health_test.rb index 4b102dd4b..cdd4c41cc 100644 --- a/test/system/admin/system_health_test.rb +++ b/test/system/admin/system_health_test.rb @@ -9,6 +9,7 @@ class Admin::SystemHealthTest < ApplicationSystemTestCase Setting.stubs(:openai_model).returns(nil) stub_healthy_sidekiq AiHealth::Probe.any_instance.stubs(:llm).returns(probe_result(:passing)) + AiHealth::Probe.any_instance.stubs(:function_calling).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:pdf_text_extraction).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:pdf_vision_processing).returns(probe_result(:passing)) AiHealth::Probe.any_instance.stubs(:openai_vector_store).returns(probe_result(:passing)) @@ -28,6 +29,7 @@ class Admin::SystemHealthTest < ApplicationSystemTestCase assert_current_path admin_system_health_path(tab: "ai") assert_selector "button[role='tab'][aria-selected='true']", text: "AI status" assert_text "Live check passed" + assert_text "Live tool call succeeded" assert_text "PDF text-extraction path" assert_text "PDF vision/native path" assert_text "Synthetic PDF check passed", count: 2