From a0c552cb3879984cd93a4abda4078d83a8bc1639 Mon Sep 17 00:00:00 2001 From: Guillem Arias Date: Mon, 25 May 2026 19:58:30 +0200 Subject: [PATCH] test(chat): make default_model tests resilient to ENV model overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit flagged on PR review: the new default_model tests asserted against Provider::*::DEFAULT_MODEL, but Chat.default_model actually returns Provider::*.effective_model.presence (which reads OPENAI_MODEL / ANTHROPIC_MODEL from the environment). With either env var set, the tests would fail intermittently even though routing was correct. - New default_model tests now assert against the provider's effective_model directly, so they verify the routing decision (which provider's value wins) without coupling to the constant. - Pre-existing "creates with default model" assertions had the same brittleness; switch them to compare against Chat.default_model so the chosen model is whatever the env / Setting cascade resolves to. Verified by running `ANTHROPIC_MODEL=claude-haiku-4-5 OPENAI_MODEL=gpt-4o bin/rails test test/models/chat_test.rb` — 16 runs, 0 failures (previously 2 pre-existing failures + 0 from the new tests). --- test/models/chat_test.rb | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/models/chat_test.rb b/test/models/chat_test.rb index 7da1c78e5..accf5e0cb 100644 --- a/test/models/chat_test.rb +++ b/test/models/chat_test.rb @@ -47,7 +47,7 @@ class ChatTest < ActiveSupport::TestCase chat = @user.chats.start!(prompt, model: nil) assert_equal 2, chat.messages.count - assert_equal Provider::Openai::DEFAULT_MODEL, chat.messages.find_by!(type: "UserMessage").ai_model + assert_equal Chat.default_model, chat.messages.find_by!(type: "UserMessage").ai_model end end @@ -58,31 +58,35 @@ class ChatTest < ActiveSupport::TestCase chat = @user.chats.start!(prompt, model: "") assert_equal 2, chat.messages.count - assert_equal Provider::Openai::DEFAULT_MODEL, chat.messages.find_by!(type: "UserMessage").ai_model + assert_equal Chat.default_model, chat.messages.find_by!(type: "UserMessage").ai_model end end - test "default_model returns claude when LLM_PROVIDER=anthropic and Anthropic is configured" do + # These three tests assert routing (which provider's effective_model wins), + # not the constant value itself — the assertion side reads through + # Provider::*.effective_model so ENV overrides like ANTHROPIC_MODEL / + # OPENAI_MODEL don't make the tests flake. + test "default_model returns Anthropic's effective_model when LLM_PROVIDER=anthropic and Anthropic is configured" do Provider::Anthropic.stubs(:configured?).returns(true) Setting.stubs(:llm_provider).returns("anthropic") - assert_equal Provider::Anthropic::DEFAULT_MODEL, Chat.default_model + assert_equal Provider::Anthropic.effective_model, Chat.default_model end - test "default_model falls back to OpenAI when Anthropic is preferred but unconfigured" do + test "default_model falls back to OpenAI's effective_model when Anthropic is preferred but unconfigured" do Provider::Anthropic.stubs(:configured?).returns(false) Provider::Openai.stubs(:configured?).returns(true) Setting.stubs(:llm_provider).returns("anthropic") - assert_equal Provider::Openai::DEFAULT_MODEL, Chat.default_model + assert_equal Provider::Openai.effective_model, Chat.default_model end - test "default_model uses Anthropic when OpenAI is unconfigured" do + test "default_model uses Anthropic's effective_model when OpenAI is unconfigured" do Provider::Anthropic.stubs(:configured?).returns(true) Provider::Openai.stubs(:configured?).returns(false) Setting.stubs(:llm_provider).returns("openai") - assert_equal Provider::Anthropic::DEFAULT_MODEL, Chat.default_model + assert_equal Provider::Anthropic.effective_model, Chat.default_model end test "creates with configured model when OPENAI_MODEL env is set" do