QT/sure
1
0
Fork 0
mirror of https://github.com/we-promise/sure.git synced 2026-04-19 12:04:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

LLM cost estimation (#223)

Browse Source 
* Password reset back button also after confirmation

Signed-off-by: Juan José Mata <juanjo.mata@gmail.com>

* Implement a filter for category (#215)

- Also implement an is empty/is null condition.

* Implement an LLM cost estimation page

Track costs across all the cost categories: auto categorization, auto merchant detection and chat.
Show warning with estimated cost when running a rule that contains AI.

* Update pricing

* Add google pricing

and fix inferred model everywhere.

* Update app/models/llm_usage.rb

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Signed-off-by: soky srm <sokysrm@gmail.com>

* FIX address review

* Linter

* Address review

- Lowered log level
- extracted the duplicated record_usage method into a shared concern

* Update app/controllers/settings/llm_usages_controller.rb

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Signed-off-by: soky srm <sokysrm@gmail.com>

* Moved attr_reader out of private

---------

Signed-off-by: Juan José Mata <juanjo.mata@gmail.com>
Signed-off-by: soky srm <sokysrm@gmail.com>
Co-authored-by: Juan José Mata <juanjo.mata@gmail.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
soky srm
2025-10-24 00:08:59 +02:00
committed by GitHub
parent 4999409082
commit bb364fab38
19 changed files with 651 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
app/models/provider/openai/auto_categorizer.rb
View File

@@ -1,11 +1,16 @@
class Provider::Openai::AutoCategorizer
def initialize(client, model: "", transactions: [], user_categories: [], custom_provider: false, langfuse_trace: nil)
include Provider::Openai::Concerns::UsageRecorder
attr_reader :client, :model, :transactions, :user_categories, :custom_provider, :langfuse_trace, :family
def initialize(client, model: "", transactions: [], user_categories: [], custom_provider: false, langfuse_trace: nil, family: nil)
@client = client
@model = model
@transactions = transactions
@user_categories = user_categories
@custom_provider = custom_provider
@langfuse_trace = langfuse_trace
@family = family
end
def auto_categorize
@@ -64,6 +69,16 @@ class Provider::Openai::AutoCategorizer
categorizations = extract_categorizations_native(response)
result = build_response(categorizations)
record_usage(
model.presence || Provider::Openai::DEFAULT_MODEL,
response.dig("usage"),
operation: "auto_categorize",
metadata: {
transaction_count: transactions.size,
category_count: user_categories.size
}
)
span&.end(output: result.map(&:to_h), usage: response.dig("usage"))
result
rescue => e
@@ -99,6 +114,16 @@ class Provider::Openai::AutoCategorizer
categorizations = extract_categorizations_generic(response)
result = build_response(categorizations)
record_usage(
model.presence || Provider::Openai::DEFAULT_MODEL,
response.dig("usage"),
operation: "auto_categorize",
metadata: {
transaction_count: transactions.size,
category_count: user_categories.size
}
)
span&.end(output: result.map(&:to_h), usage: response.dig("usage"))
result
rescue => e
@@ -106,8 +131,6 @@ class Provider::Openai::AutoCategorizer
raise
end
attr_reader :client, :model, :transactions, :user_categories, :custom_provider, :langfuse_trace
AutoCategorization = Provider::LlmConcept::AutoCategorization
def build_response(categorizations)

29
app/models/provider/openai/auto_merchant_detector.rb
View File

@@ -1,11 +1,16 @@
class Provider::Openai::AutoMerchantDetector
def initialize(client, model: "", transactions:, user_merchants:, custom_provider: false, langfuse_trace: nil)
include Provider::Openai::Concerns::UsageRecorder
attr_reader :client, :model, :transactions, :user_merchants, :custom_provider, :langfuse_trace, :family
def initialize(client, model: "", transactions:, user_merchants:, custom_provider: false, langfuse_trace: nil, family: nil)
@client = client
@model = model
@transactions = transactions
@user_merchants = user_merchants
@custom_provider = custom_provider
@langfuse_trace = langfuse_trace
@family = family
end
def auto_detect_merchants
@@ -85,6 +90,16 @@ class Provider::Openai::AutoMerchantDetector
merchants = extract_merchants_native(response)
result = build_response(merchants)
record_usage(
model.presence || Provider::Openai::DEFAULT_MODEL,
response.dig("usage"),
operation: "auto_detect_merchants",
metadata: {
transaction_count: transactions.size,
merchant_count: user_merchants.size
}
)
span&.end(output: result.map(&:to_h), usage: response.dig("usage"))
result
rescue => e
@@ -120,6 +135,16 @@ class Provider::Openai::AutoMerchantDetector
merchants = extract_merchants_generic(response)
result = build_response(merchants)
record_usage(
model.presence || Provider::Openai::DEFAULT_MODEL,
response.dig("usage"),
operation: "auto_detect_merchants",
metadata: {
transaction_count: transactions.size,
merchant_count: user_merchants.size
}
)
span&.end(output: result.map(&:to_h), usage: response.dig("usage"))
result
rescue => e
@@ -127,8 +152,6 @@ class Provider::Openai::AutoMerchantDetector
raise
end
attr_reader :client, :model, :transactions, :user_merchants, :custom_provider, :langfuse_trace
AutoDetectedMerchant = Provider::LlmConcept::AutoDetectedMerchant
def build_response(categorizations)

47
app/models/provider/openai/concerns/usage_recorder.rb Normal file
View File

@@ -0,0 +1,47 @@
module Provider::Openai::Concerns::UsageRecorder
extend ActiveSupport::Concern
private
# Records LLM usage for a family
# Handles both old (prompt_tokens/completion_tokens) and new (input_tokens/output_tokens) API formats
# Automatically infers provider from model name
# Returns nil if pricing is unavailable (e.g., custom/self-hosted models)
def record_usage(model_name, usage_data, operation:, metadata: {})
return unless family && usage_data
# Handle both old and new OpenAI API response formats
# Old format: prompt_tokens, completion_tokens, total_tokens
# New format: input_tokens, output_tokens, total_tokens
prompt_tokens = usage_data["prompt_tokens"] || usage_data["input_tokens"] || 0
completion_tokens = usage_data["completion_tokens"] || usage_data["output_tokens"] || 0
total_tokens = usage_data["total_tokens"] || 0
estimated_cost = LlmUsage.calculate_cost(
model: model_name,
prompt_tokens: prompt_tokens,
completion_tokens: completion_tokens
)
# Log when we can't estimate the cost (e.g., custom/self-hosted models)
if estimated_cost.nil?
Rails.logger.info("Recording LLM usage without cost estimate for unknown model: #{model_name} (custom provider: #{custom_provider})")
end
inferred_provider = LlmUsage.infer_provider(model_name)
family.llm_usages.create!(
provider: inferred_provider,
model: model_name,
operation: operation,
prompt_tokens: prompt_tokens,
completion_tokens: completion_tokens,
total_tokens: total_tokens,
estimated_cost: estimated_cost,
metadata: metadata
)
Rails.logger.info("LLM usage recorded - Operation: #{operation}, Cost: #{estimated_cost.inspect}")
rescue => e
Rails.logger.error("Failed to record LLM usage: #{e.message}")
end
end