From e05869a4f4a45f93e9207accbc296f4cd5e17971 Mon Sep 17 00:00:00 2001 From: Garrett Date: Sat, 7 Feb 2026 01:47:39 +0000 Subject: [PATCH 001/277] feat: Allow creating a budget up to 2 years ahead --- app/models/budget.rb | 21 +++++++---- test/models/budget_test.rb | 74 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/app/models/budget.rb b/app/models/budget.rb index c345801fc..eee6a3254 100644 --- a/app/models/budget.rb +++ b/app/models/budget.rb @@ -29,13 +29,14 @@ class Budget < ApplicationRecord end def budget_date_valid?(date, family:) - if family.uses_custom_month_start? - budget_start = family.custom_month_start_for(date) - budget_start >= oldest_valid_budget_date(family) && budget_start <= family.custom_month_end_for(Date.current) + budget_start = if family.uses_custom_month_start? + family.custom_month_start_for(date) else - beginning_of_month = date.beginning_of_month - beginning_of_month >= oldest_valid_budget_date(family) && beginning_of_month <= Date.current.end_of_month + date.beginning_of_month end + + budget_start >= oldest_valid_budget_date(family) && + budget_start <= latest_valid_budget_start_date(family) end def find_or_bootstrap(family, start_date:) @@ -70,6 +71,14 @@ class Budget < ApplicationRecord oldest_entry_date = family.oldest_entry_date.beginning_of_month [ two_years_ago, oldest_entry_date ].min end + + def latest_valid_budget_start_date(family) + if family.uses_custom_month_start? + family.current_custom_month_period.start_date + 2.years + else + Date.current.beginning_of_month + 2.years + end + end end def period @@ -151,8 +160,6 @@ class Budget < ApplicationRecord end def next_budget_param - return nil if current? - next_date = start_date + 1.month return nil unless self.class.budget_date_valid?(next_date, family: family) diff --git a/test/models/budget_test.rb b/test/models/budget_test.rb index cd3e95307..b8cc8bdf2 100644 --- a/test/models/budget_test.rb +++ b/test/models/budget_test.rb @@ -58,8 +58,35 @@ class BudgetTest < ActiveSupport::TestCase refute Budget.budget_date_valid?(3.years.ago.beginning_of_month, family: @family) end - test "budget_date_valid? does not allow future dates beyond current month" do - refute Budget.budget_date_valid?(2.months.from_now, family: @family) + test "budget_date_valid? allows future dates up to 2 years ahead" do + travel_to Date.current.beginning_of_month do + assert Budget.budget_date_valid?(Date.current.beginning_of_month + 1.month, family: @family) + assert Budget.budget_date_valid?(Date.current.beginning_of_month + 2.years, family: @family) + end + end + + test "budget_date_valid? does not allow future dates beyond 2 years ahead" do + travel_to Date.current.beginning_of_month do + refute Budget.budget_date_valid?(Date.current.beginning_of_month + 2.years + 1.month, family: @family) + end + end + + test "budget_date_valid? for custom month start allows dates up to 2 years ahead" do + @family.update!(month_start_day: 15) + + travel_to Date.current.beginning_of_month do + cap_start = @family.current_custom_month_period.start_date + 2.years + assert Budget.budget_date_valid?(cap_start, family: @family) + end + end + + test "budget_date_valid? for custom month start does not allow dates beyond 2 years ahead" do + @family.update!(month_start_day: 15) + + travel_to Date.current.beginning_of_month do + beyond_cap = @family.current_custom_month_period.start_date + 2.years + 1.month + refute Budget.budget_date_valid?(beyond_cap, family: @family) + end end test "previous_budget_param returns nil when date is too old" do @@ -75,6 +102,49 @@ class BudgetTest < ActiveSupport::TestCase assert_nil budget.previous_budget_param end + test "next_budget_param returns next month when current month budget is selected" do + travel_to Date.current.beginning_of_month do + budget = Budget.create!( + family: @family, + start_date: Date.current.beginning_of_month, + end_date: Date.current.end_of_month, + currency: "USD" + ) + + assert_equal Budget.date_to_param(Date.current.beginning_of_month + 1.month), budget.next_budget_param + end + end + + test "next_budget_param returns nil at future cap" do + travel_to Date.current.beginning_of_month do + cap_start = Date.current.beginning_of_month + 2.years + budget = Budget.create!( + family: @family, + start_date: cap_start, + end_date: cap_start.end_of_month, + currency: "USD" + ) + + assert_nil budget.next_budget_param + end + end + + test "next_budget_param returns nil at future cap for custom month start" do + @family.update!(month_start_day: 15) + + travel_to Date.current.beginning_of_month do + cap_start = @family.current_custom_month_period.start_date + 2.years + budget = Budget.create!( + family: @family, + start_date: cap_start, + end_date: cap_start + 1.month - 1.day, + currency: "USD" + ) + + assert_nil budget.next_budget_param + end + end + test "actual_spending nets refunds against expenses in same category" do family = families(:dylan_family) budget = Budget.find_or_bootstrap(family, start_date: Date.current.beginning_of_month) From be42988adf04c939aa90cb4e409c753f9853a30d Mon Sep 17 00:00:00 2001 From: soky srm Date: Tue, 7 Apr 2026 20:46:05 +0200 Subject: [PATCH 002/277] Add throttling and cross-rate for twelve data (#1396) * Add throttling and cross-rate for twelve data * FIX yahoo precision also * FIXES * Update importer.rb * Fixes * Revert job * Fixes --- app/models/account/market_data_importer.rb | 6 - app/models/exchange_rate/importer.rb | 44 ++++- app/models/exchange_rate/provided.rb | 39 ++++- app/models/market_data_importer.rb | 6 - app/models/provider/twelve_data.rb | 94 ++++++++++- app/models/provider/yahoo_finance.rb | 2 +- .../account/market_data_importer_test.rb | 27 +--- test/models/exchange_rate/importer_test.rb | 58 ++++++- test/models/market_data_importer_test.rb | 12 +- test/models/provider/twelve_data_test.rb | 152 ++++++++++++++++++ 10 files changed, 379 insertions(+), 61 deletions(-) create mode 100644 test/models/provider/twelve_data_test.rb diff --git a/app/models/account/market_data_importer.rb b/app/models/account/market_data_importer.rb index c6caf7980..1c380c0dc 100644 --- a/app/models/account/market_data_importer.rb +++ b/app/models/account/market_data_importer.rb @@ -24,18 +24,12 @@ class Account::MarketDataImporter .each do |source_currency, date| key = [ source_currency, account.currency ] pair_dates[key] = [ pair_dates[key], date ].compact.min - - inverse_key = [ account.currency, source_currency ] - pair_dates[inverse_key] = [ pair_dates[inverse_key], date ].compact.min end # 2. ACCOUNT-BASED PAIR – convert the account currency to the family currency (if different) if foreign_account? key = [ account.currency, account.family.currency ] pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min - - inverse_key = [ account.family.currency, account.currency ] - pair_dates[inverse_key] = [ pair_dates[inverse_key], account.start_date ].compact.min end pair_dates.each do |(source, target), start_date| diff --git a/app/models/exchange_rate/importer.rb b/app/models/exchange_rate/importer.rb index a077401b3..247030b6e 100644 --- a/app/models/exchange_rate/importer.rb +++ b/app/models/exchange_rate/importer.rb @@ -15,6 +15,7 @@ class ExchangeRate::Importer def import_provider_rates if !clear_cache && all_rates_exist? Rails.logger.info("No new rates to sync for #{from} to #{to} between #{start_date} and #{end_date}, skipping") + backfill_inverse_rates_if_needed return end @@ -59,6 +60,25 @@ class ExchangeRate::Importer end upsert_rows(gapfilled_rates) + + # Compute and upsert inverse rates (e.g., EUR→USD from USD→EUR) to avoid + # separate API calls for the reverse direction. + inverse_rates = gapfilled_rates.filter_map do |row| + next if row[:rate].to_f <= 0 + + { + from_currency: row[:to_currency], + to_currency: row[:from_currency], + date: row[:date], + rate: (BigDecimal("1") / BigDecimal(row[:rate].to_s)).round(12) + } + end + + upsert_rows(inverse_rates) + + # Also backfill inverse rows for any forward rates that existed in the DB + # before effective_start_date (i.e. dates not covered by gapfilled_rates). + backfill_inverse_rates_if_needed end private @@ -84,7 +104,7 @@ class ExchangeRate::Importer # Since provider may not return values on weekends and holidays, we grab the first rate from the provider that is on or before the start date def start_rate_value - provider_rate_value = provider_rates.select { |date, _| date <= start_date }.max_by { |date, _| date }&.last + provider_rate_value = provider_rates.select { |date, _| date <= start_date }.max_by { |date, _| date }&.last&.rate db_rate_value = db_rates[start_date]&.rate provider_rate_value || db_rate_value end @@ -128,6 +148,28 @@ class ExchangeRate::Importer end end + # When forward rates already exist but inverse rates are missing (e.g. from a + # deployment before inverse computation was added), backfill them from the DB + # without making any provider API calls. + def backfill_inverse_rates_if_needed + existing_inverse_dates = ExchangeRate.where(from_currency: to, to_currency: from, date: start_date..end_date).pluck(:date).to_set + return if existing_inverse_dates.size >= expected_count + + inverse_rows = db_rates.filter_map do |_date, rate| + next if existing_inverse_dates.include?(rate.date) + next if rate.rate.to_f <= 0 + + { + from_currency: to, + to_currency: from, + date: rate.date, + rate: (BigDecimal("1") / BigDecimal(rate.rate.to_s)).round(12) + } + end + + upsert_rows(inverse_rows) if inverse_rows.any? + end + def all_rates_exist? db_count == expected_count end diff --git a/app/models/exchange_rate/provided.rb b/app/models/exchange_rate/provided.rb index 8a3f88af3..46e2a25de 100644 --- a/app/models/exchange_rate/provided.rb +++ b/app/models/exchange_rate/provided.rb @@ -69,14 +69,37 @@ module ExchangeRate::Provided return 0 end - ExchangeRate::Importer.new( - exchange_rate_provider: provider, - from: from, - to: to, - start_date: start_date, - end_date: end_date, - clear_cache: clear_cache - ).import_provider_rates + # Prevent concurrent syncs from fetching the same currency pair for overlapping + # date ranges. The lock is scoped to (pair + start_date) so that a broader range + # (e.g. daily job needing older history) is not blocked by a narrower account sync. + # + # Uses an owner-token pattern: the lock value is a unique token so the ensure + # block only deletes its own lock, not one acquired by a different worker after + # expiry. TTL is 5 minutes to cover worst-case throttle + rate-limit retry waits + # (~3 minutes with TwelveData). + lock_key = "exchange_rate_import:#{from}:#{to}:#{start_date}" + lock_token = SecureRandom.uuid + acquired = Rails.cache.write(lock_key, lock_token, expires_in: 5.minutes, unless_exist: true) + + unless acquired + Rails.logger.info("Skipping exchange rate import for #{from}/#{to} from #{start_date} — already in progress") + return 0 + end + + begin + ExchangeRate::Importer.new( + exchange_rate_provider: provider, + from: from, + to: to, + start_date: start_date, + end_date: end_date, + clear_cache: clear_cache + ).import_provider_rates + ensure + # Only delete the lock if we still own it (it hasn't expired and been + # re-acquired by another worker). + Rails.cache.delete(lock_key) if Rails.cache.read(lock_key) == lock_token + end end end end diff --git a/app/models/market_data_importer.rb b/app/models/market_data_importer.rb index d1ac7d7d7..86c3c2351 100644 --- a/app/models/market_data_importer.rb +++ b/app/models/market_data_importer.rb @@ -76,9 +76,6 @@ class MarketDataImporter .each do |(source, target), date| key = [ source, target ] pair_dates[key] = [ pair_dates[key], date ].compact.min - - inverse_key = [ target, source ] - pair_dates[inverse_key] = [ pair_dates[inverse_key], date ].compact.min end # 2. ACCOUNT-BASED PAIRS – use the account's oldest entry date @@ -94,9 +91,6 @@ class MarketDataImporter key = [ account.source, account.target ] pair_dates[key] = [ pair_dates[key], chosen_date ].compact.min - - inverse_key = [ account.target, account.source ] - pair_dates[inverse_key] = [ pair_dates[inverse_key], chosen_date ].compact.min end # Convert to array of hashes for ease of use diff --git a/app/models/provider/twelve_data.rb b/app/models/provider/twelve_data.rb index d360e5189..f448ef8ba 100644 --- a/app/models/provider/twelve_data.rb +++ b/app/models/provider/twelve_data.rb @@ -6,6 +6,10 @@ class Provider::TwelveData < Provider Error = Class.new(Provider::Error) InvalidExchangeRateError = Class.new(Error) InvalidSecurityPriceError = Class.new(Error) + RateLimitError = Class.new(Error) + + # Minimum delay between requests to avoid rate limiting (in seconds) + MIN_REQUEST_INTERVAL = 1.0 # Pattern to detect plan upgrade errors in API responses PLAN_UPGRADE_PATTERN = /available starting with (\w+)/i @@ -59,20 +63,23 @@ class Provider::TwelveData < Provider def fetch_exchange_rate(from:, to:, date:) with_provider_response do + throttle_request response = client.get("#{base_url}/exchange_rate") do |req| req.params["symbol"] = "#{from}/#{to}" req.params["date"] = date.to_s end - rate = JSON.parse(response.body).dig("rate") + parsed = JSON.parse(response.body) + check_api_error!(parsed) - Rate.new(date: date.to_date, from:, to:, rate: rate) + Rate.new(date: date.to_date, from:, to:, rate: parsed.dig("rate")) end end def fetch_exchange_rates(from:, to:, start_date:, end_date:) with_provider_response do # Try to fetch the currency pair via the time_series API (consumes 1 credit) - this might not return anything as the API does not provide time series data for all possible currency pairs + throttle_request response = client.get("#{base_url}/time_series") do |req| req.params["symbol"] = "#{from}/#{to}" req.params["start_date"] = start_date.to_s @@ -81,11 +88,13 @@ class Provider::TwelveData < Provider end parsed = JSON.parse(response.body) + check_api_error!(parsed) data = parsed.dig("values") # If currency pair is not available, try to fetch via the time_series/cross API (consumes 5 credits) if data.nil? Rails.logger.info("#{self.class.name}: Currency pair #{from}/#{to} not available, fetching via time_series/cross API") + throttle_request(credits: 5) response = client.get("#{base_url}/time_series/cross") do |req| req.params["base"] = from req.params["quote"] = to @@ -95,6 +104,7 @@ class Provider::TwelveData < Provider end parsed = JSON.parse(response.body) + check_api_error!(parsed) data = parsed.dig("values") end @@ -123,12 +133,14 @@ class Provider::TwelveData < Provider def search_securities(symbol, country_code: nil, exchange_operating_mic: nil) with_provider_response do + throttle_request response = client.get("#{base_url}/symbol_search") do |req| req.params["symbol"] = symbol req.params["outputsize"] = 25 end parsed = JSON.parse(response.body) + check_api_error!(parsed) data = parsed.dig("data") if data.nil? @@ -153,19 +165,23 @@ class Provider::TwelveData < Provider def fetch_security_info(symbol:, exchange_operating_mic:) with_provider_response do + throttle_request response = client.get("#{base_url}/profile") do |req| req.params["symbol"] = symbol req.params["mic_code"] = exchange_operating_mic end profile = JSON.parse(response.body) + check_api_error!(profile) + throttle_request response = client.get("#{base_url}/logo") do |req| req.params["symbol"] = symbol req.params["mic_code"] = exchange_operating_mic end logo = JSON.parse(response.body) + check_api_error!(logo) SecurityInfo.new( symbol: symbol, @@ -191,6 +207,7 @@ class Provider::TwelveData < Provider def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:) with_provider_response do + throttle_request response = client.get("#{base_url}/time_series") do |req| req.params["symbol"] = symbol req.params["mic_code"] = exchange_operating_mic @@ -200,6 +217,7 @@ class Provider::TwelveData < Provider end parsed = JSON.parse(response.body) + check_api_error!(parsed) values = parsed.dig("values") if values.nil? @@ -237,10 +255,11 @@ class Provider::TwelveData < Provider def client @client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday| faraday.request(:retry, { - max: 2, - interval: 0.05, + max: 3, + interval: 1.0, interval_randomness: 0.5, - backoff_factor: 2 + backoff_factor: 2, + exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ] }) faraday.request :json @@ -248,4 +267,69 @@ class Provider::TwelveData < Provider faraday.headers["Authorization"] = "apikey #{api_key}" end end + + # Paces API requests to stay within TwelveData's rate limits. Sleeps inline + # because the API physically cannot be called faster — this is unavoidable + # with a rate-limited provider. The 5-minute cache lock TTL in + # ExchangeRate::Provided accounts for worst-case throttle waits. + def throttle_request(credits: 1) + # Layer 1: Per-instance minimum interval between calls + @last_request_time ||= Time.at(0) + elapsed = Time.current - @last_request_time + sleep_time = min_request_interval - elapsed + sleep(sleep_time) if sleep_time > 0 + + # Layer 2: Global per-minute credit counter via cache (Redis in prod). + # Read current usage first — if adding these credits would exceed the limit, + # wait for the next minute BEFORE incrementing. This ensures credits are + # charged to the minute the request actually fires in, not a stale minute + # we slept through (which would undercount the new minute's usage). + minute_key = "twelve_data:credits:#{Time.current.to_i / 60}" + current_count = Rails.cache.read(minute_key).to_i + + if current_count + credits > max_requests_per_minute + wait_seconds = 60 - (Time.current.to_i % 60) + 1 + Rails.logger.info("TwelveData: #{current_count + credits}/#{max_requests_per_minute} credits this minute, waiting #{wait_seconds}s") + sleep(wait_seconds) + end + + # Charge credits to the minute the request actually fires in + active_minute_key = "twelve_data:credits:#{Time.current.to_i / 60}" + Rails.cache.increment(active_minute_key, credits, expires_in: 120.seconds) + + # Set timestamp after all waits so the next call's 1s pacing is measured + # from when this request actually fires, not from before the minute wait. + @last_request_time = Time.current + end + + def min_request_interval + ENV.fetch("TWELVE_DATA_MIN_REQUEST_INTERVAL", MIN_REQUEST_INTERVAL).to_f + end + + def max_requests_per_minute + ENV.fetch("TWELVE_DATA_MAX_REQUESTS_PER_MINUTE", 7).to_i + end + + def check_api_error!(parsed) + return unless parsed.is_a?(Hash) && parsed["code"].present? + + if parsed["code"] == 429 + raise RateLimitError, parsed["message"] || "Rate limit exceeded" + end + + raise Error, "API error (code: #{parsed["code"]}): #{parsed["message"] || "Unknown error"}" + end + + def default_error_transformer(error) + case error + when RateLimitError + error + when Faraday::TooManyRequestsError + RateLimitError.new("TwelveData rate limit exceeded", details: error.response&.dig(:body)) + when Faraday::Error + self.class::Error.new(error.message, details: error.response&.dig(:body)) + else + self.class::Error.new(error.message) + end + end end diff --git a/app/models/provider/yahoo_finance.rb b/app/models/provider/yahoo_finance.rb index d4a992184..d08e29dc0 100644 --- a/app/models/provider/yahoo_finance.rb +++ b/app/models/provider/yahoo_finance.rb @@ -438,7 +438,7 @@ class Provider::YahooFinance < Provider date: Time.at(timestamp).utc.to_date, from: from, to: to, - rate: (1.0 / close_rate.to_f).round(8) + rate: (BigDecimal("1") / BigDecimal(close_rate.to_s)).round(12) ) end diff --git a/test/models/account/market_data_importer_test.rb b/test/models/account/market_data_importer_test.rb index 0b3f7066d..520185f73 100644 --- a/test/models/account/market_data_importer_test.rb +++ b/test/models/account/market_data_importer_test.rb @@ -41,6 +41,7 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase expected_start_date = (existing_date + 1.day) - EXCHANGE_RATE_BUFFER end_date = Date.current.in_time_zone("America/New_York").to_date + # Only the forward pair (CAD→USD) should be fetched; inverse (USD→CAD) is computed automatically @provider.expects(:fetch_exchange_rates) .with(from: "CAD", to: "USD", @@ -50,20 +51,15 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase OpenStruct.new(from: "CAD", to: "USD", date: existing_date, rate: 1.5) ])) - @provider.expects(:fetch_exchange_rates) - .with(from: "USD", - to: "CAD", - start_date: expected_start_date, - end_date: end_date) - .returns(provider_success_response([ - OpenStruct.new(from: "USD", to: "CAD", date: existing_date, rate: 0.67) - ])) - before = ExchangeRate.count Account::MarketDataImporter.new(account).import_all after = ExchangeRate.count - assert_operator after, :>, before + 1, "Should insert at least two new exchange-rate rows" + assert_operator after, :>, before + 1, "Should insert at least two new exchange-rate rows (forward + computed inverse)" + + # Verify inverse rates were computed from the forward rates + assert ExchangeRate.where(from_currency: "USD", to_currency: "CAD").where("date > ?", existing_date).exists?, + "Inverse rates should be computed automatically" end test "syncs security prices for securities traded by the account" do @@ -237,7 +233,7 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase expected_start_date = (existing_date + 1.day) - EXCHANGE_RATE_BUFFER end_date = Date.current.in_time_zone("America/New_York").to_date - # Simulate provider returning an error response + # Only the forward pair (CAD→USD) should be fetched; inverse is computed automatically @provider.expects(:fetch_exchange_rates) .with(from: "CAD", to: "USD", @@ -247,15 +243,6 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase Provider::TwelveData::Error.new("Rate limit exceeded", details: { code: 429, message: "Rate limit exceeded" }) )) - @provider.expects(:fetch_exchange_rates) - .with(from: "USD", - to: "CAD", - start_date: expected_start_date, - end_date: end_date) - .returns(provider_error_response( - Provider::TwelveData::Error.new("Rate limit exceeded", details: { code: 429, message: "Rate limit exceeded" }) - )) - before = ExchangeRate.count # Should not raise an error, just log and continue diff --git a/test/models/exchange_rate/importer_test.rb b/test/models/exchange_rate/importer_test.rb index dab40fa81..cae3c5015 100644 --- a/test/models/exchange_rate/importer_test.rb +++ b/test/models/exchange_rate/importer_test.rb @@ -61,9 +61,9 @@ class ExchangeRate::ImporterTest < ActiveSupport::TestCase end_date: Date.current ).import_provider_rates - db_rates = ExchangeRate.order(:date) - assert_equal 4, db_rates.count - assert_equal [ 1.2, 1.25, 1.3, 1.3 ], db_rates.map(&:rate) + forward_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date) + assert_equal 4, forward_rates.count + assert_equal [ 1.2, 1.25, 1.3, 1.3 ], forward_rates.map(&:rate) end test "no provider calls when all rates exist" do @@ -137,7 +137,57 @@ class ExchangeRate::ImporterTest < ActiveSupport::TestCase end_date: future_date ).import_provider_rates - assert_equal 1, ExchangeRate.count + # 1 forward rate + 1 inverse rate + assert_equal 2, ExchangeRate.count + end + + test "upserts inverse rates alongside forward rates" do + ExchangeRate.delete_all + + provider_response = provider_success_response([ + OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 0.85) + ]) + + @provider.expects(:fetch_exchange_rates) + .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current) + .returns(provider_response) + + ExchangeRate::Importer.new( + exchange_rate_provider: @provider, + from: "USD", + to: "EUR", + start_date: Date.current, + end_date: Date.current + ).import_provider_rates + + forward = ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: Date.current) + inverse = ExchangeRate.find_by(from_currency: "EUR", to_currency: "USD", date: Date.current) + + assert_not_nil forward, "Forward rate should be stored" + assert_not_nil inverse, "Inverse rate should be computed and stored" + assert_in_delta 0.85, forward.rate.to_f, 0.0001 + assert_in_delta (1.0 / 0.85), inverse.rate.to_f, 0.0001 + end + + test "handles rate limit error gracefully" do + ExchangeRate.delete_all + + rate_limit_error = Provider::TwelveData::RateLimitError.new("Rate limit exceeded") + + @provider.expects(:fetch_exchange_rates).once.returns( + provider_error_response(rate_limit_error) + ) + + # Should not raise — logs warning and returns without importing + ExchangeRate::Importer.new( + exchange_rate_provider: @provider, + from: "USD", + to: "EUR", + start_date: Date.current, + end_date: Date.current + ).import_provider_rates + + assert_equal 0, ExchangeRate.count, "No rates should be imported on rate limit error" end private diff --git a/test/models/market_data_importer_test.rb b/test/models/market_data_importer_test.rb index 37d3070fe..27c4f5f0b 100644 --- a/test/models/market_data_importer_test.rb +++ b/test/models/market_data_importer_test.rb @@ -43,6 +43,7 @@ class MarketDataImporterTest < ActiveSupport::TestCase expected_start_date = (SNAPSHOT_START_DATE + 1.day) - EXCHANGE_RATE_BUFFER end_date = Date.current.in_time_zone("America/New_York").to_date + # Only the forward pair (CAD→USD) should be fetched; inverse (USD→CAD) is computed automatically @provider.expects(:fetch_exchange_rates) .with(from: "CAD", to: "USD", @@ -52,20 +53,11 @@ class MarketDataImporterTest < ActiveSupport::TestCase OpenStruct.new(from: "CAD", to: "USD", date: SNAPSHOT_START_DATE, rate: 1.5) ])) - @provider.expects(:fetch_exchange_rates) - .with(from: "USD", - to: "CAD", - start_date: expected_start_date, - end_date: end_date) - .returns(provider_success_response([ - OpenStruct.new(from: "USD", to: "CAD", date: SNAPSHOT_START_DATE, rate: 0.67) - ])) - before = ExchangeRate.count MarketDataImporter.new(mode: :snapshot).import_exchange_rates after = ExchangeRate.count - assert_operator after, :>, before + 1, "Should insert at least two new exchange-rate rows" + assert_operator after, :>, before + 1, "Should insert at least two new exchange-rate rows (forward + computed inverse)" end test "syncs security prices" do diff --git a/test/models/provider/twelve_data_test.rb b/test/models/provider/twelve_data_test.rb new file mode 100644 index 000000000..52a8b2719 --- /dev/null +++ b/test/models/provider/twelve_data_test.rb @@ -0,0 +1,152 @@ +require "test_helper" + +class Provider::TwelveDataTest < ActiveSupport::TestCase + setup do + @provider = Provider::TwelveData.new("test_api_key") + end + + # ================================ + # Rate Limit Detection Tests + # ================================ + + test "detects rate limit from JSON body code 429" do + rate_limit_body = { + "code" => 429, + "message" => "You have run out of API credits for the current minute.", + "status" => "error" + }.to_json + + mock_response = mock + mock_response.stubs(:body).returns(rate_limit_body) + + @provider.stubs(:throttle_request) + @provider.stubs(:client).returns(mock_client = mock) + mock_client.stubs(:get).returns(mock_response) + + result = @provider.fetch_exchange_rates(from: "USD", to: "EUR", start_date: Date.current, end_date: Date.current) + + assert_not result.success? + assert_instance_of Provider::TwelveData::RateLimitError, result.error + end + + test "detects rate limit on single exchange rate fetch" do + rate_limit_body = { + "code" => 429, + "message" => "Rate limit exceeded" + }.to_json + + mock_response = mock + mock_response.stubs(:body).returns(rate_limit_body) + + @provider.stubs(:throttle_request) + @provider.stubs(:client).returns(mock_client = mock) + mock_client.stubs(:get).returns(mock_response) + + result = @provider.fetch_exchange_rate(from: "USD", to: "EUR", date: Date.current) + + assert_not result.success? + assert_instance_of Provider::TwelveData::RateLimitError, result.error + end + + test "does not fall through to cross API when rate limited" do + rate_limit_body = { + "code" => 429, + "message" => "Rate limit exceeded" + }.to_json + + mock_response = mock + mock_response.stubs(:body).returns(rate_limit_body) + + @provider.stubs(:throttle_request) + mock_client = mock + # Should only be called once (time_series), NOT a second time (time_series/cross) + mock_client.expects(:get).once.returns(mock_response) + @provider.stubs(:client).returns(mock_client) + + result = @provider.fetch_exchange_rates(from: "USD", to: "EUR", start_date: Date.current, end_date: Date.current) + + assert_not result.success? + assert_instance_of Provider::TwelveData::RateLimitError, result.error + end + + # ================================ + # Error Transformer Tests + # ================================ + + test "default_error_transformer preserves RateLimitError" do + error = Provider::TwelveData::RateLimitError.new("Rate limit exceeded") + + result = @provider.send(:with_provider_response) { raise error } + + assert_not result.success? + assert_instance_of Provider::TwelveData::RateLimitError, result.error + end + + test "default_error_transformer converts Faraday 429 to RateLimitError" do + error = Faraday::TooManyRequestsError.new("Too Many Requests", { body: "Rate limited" }) + + result = @provider.send(:with_provider_response) { raise error } + + assert_not result.success? + assert_instance_of Provider::TwelveData::RateLimitError, result.error + end + + test "default_error_transformer wraps generic errors as Error" do + error = StandardError.new("Something went wrong") + + result = @provider.send(:with_provider_response) { raise error } + + assert_not result.success? + assert_instance_of Provider::TwelveData::Error, result.error + end + + # ================================ + # Throttle Tests + # ================================ + + test "throttle_request enforces minimum interval between calls" do + @provider.send(:instance_variable_set, :@last_request_time, Time.current) + + # Stub sleep to capture the call without actually sleeping + sleep_called_with = nil + @provider.define_singleton_method(:sleep) { |duration| sleep_called_with = duration } + + # Stub cache to return under limit (read returns current count, increment charges) + Rails.cache.stubs(:read).returns(0) + Rails.cache.stubs(:increment).returns(1) + + @provider.send(:throttle_request) + + assert_not_nil sleep_called_with, "Should have called sleep to enforce minimum interval" + assert_operator sleep_called_with, :>, 0 + end + + test "throttle_request waits when per-minute credit limit is exceeded" do + # Stub cache read to return count at limit (adding 1 more would exceed 7) + Rails.cache.stubs(:read).returns(7) + Rails.cache.stubs(:increment).returns(8) + + sleep_called = false + @provider.define_singleton_method(:sleep) { |_duration| sleep_called = true } + + @provider.send(:throttle_request) + + assert sleep_called, "Should have called sleep when credit limit exceeded" + end + + test "throttle_request does not wait when under credit limit" do + # Set last_request_time far in the past so per-instance throttle doesn't trigger + @provider.send(:instance_variable_set, :@last_request_time, Time.at(0)) + + # Stub cache to return under limit + Rails.cache.stubs(:read).returns(3) + Rails.cache.stubs(:increment).returns(4) + + sleep_called = false + @provider.define_singleton_method(:sleep) { |_duration| sleep_called = true } + + @provider.send(:throttle_request) + + assert_not sleep_called, "Should not sleep when under credit limit" + end +end From 2658c36b05c97b3596f8bca7daea3a4bb097fb55 Mon Sep 17 00:00:00 2001 From: Tao Chen <42793494+IamTaoChen@users.noreply.github.com> Date: Wed, 8 Apr 2026 02:52:14 +0800 Subject: [PATCH 003/277] feat(select): improve merchant dropdown behavior and placement controls (#1364) * feat(select): improve merchant dropdown behavior and placement controls - add configurable menu_placement strategy to DS::Select (auto/down/up) with safe normalization forward menu_placement through StyledFormBuilder#collection_select - force Merchant dropdown to open downward in transaction create and editor forms - fix select option/search text contrast by applying text-primary in DS select menu - prevent form jump on open by scrolling only inside dropdown content instead of using scrollIntoView - clamp internal dropdown scroll to valid bounds for stability - refactor select controller placement logic for readability (placementMode, clamp) without changing behavior * set menu_placement=auto for metchant selector --- app/components/DS/select.html.erb | 6 ++--- app/components/DS/select.rb | 11 ++++++-- app/helpers/styled_form_builder.rb | 1 + .../controllers/select_controller.js | 27 ++++++++++++++++--- app/views/transactions/_form.html.erb | 5 +++- app/views/transactions/show.html.erb | 2 +- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/app/components/DS/select.html.erb b/app/components/DS/select.html.erb index 170d3ee26..3ae5ce0a6 100644 --- a/app/components/DS/select.html.erb +++ b/app/components/DS/select.html.erb @@ -1,6 +1,6 @@ <%# locals: form:, method:, collection:, options: {} %> -
form-dropdown" data-action="dropdown:select->form-dropdown#onSelect"> +
form-dropdown" data-select-menu-placement-value="<%= menu_placement %>" data-action="dropdown:select->form-dropdown#onSelect">
<%= form.label method, options[:label], class: "form-field__label" if options[:label].present? %> @@ -27,7 +27,7 @@ " autocomplete="off" - class="bg-container text-sm placeholder:text-secondary font-normal h-10 pl-10 w-full border-none rounded-lg focus:outline-hidden focus:ring-0" + class="bg-container text-primary text-sm placeholder:text-secondary font-normal h-10 pl-10 w-full border-none rounded-lg focus:outline-hidden focus:ring-0" data-list-filter-target="input" data-action="list-filter#filter"> <%= helpers.icon("search", class: "absolute inset-0 ml-2 transform top-1/2 -translate-y-1/2") %> @@ -39,7 +39,7 @@ <% is_selected = item[:value] == selected_value %> <% obj = item[:object] %> -
" +
" role="option" tabindex="0" aria-selected="<%= is_selected %>" diff --git a/app/components/DS/select.rb b/app/components/DS/select.rb index abbd48ada..b19166bfb 100644 --- a/app/components/DS/select.rb +++ b/app/components/DS/select.rb @@ -1,18 +1,20 @@ module DS class Select < ViewComponent::Base - attr_reader :form, :method, :items, :selected_value, :placeholder, :variant, :searchable, :options + attr_reader :form, :method, :items, :selected_value, :placeholder, :variant, :searchable, :menu_placement, :options VARIANTS = %i[simple logo badge].freeze + MENU_PLACEMENTS = %w[auto down up].freeze HEX_COLOR_REGEX = /\A#[0-9a-fA-F]{3}(?:[0-9a-fA-F]{3})?\z/ RGB_COLOR_REGEX = /\Argb\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*\)\z/ DEFAULT_COLOR = "#737373" - def initialize(form:, method:, items:, selected: nil, placeholder: I18n.t("helpers.select.default_label"), variant: :simple, include_blank: nil, searchable: false, **options) + def initialize(form:, method:, items:, selected: nil, placeholder: I18n.t("helpers.select.default_label"), variant: :simple, include_blank: nil, searchable: false, menu_placement: :auto, **options) @form = form @method = method @placeholder = placeholder @variant = variant @searchable = searchable + @menu_placement = normalize_menu_placement(menu_placement) @options = options normalized_items = normalize_items(items) @@ -61,6 +63,11 @@ module DS private + def normalize_menu_placement(value) + normalized = value.to_s.downcase + MENU_PLACEMENTS.include?(normalized) ? normalized : "auto" + end + def normalize_items(collection) collection.map do |item| case item diff --git a/app/helpers/styled_form_builder.rb b/app/helpers/styled_form_builder.rb index bb51ab2ff..ba3b5f81e 100644 --- a/app/helpers/styled_form_builder.rb +++ b/app/helpers/styled_form_builder.rb @@ -44,6 +44,7 @@ class StyledFormBuilder < ActionView::Helpers::FormBuilder selected: selected_value, placeholder: placeholder, searchable: options.fetch(:searchable, false), + menu_placement: options[:menu_placement], variant: options.fetch(:variant, :simple), include_blank: options[:include_blank], label: options[:label], diff --git a/app/javascript/controllers/select_controller.js b/app/javascript/controllers/select_controller.js index 23b56051d..e4b07bb3b 100644 --- a/app/javascript/controllers/select_controller.js +++ b/app/javascript/controllers/select_controller.js @@ -2,9 +2,9 @@ import { Controller } from "@hotwired/stimulus" import { autoUpdate } from "@floating-ui/dom" export default class extends Controller { - static targets = ["button", "menu", "input"] + static targets = ["button", "menu", "input", "content"] static values = { - placement: { type: String, default: "bottom-start" }, + menuPlacement: { type: String, default: "auto" }, offset: { type: Number, default: 6 } } @@ -103,7 +103,25 @@ export default class extends Controller { scrollToSelected() { const selected = this.menuTarget.querySelector(".bg-container-inset") - if (selected) selected.scrollIntoView({ block: "center" }) + if (!selected) return + + const container = this.hasContentTarget ? this.contentTarget : this.menuTarget + const containerRect = container.getBoundingClientRect() + const selectedRect = selected.getBoundingClientRect() + const delta = selectedRect.top - containerRect.top - (container.clientHeight - selectedRect.height) / 2 + + const nextScrollTop = container.scrollTop + delta + const maxScrollTop = Math.max(0, container.scrollHeight - container.clientHeight) + container.scrollTop = this.clamp(nextScrollTop, 0, maxScrollTop) + } + + clamp(value, min, max) { + return Math.min(max, Math.max(min, value)) + } + + placementMode() { + const mode = (this.menuPlacementValue || "auto").toLowerCase() + return ["auto", "down", "up"].includes(mode) ? mode : "auto" } handleOutsideClick(event) { @@ -163,7 +181,8 @@ export default class extends Controller { const spaceBelow = containerRect.bottom - buttonRect.bottom const spaceAbove = buttonRect.top - containerRect.top - const shouldOpenUp = spaceBelow < menuHeight && spaceAbove > spaceBelow + const placement = this.placementMode() + const shouldOpenUp = placement === "up" || (placement === "auto" && spaceBelow < menuHeight && spaceAbove > spaceBelow) this.menuTarget.style.left = "0" this.menuTarget.style.width = "100%" diff --git a/app/views/transactions/_form.html.erb b/app/views/transactions/_form.html.erb index 062d87c8b..2ff28e05e 100644 --- a/app/views/transactions/_form.html.erb +++ b/app/views/transactions/_form.html.erb @@ -34,7 +34,10 @@ Current.family.available_merchants_for(Current.user).alphabetically, :id, :name, { include_blank: t(".none"), - label: t(".merchant_label") } %> + label: t(".merchant_label"), + variant: :logo, + searchable: true, + menu_placement: :auto } %> <%= ef.select :tag_ids, Current.family.tags.alphabetically.pluck(:name, :id), { diff --git a/app/views/transactions/show.html.erb b/app/views/transactions/show.html.erb index 1893b3364..eb7dbb4cf 100644 --- a/app/views/transactions/show.html.erb +++ b/app/views/transactions/show.html.erb @@ -109,7 +109,7 @@ :id, :name, { include_blank: t(".none"), label: t(".merchant_label"), - class: "text-subdued", variant: :logo, searchable: true, disabled: @entry.split_child? || !can_annotate_entry? }, + variant: :logo, searchable: true, menu_placement: :auto, disabled: @entry.split_child? || !can_annotate_entry? }, "data-auto-submit-form-target": "auto" %> <%= ef.select :tag_ids, Current.family.tags.alphabetically.pluck(:name, :id), From 133582ef4ddcacf30c7d8fd873ea179b0f267925 Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Tue, 7 Apr 2026 14:53:04 -0400 Subject: [PATCH 004/277] Fix balance sheet dashboard section on iPhone (#1318) * Update _balance_sheet.html.erb Update Balance Sheet to work on iPhone Pro Max without scrolling Signed-off-by: Derek Brown * Update _group_weight.html.erb Make the % icon smaller (5 bars not 10) to fit better on smaller format devices Signed-off-by: Derek Brown * Update _group_weight.html.erb Resolve Codex comment Signed-off-by: Derek Brown * Update _balance_sheet.html.erb Increasing width of the weight column Signed-off-by: Derek Brown --------- Signed-off-by: Derek Brown --- .../pages/dashboard/_balance_sheet.html.erb | 30 +++++++++---------- .../pages/dashboard/_group_weight.html.erb | 6 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/app/views/pages/dashboard/_balance_sheet.html.erb b/app/views/pages/dashboard/_balance_sheet.html.erb index 0f8710581..fafc75e2f 100644 --- a/app/views/pages/dashboard/_balance_sheet.html.erb +++ b/app/views/pages/dashboard/_balance_sheet.html.erb @@ -37,12 +37,12 @@
-
Name
-
-
+
Name
+
+

Weight

-
+

Value

@@ -55,34 +55,34 @@ <%= idx == classification_group.account_groups.size - 1 ? "rounded-b-lg" : "" %> "> -
+
<%= icon("chevron-right", class: "group-open:rotate-90") %>

<%= account_group.name %>

-
-
+
+
<%= render "pages/dashboard/group_weight", weight: account_group.weight, color: account_group.color %>
-
-

<%= format_money(account_group.total_money) %>

+
+

<%= format_money(account_group.total_money, precision: 0) %>
<% account_group.accounts.each_with_index do |account, idx| %> -
-
+
+
<%= render "accounts/logo", account: account, size: "sm", color: account_group.color %> <%= link_to account.name, account_path(account) %>
-
-
+
+
<% # Calculate weight as percentage of classification total classification_total = classification_group.total_money.amount @@ -91,8 +91,8 @@ <%= render "pages/dashboard/group_weight", weight: account_weight, color: account_group.color %>
-
-

<%= format_money(account.balance_money) %>

+
+

<%= format_money(account.balance_money, precision: 0) %>

diff --git a/app/views/pages/dashboard/_group_weight.html.erb b/app/views/pages/dashboard/_group_weight.html.erb index d3bf08e5c..4cf5e827d 100644 --- a/app/views/pages/dashboard/_group_weight.html.erb +++ b/app/views/pages/dashboard/_group_weight.html.erb @@ -4,9 +4,9 @@
- <% 10.times do |i| %> -
" style="background-color: <%= color %>;">
+ <% 5.times do |i| %> +
" style="background-color: <%= color %>;">
<% end %>
-

<%= number_to_percentage(effective_weight, precision: 2) %>

+

<%= number_to_percentage(effective_weight, precision: 0) %>

From 8e81e967fcdc3321f924c1689b295608b86ac3a7 Mon Sep 17 00:00:00 2001 From: "Sure Admin (bot)" Date: Wed, 8 Apr 2026 18:32:43 +0200 Subject: [PATCH 005/277] docs: add Yahoo Finance IPv6 troubleshooting note (#1404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test push * remove test file * docs: add Yahoo Finance IPv6 troubleshooting note to compose.example.yml In Podman/IPv6 environments, DNS may resolve fc.yahoo.com to an IPv6 address first. Since IPv6 is often not configured in containers, the connection hangs instead of falling back to IPv4. Thanks to @IndicareLeve for identifying and sharing the fix. * Revise IPv6 connection notes in compose.example.yml Light edits Signed-off-by: Juan José Mata --------- Signed-off-by: Juan José Mata Co-authored-by: SureBot Co-authored-by: Juan José Mata --- compose.example.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/compose.example.yml b/compose.example.yml index b52c047e1..ea45dc858 100644 --- a/compose.example.yml +++ b/compose.example.yml @@ -26,6 +26,21 @@ # # https://github.com/we-promise/sure/discussions/categories/general # +# IPv6 Note (Yahoo Finance Sync, others?): +# ---------------------------------------- +# If you encounter "Failed to open TCP connection" errors to hosts like +# fc.yahoo.com in your environment, the issue may be that DNS resolves +# to an IPv6 address first. Since IPv6 may not be configured in the container, +# the connection hangs. To work around this, you can hardcode the IPv4 +# addresses in your container's /etc/hosts or set custom DNS: +# +# extra_hosts: +# - "fc.yahoo.com:173.223.12.81" +# - "query1.finance.yahoo.com:98.136.48.169" +# - "query2.finance.yahoo.com:98.136.48.169" +# +# Or use explicit DNS servers that prefer IPv4 (already configured below). +# x-db-env: &db_env POSTGRES_USER: ${POSTGRES_USER:-sure_user} From f699660479f4f2bedacf59d4a5203d7cc189e4a7 Mon Sep 17 00:00:00 2001 From: "Pedro J. Aramburu" Date: Wed, 8 Apr 2026 16:05:58 -0300 Subject: [PATCH 006/277] Add exchange rate feature with multi-currency transactions and transfers support (#1099) Co-authored-by: Pedro J. Aramburu --- app/components/DS/select.html.erb | 3 +- app/components/UI/account/chart.rb | 6 +- app/controllers/exchange_rates_controller.rb | 36 +++ app/controllers/reports_controller.rb | 12 +- app/controllers/transactions_controller.rb | 26 +- app/controllers/transfers_controller.rb | 32 +- .../exchange_rate_form_controller.js | 298 ++++++++++++++++++ .../controllers/form_dropdown_controller.js | 3 + .../transaction_form_controller.js | 60 ++++ .../controllers/transfer_form_controller.js | 59 ++++ app/models/balance/sync_cache.rb | 15 +- app/models/holding/forward_calculator.rb | 6 +- app/models/holding/portfolio_cache.rb | 6 +- app/models/holding/reverse_calculator.rb | 6 +- app/models/transaction.rb | 37 +++ app/models/transfer/creator.rb | 16 +- app/views/shared/_exchange_rate_tabs.html.erb | 39 +++ app/views/shared/_money_field.html.erb | 18 +- app/views/transactions/_form.html.erb | 64 +++- app/views/transfers/_form.html.erb | 50 ++- config/locales/views/shared/ca.yml | 6 + config/locales/views/shared/de.yml | 6 + config/locales/views/shared/en.yml | 6 + config/locales/views/shared/es.yml | 6 + config/locales/views/shared/fr.yml | 6 + config/locales/views/shared/nb.yml | 32 +- config/locales/views/shared/nl.yml | 6 + config/locales/views/shared/pt-BR.yml | 6 + config/locales/views/shared/ro.yml | 6 + config/locales/views/shared/tr.yml | 6 + config/locales/views/shared/zh-CN.yml | 6 + config/locales/views/shared/zh-TW.yml | 6 + config/locales/views/transfers/en.yml | 8 + config/locales/views/transfers/es.yml | 8 + config/routes.rb | 2 + lib/money.rb | 20 +- test/application_system_test_case.rb | 14 + .../exchange_rates_controller_test.rb | 173 ++++++++++ .../transactions_controller_test.rb | 153 +++++++++ test/controllers/transfers_controller_test.rb | 137 ++++++++ test/lib/money_test.rb | 21 +- .../models/balance/forward_calculator_test.rb | 14 +- test/models/balance/sync_cache_test.rb | 156 +++++++++ test/models/holding_test.rb | 5 +- test/models/transaction_test.rb | 60 ++++ test/models/transfer/creator_test.rb | 100 ++++++ .../transactions_form_exchange_rate_test.rb | 146 +++++++++ test/system/transfers_test.rb | 52 +++ 48 files changed, 1886 insertions(+), 73 deletions(-) create mode 100644 app/controllers/exchange_rates_controller.rb create mode 100644 app/javascript/controllers/exchange_rate_form_controller.js create mode 100644 app/javascript/controllers/transaction_form_controller.js create mode 100644 app/javascript/controllers/transfer_form_controller.js create mode 100644 app/views/shared/_exchange_rate_tabs.html.erb create mode 100644 test/controllers/exchange_rates_controller_test.rb create mode 100644 test/models/balance/sync_cache_test.rb create mode 100644 test/system/transactions_form_exchange_rate_test.rb diff --git a/app/components/DS/select.html.erb b/app/components/DS/select.html.erb index 3ae5ce0a6..8f1ffe3e4 100644 --- a/app/components/DS/select.html.erb +++ b/app/components/DS/select.html.erb @@ -8,7 +8,8 @@ value: @selected_value, data: { "form-dropdown-target": "input", - "auto-submit-target": "auto" + "auto-submit-target": "auto", + **(options.dig(:html_options, :data) || {}) } %>
<% unless options[:hide_currency] %>
+ <% currency_data = (options[:currency_data] || {}).merge({ + "money-field-target": "currency", + "auto-submit-form-target": ("auto" if options[:auto_submit]) + }.compact) + # Preserve any existing action and append money-field handler + existing_action = currency_data.delete("action") + currency_data["action"] = ["change->money-field#handleCurrencyChange", existing_action].compact.join(" ") + %> <%= form.select currency_method, Money::Currency.as_options.map(&:iso_code), { inline: true, selected: currency.iso_code }, { class: "w-fit pr-5 disabled:text-subdued form-field__input", disabled: options[:disable_currency], - data: { - "money-field-target": "currency", - action: "change->money-field#handleCurrencyChange", - "auto-submit-form-target": ("auto" if options[:auto_submit]) - }.compact + data: currency_data } %>
<% end %> diff --git a/app/views/transactions/_form.html.erb b/app/views/transactions/_form.html.erb index 2ff28e05e..947f5444f 100644 --- a/app/views/transactions/_form.html.erb +++ b/app/views/transactions/_form.html.erb @@ -1,6 +1,7 @@ <%# locals: (entry:, categories:) %> -<%= styled_form_with model: entry, url: transactions_path, class: "space-y-4" do |f| %> +<% account_currencies = Current.family.accounts.map { |a| [a.id, a.currency] }.to_h.to_json %> +<%= styled_form_with model: entry, url: transactions_path, class: "space-y-4", data: { controller: "transaction-form", transaction_form_exchange_rate_url_value: exchange_rate_path, transaction_form_account_currencies_value: account_currencies } do |f| %> <% if entry.errors.any? %> <%= render "shared/form_errors", model: entry %> <% end %> @@ -16,16 +17,69 @@ <%= f.text_field :name, label: t(".description"), placeholder: t(".description_placeholder"), required: true %> <% if @entry.account_id %> - <%= f.hidden_field :account_id %> + <%= f.hidden_field :account_id, data: { transaction_form_target: "account" } %> <% else %> - <%= f.collection_select :account_id, accessible_accounts.manual.active.alphabetically, :id, :name, { prompt: t(".account_prompt"), label: t(".account"), selected: Current.user.default_account_for_transactions&.id, variant: :logo }, required: true, class: "form-field__input text-ellipsis" %> + <%= f.collection_select :account_id, accessible_accounts.manual.active.alphabetically, :id, :name, { prompt: t(".account_prompt"), label: t(".account"), selected: Current.user.default_account_for_transactions&.id, variant: :logo }, required: true, class: "form-field__input text-ellipsis", data: { transaction_form_target: "account", action: "change->transaction-form#checkCurrencyDifference" } %> <% end %> - <%= f.money_field :amount, label: t(".amount"), required: true %> + <%= f.money_field :amount, + label: t(".amount"), + required: true, + container_class: "money-field-wrapper", + amount_data: { transaction_form_target: "amount", action: "input->transaction-form#onAmountChange" }, + currency_data: { transaction_form_target: "currency", action: "change->transaction-form#onCurrencyChange" } %> + <%= f.fields_for :entryable do |ef| %> <%= ef.collection_select :category_id, categories, :id, :name, { prompt: t(".category_prompt"), label: t(".category"), variant: :badge, searchable: true } %> <% end %> - <%= f.date_field :date, label: t(".date"), required: true, min: Entry.min_supported_date, max: Date.current, value: Date.current %> + + <%= f.date_field :date, + label: t(".date"), + required: true, + min: Entry.min_supported_date, + max: Date.current, + value: f.object.date || Date.current, + data: { transaction_form_target: "date", action: "change->transaction-form#checkCurrencyDifference" } %> + + <% convert_input = capture do %> + <%= f.fields_for :entryable do |ef| %> + <%= ef.number_field :exchange_rate, + label: t("shared.exchange_rate_tabs.exchange_rate"), + min: "0.00000000000001", + step: "0.00000000000001", + placeholder: "1.0", + class: "form-field__input", + data: { + transaction_form_target: "exchangeRateField", + action: "input->transaction-form#onConvertExchangeRateChange" + } %> + <% end %> + <% end %> + + <% destination_input = capture do %> + <%= number_field_tag :destination_amount, + nil, + id: "transaction_form_destination_amount", + class: "form-field__input", + min: "0", + step: "0.00000001", + placeholder: "92", + data: { + transaction_form_target: "destinationAmount", + action: "input->transaction-form#onCalculateRateDestinationAmountChange" + } %> + <% end %> + + <%= render "shared/exchange_rate_tabs", + controller_id: "transaction-form", + controller_key: "transaction_form", + help_text: t("shared.exchange_rate_tabs.exchange_rate_help"), + convert_tab_label: t("shared.exchange_rate_tabs.convert_tab"), + calculate_rate_tab_label: t("shared.exchange_rate_tabs.calculate_rate_tab"), + destination_amount_label: t("shared.exchange_rate_tabs.destination_amount"), + exchange_rate_label: t("shared.exchange_rate_tabs.exchange_rate"), + convert_input: convert_input, + destination_input: destination_input %> <%= render DS::Disclosure.new(title: t(".details")) do %> diff --git a/app/views/transfers/_form.html.erb b/app/views/transfers/_form.html.erb index 39684a9cd..a7696692a 100644 --- a/app/views/transfers/_form.html.erb +++ b/app/views/transfers/_form.html.erb @@ -1,4 +1,5 @@ -<%= styled_form_with model: transfer, class: "space-y-4", data: { turbo_frame: "_top", controller: "transfer-form" } do |f| %> +<% account_currencies = Current.family.accounts.map { |a| [a.id, a.currency] }.to_h.to_json %> +<%= styled_form_with model: transfer, class: "space-y-4", data: { turbo_frame: "_top", controller: "transfer-form", transfer_form_exchange_rate_url_value: exchange_rate_path, transfer_form_account_currencies_value: account_currencies } do |f| %> <% if transfer.errors.present? %>
<%= icon "circle-alert", size: "sm" %> @@ -27,10 +28,49 @@ <% end %>
- <%= f.collection_select :from_account_id, @accounts, :id, :name, { prompt: t(".select_account"), label: t(".from"), selected: @from_account_id, variant: :logo }, required: true %> - <%= f.collection_select :to_account_id, @accounts, :id, :name, { prompt: t(".select_account"), label: t(".to"), variant: :logo }, required: true %> - <%= f.number_field :amount, label: t(".amount"), required: true, min: 0, placeholder: "100", step: 0.00000001 %> - <%= f.date_field :date, value: transfer.inflow_transaction&.entry&.date || Date.current, label: t(".date"), required: true, max: Date.current %> + <%= f.collection_select :from_account_id, @accounts, :id, :name, { prompt: t(".select_account"), label: t(".from"), selected: @from_account_id, variant: :logo }, { required: true, data: { transfer_form_target: "fromAccount", action: "change->transfer-form#checkCurrencyDifference" } } %> + <%= f.collection_select :to_account_id, @accounts, :id, :name, { prompt: t(".select_account"), label: t(".to"), variant: :logo }, { required: true, data: { transfer_form_target: "toAccount", action: "change->transfer-form#checkCurrencyDifference" } } %> + + <%= f.date_field :date, value: transfer.inflow_transaction&.entry&.date || Date.current, label: t(".date"), required: true, max: Date.current, data: { transfer_form_target: "date", action: "change->transfer-form#checkCurrencyDifference" } %> + + <%= f.number_field :amount, label: t(".source_amount"), required: true, min: 0, placeholder: "100", step: 0.00000001, data: { transfer_form_target: "amount", action: "input->transfer-form#onSourceAmountChange" } %> + + <% convert_input = capture do %> + <%= f.number_field :exchange_rate, + label: t("shared.exchange_rate_tabs.exchange_rate"), + min: "0.00000000000001", + step: "0.00000000000001", + placeholder: "1.0", + class: "form-field__input", + data: { + transfer_form_target: "exchangeRateField", + action: "input->transfer-form#onConvertExchangeRateChange" + } %> + <% end %> + + <% destination_input = capture do %> + <%= tag.input type: "number", + id: "transfer_form_destination_amount", + class: "form-field__input", + min: "0", + step: "0.00000001", + placeholder: "92", + data: { + transfer_form_target: "destinationAmount", + action: "input->transfer-form#onCalculateRateDestinationAmountChange" + } %> + <% end %> + + <%= render "shared/exchange_rate_tabs", + controller_id: "transfer-form", + controller_key: "transfer_form", + help_text: t("shared.exchange_rate_tabs.exchange_rate_help"), + convert_tab_label: t("shared.exchange_rate_tabs.convert_tab"), + calculate_rate_tab_label: t("shared.exchange_rate_tabs.calculate_rate_tab"), + destination_amount_label: t("shared.exchange_rate_tabs.destination_amount"), + exchange_rate_label: t("shared.exchange_rate_tabs.exchange_rate"), + convert_input: convert_input, + destination_input: destination_input %>
diff --git a/config/locales/views/shared/ca.yml b/config/locales/views/shared/ca.yml index 39c098d23..7dcbd7a99 100644 --- a/config/locales/views/shared/ca.yml +++ b/config/locales/views/shared/ca.yml @@ -8,6 +8,12 @@ ca: title: Segur que vols continuar? money_field: label: Import + exchange_rate_tabs: + calculate_rate_tab: Calcular la taxa FX + convert_tab: Convertir amb la taxa FX + destination_amount: Quantitat de destinació + exchange_rate: Taxa de canvi + exchange_rate_help: Trieu com introduir la quantitat. syncing_notice: syncing: S'estan sincronitzant les dades dels comptes... transaction_tabs: diff --git a/config/locales/views/shared/de.yml b/config/locales/views/shared/de.yml index 1b0cfb1bf..f0af087e2 100644 --- a/config/locales/views/shared/de.yml +++ b/config/locales/views/shared/de.yml @@ -8,6 +8,12 @@ de: title: Bist du sicher money_field: label: Betrag + exchange_rate_tabs: + calculate_rate_tab: FX-Kurs berechnen + convert_tab: Mit FX-Kurs konvertieren + destination_amount: Zielbetrag + exchange_rate: Wechselkurs + exchange_rate_help: Wählen Sie, wie Sie den Betrag eingeben möchten. syncing_notice: syncing: Kontodaten werden synchronisiert... trend_change: diff --git a/config/locales/views/shared/en.yml b/config/locales/views/shared/en.yml index f94797c77..98a6e479a 100644 --- a/config/locales/views/shared/en.yml +++ b/config/locales/views/shared/en.yml @@ -8,6 +8,12 @@ en: title: Are you sure? money_field: label: Amount + exchange_rate_tabs: + calculate_rate_tab: Calculate FX rate + convert_tab: Convert with FX rate + destination_amount: Destination amount + exchange_rate: Exchange rate + exchange_rate_help: Choose how to enter the amount. syncing_notice: syncing: Syncing accounts data... require_admin: "Only admins can perform this action" diff --git a/config/locales/views/shared/es.yml b/config/locales/views/shared/es.yml index db6ceb073..4b3f5914f 100644 --- a/config/locales/views/shared/es.yml +++ b/config/locales/views/shared/es.yml @@ -8,6 +8,12 @@ es: title: ¿Estás seguro? money_field: label: Importe + exchange_rate_tabs: + calculate_rate_tab: Calcular tasa + convert_tab: Convertir con tasa + destination_amount: Importe de destino + exchange_rate: Tasa de cambio + exchange_rate_help: Elige cómo introducir el importe. syncing_notice: syncing: Sincronizando datos de cuentas... trend_change: diff --git a/config/locales/views/shared/fr.yml b/config/locales/views/shared/fr.yml index 0ecdb2d32..8e95f5bbf 100644 --- a/config/locales/views/shared/fr.yml +++ b/config/locales/views/shared/fr.yml @@ -20,6 +20,12 @@ fr: title: Êtes-vous sûr ? money_field: label: Montant + exchange_rate_tabs: + calculate_rate_tab: Calculer le taux FX + convert_tab: Convertir avec le taux FX + destination_amount: Montant de destination + exchange_rate: Taux de change + exchange_rate_help: Choisissez comment entrer le montant. syncing_notice: syncing: Synchronisation des données de compte... trend_change: diff --git a/config/locales/views/shared/nb.yml b/config/locales/views/shared/nb.yml index a96b72952..c4e6eb8b0 100644 --- a/config/locales/views/shared/nb.yml +++ b/config/locales/views/shared/nb.yml @@ -1,14 +1,20 @@ ---- -nb: - shared: - confirm_modal: - accept: Bekreft - body_html: "

Du vil ikke kunne angre denne beslutningen

" - cancel: Avbryt - title: Er du sikker? - money_field: - label: Beløp - syncing_notice: - syncing: Synkroniserer kontodata... - trend_change: +--- +nb: + shared: + confirm_modal: + accept: Bekreft + body_html: "

Du vil ikke kunne angre denne beslutningen

" + cancel: Avbryt + title: Er du sikker? + money_field: + label: Beløp + exchange_rate_tabs: + calculate_rate_tab: Beregn FX-kurs + convert_tab: Konverter med FX-kurs + destination_amount: Beløp for destinasjon + exchange_rate: Vekslingskurs + exchange_rate_help: Velg hvordan du vil fylle inn beløpet. + syncing_notice: + syncing: Synkroniserer kontodata... + trend_change: no_change: "ingen endring" \ No newline at end of file diff --git a/config/locales/views/shared/nl.yml b/config/locales/views/shared/nl.yml index 569c00e0d..a63b576d0 100644 --- a/config/locales/views/shared/nl.yml +++ b/config/locales/views/shared/nl.yml @@ -8,6 +8,12 @@ nl: title: Weet u het zeker? money_field: label: Bedrag + exchange_rate_tabs: + calculate_rate_tab: FX-tarief berekenen + convert_tab: Converteren met FX-tarief + destination_amount: Doelbedrag + exchange_rate: Wisselkoers + exchange_rate_help: Kies hoe u het bedrag wilt invoeren. syncing_notice: syncing: Accountsgegevens synchroniseren... trend_change: diff --git a/config/locales/views/shared/pt-BR.yml b/config/locales/views/shared/pt-BR.yml index cd14facf5..73fc47727 100644 --- a/config/locales/views/shared/pt-BR.yml +++ b/config/locales/views/shared/pt-BR.yml @@ -8,6 +8,12 @@ pt-BR: title: Tem certeza? money_field: label: Valor + exchange_rate_tabs: + calculate_rate_tab: Calcular taxa de câmbio + convert_tab: Converter com taxa de câmbio + destination_amount: Valor de destino + exchange_rate: Taxa de câmbio + exchange_rate_help: Escolha como inserir o valor. syncing_notice: syncing: Sincronizando dados das contas... trend_change: diff --git a/config/locales/views/shared/ro.yml b/config/locales/views/shared/ro.yml index b97866bd3..9cb591e89 100644 --- a/config/locales/views/shared/ro.yml +++ b/config/locales/views/shared/ro.yml @@ -8,6 +8,12 @@ ro: title: Ești sigur? money_field: label: Sumă + exchange_rate_tabs: + calculate_rate_tab: Calculați rata FX + convert_tab: Convertire cu rata FX + destination_amount: Suma de destinație + exchange_rate: Rata de schimb + exchange_rate_help: Alege cum dorești să introduci suma. syncing_notice: syncing: Se sincronizează datele conturilor... trend_change: diff --git a/config/locales/views/shared/tr.yml b/config/locales/views/shared/tr.yml index 8970dc097..045e211c2 100644 --- a/config/locales/views/shared/tr.yml +++ b/config/locales/views/shared/tr.yml @@ -8,6 +8,12 @@ tr: title: Emin misiniz? money_field: label: Tutar + exchange_rate_tabs: + calculate_rate_tab: Döviz Kuru Hesapla + convert_tab: Döviz Kuru ile Dönüştür + destination_amount: Hedef Tutar + exchange_rate: Döviz Kuru + exchange_rate_help: Tutar girişiniz için yöntemi seçin. syncing_notice: syncing: Hesap verileri senkronize ediliyor... trend_change: diff --git a/config/locales/views/shared/zh-CN.yml b/config/locales/views/shared/zh-CN.yml index d37cc3c3f..a84a48379 100644 --- a/config/locales/views/shared/zh-CN.yml +++ b/config/locales/views/shared/zh-CN.yml @@ -8,6 +8,12 @@ zh-CN: title: 确定要执行此操作？ money_field: label: 金额 + exchange_rate_tabs: + calculate_rate_tab: 计算汇率 + convert_tab: 用汇率转换 + destination_amount: 目标金额 + exchange_rate: 汇率 + exchange_rate_help: 选择如何输入金额。 syncing_notice: syncing: 账户数据同步中... transaction_tabs: diff --git a/config/locales/views/shared/zh-TW.yml b/config/locales/views/shared/zh-TW.yml index a29edfe67..b1e193f22 100644 --- a/config/locales/views/shared/zh-TW.yml +++ b/config/locales/views/shared/zh-TW.yml @@ -8,6 +8,12 @@ zh-TW: title: 您確定嗎？ money_field: label: 金額 + exchange_rate_tabs: + calculate_rate_tab: 計算匯率 + convert_tab: 用匯率轉換 + destination_amount: 目標金額 + exchange_rate: 匯率 + exchange_rate_help: 選擇如何輸入金額。 syncing_notice: syncing: 正在同步帳戶資料... trend_change: diff --git a/config/locales/views/transfers/en.yml b/config/locales/views/transfers/en.yml index 669bae2d3..9116f4f6b 100644 --- a/config/locales/views/transfers/en.yml +++ b/config/locales/views/transfers/en.yml @@ -7,11 +7,19 @@ en: success: Transfer removed form: amount: Amount + calculate_rate_tab: Calculate FX rate + convert_tab: Convert with FX rate date: Date + destination_amount: Destination amount + destination_amount_display: "Destination amount: %{amount}" + exchange_rate: Exchange rate + exchange_rate_display: "Exchange rate: %{rate}" + exchange_rate_help: Choose how to enter the transfer amount. expense: Expense from: From income: Income select_account: Select account + source_amount: Source amount submit: Create transfer to: To transfer: Transfer diff --git a/config/locales/views/transfers/es.yml b/config/locales/views/transfers/es.yml index f16061fa3..419c9b719 100644 --- a/config/locales/views/transfers/es.yml +++ b/config/locales/views/transfers/es.yml @@ -7,11 +7,19 @@ es: success: Transferencia eliminada form: amount: Importe + calculate_rate_tab: Calcular tasa + convert_tab: Convertir con tasa date: Fecha + destination_amount: Importe de destino + destination_amount_display: "Importe de destino: %{amount}" + exchange_rate: Tasa de cambio + exchange_rate_display: "Tasa de cambio: %{rate}" + exchange_rate_help: Elige cómo introducir el importe de la transferencia. expense: Gasto from: Desde income: Ingreso select_account: Seleccionar cuenta + source_amount: Importe de origen submit: Crear transferencia to: Hacia transfer: Transferencia diff --git a/config/routes.rb b/config/routes.rb index 8082c1efe..5f723eed0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -249,6 +249,8 @@ Rails.application.routes.draw do end end + get :exchange_rate, to: "exchange_rates#show" + resources :transfers, only: %i[new create destroy show update] resources :imports, only: %i[index new show create update destroy] do diff --git a/lib/money.rb b/lib/money.rb index ed1e00d48..0d1e73a88 100644 --- a/lib/money.rb +++ b/lib/money.rb @@ -39,16 +39,30 @@ class Money validate! end - def exchange_to(other_currency, date: Date.current, fallback_rate: nil) + # Exchange money to another currency + # Params: + # other_currency: target currency code (e.g. "USD") + # date: date for historical rates (default: Date.current) + # custom_rate: explicit exchange rate to use (skips lookup if provided, including nil check) + # Priority: + # 1. Use custom_rate if explicitly provided (not nil) + # 2. Look up rate via store.find_or_fetch_rate + # 3. Raise ConversionError if no valid rate available + def exchange_to(other_currency, date: Date.current, custom_rate: nil) iso_code = currency.iso_code other_iso_code = Money::Currency.new(other_currency).iso_code if iso_code == other_iso_code self else - exchange_rate = store.find_or_fetch_rate(from: iso_code, to: other_iso_code, date: date)&.rate || fallback_rate + # Use custom rate if provided, otherwise look it up + if custom_rate.present? + exchange_rate = custom_rate.to_d + else + exchange_rate = store.find_or_fetch_rate(from: iso_code, to: other_iso_code, date: date)&.rate + end - raise ConversionError.new(from_currency: iso_code, to_currency: other_iso_code, date: date) unless exchange_rate + raise ConversionError.new(from_currency: iso_code, to_currency: other_iso_code, date: date) unless exchange_rate && exchange_rate > 0 Money.new(amount * exchange_rate, other_iso_code) end diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb index 69724cda0..313c12efe 100644 --- a/test/application_system_test_case.rb +++ b/test/application_system_test_case.rb @@ -65,4 +65,18 @@ class ApplicationSystemTestCase < ActionDispatch::SystemTestCase yield end end + + # Interact with DS::Select custom dropdown components. + # DS::Select renders as a button + listbox — not a native + + <% [ + ["twelve_data", t(".providers.twelve_data"), t(".twelve_data_hint")], + ["yahoo_finance", t(".providers.yahoo_finance"), t(".yahoo_finance_hint")], + ["tiingo", t(".providers.tiingo"), t(".requires_api_key")], + ["eodhd", t(".providers.eodhd"), t(".requires_api_key_eodhd")], + ["alpha_vantage", t(".providers.alpha_vantage"), t(".requires_api_key_alpha_vantage")], + ["mfapi", t(".providers.mfapi"), t(".mfapi_hint")], + ].each do |value, label, hint| %> + + <% end %>
<% end %> +
+ + <% if ENV["EXCHANGE_RATE_PROVIDER"].present? || ENV["SECURITIES_PROVIDERS"].present? || ENV["SECURITIES_PROVIDER"].present? %> +
+
+ <%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5 shrink-0") %> +

+ <%= t(".env_configured_message") %> +

+
+
<% end %>
diff --git a/app/views/settings/hostings/_tiingo_settings.html.erb b/app/views/settings/hostings/_tiingo_settings.html.erb new file mode 100644 index 000000000..af2cf0ff8 --- /dev/null +++ b/app/views/settings/hostings/_tiingo_settings.html.erb @@ -0,0 +1,37 @@ +
+
+

<%= t(".title") %>

+ <% if ENV["TIINGO_API_KEY"].present? %> +

<%= t(".env_configured_message") %>

+ <% else %> +
+ <%= t(".description") %> +
+ <%= t(".show_details") %> +
    +
  1. <%= t(".step_1_html") %>
    2. +
  2. <%= t(".step_2_html") %>
    3. +
  3. <%= t(".step_3") %>
    4. +
+
+
+ <% end %> +
+ + <%= styled_form_with model: Setting.new, + url: settings_hosting_path, + method: :patch, + data: { + controller: "auto-submit-form", + "auto-submit-form-trigger-event-value": "blur" + } do |form| %> + <% has_key = ENV["TIINGO_API_KEY"].present? || Setting.tiingo_api_key.present? %> + <%= form.text_field :tiingo_api_key, + label: t(".label"), + type: "password", + placeholder: t(".placeholder"), + value: has_key ? "********" : "", + disabled: ENV["TIINGO_API_KEY"].present?, + data: { "auto-submit-form-target": "auto" } %> + <% end %> +
diff --git a/app/views/settings/hostings/_twelve_data_settings.html.erb b/app/views/settings/hostings/_twelve_data_settings.html.erb index b6275491a..f5d725753 100644 --- a/app/views/settings/hostings/_twelve_data_settings.html.erb +++ b/app/views/settings/hostings/_twelve_data_settings.html.erb @@ -7,17 +7,11 @@
<%= t(".description") %>
- (show details) + <%= t(".show_details") %>
    -
  1. - Visit twelvedata.com and create a free Twelve Data Developer account. -
    2. -
  2. - Go to the API Keys page. -
    3. -
  3. - Reveal your Secret Key and paste it below. -
    4. +
  4. <%= t(".step_1_html") %>
    5. +
  5. <%= t(".step_2_html") %>
    6. +
  6. <%= t(".step_3") %>
@@ -31,11 +25,12 @@ controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %> + <% has_key = ENV["TWELVE_DATA_API_KEY"].present? || Setting.twelve_data_api_key.present? %> <%= form.text_field :twelve_data_api_key, label: t(".label"), type: "password", placeholder: t(".placeholder"), - value: ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key), + value: has_key ? "********" : "", disabled: ENV["TWELVE_DATA_API_KEY"].present?, container_class: @twelve_data_usage.present? && !@twelve_data_usage.success? ? "border-red-500" : "", data: { "auto-submit-form-target": "auto" } %> diff --git a/app/views/settings/hostings/_yahoo_finance_settings.html.erb b/app/views/settings/hostings/_yahoo_finance_settings.html.erb index 33676f547..2d9c0bb0e 100644 --- a/app/views/settings/hostings/_yahoo_finance_settings.html.erb +++ b/app/views/settings/hostings/_yahoo_finance_settings.html.erb @@ -20,16 +20,12 @@ <%= t(".status_inactive") %>

-
+
- <%= icon("alert-circle", class: "w-5 h-5 text-destructive-600 mt-0.5 shrink-0") %> + <%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5 shrink-0") %>
-

- <%= t(".connection_failed") %> -

-
-

<%= t(".troubleshooting") %>

-
+

<%= t(".connection_failed") %>

+

<%= t(".troubleshooting") %>

diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erb index 354cf86a4..6312f900e 100644 --- a/app/views/settings/hostings/show.html.erb +++ b/app/views/settings/hostings/show.html.erb @@ -17,6 +17,15 @@ <% if @show_twelve_data_settings %> <%= render "settings/hostings/twelve_data_settings" %> <% end %> + <% if @show_tiingo_settings %> + <%= render "settings/hostings/tiingo_settings" %> + <% end %> + <% if @show_eodhd_settings %> + <%= render "settings/hostings/eodhd_settings" %> + <% end %> + <% if @show_alpha_vantage_settings %> + <%= render "settings/hostings/alpha_vantage_settings" %> + <% end %>
<% end %> <%= settings_section title: t(".sync_settings") do %> diff --git a/app/views/shared/_money_field.html.erb b/app/views/shared/_money_field.html.erb index 8ef25151f..386b7ebe6 100644 --- a/app/views/shared/_money_field.html.erb +++ b/app/views/shared/_money_field.html.erb @@ -9,8 +9,8 @@
data-money-field-precision-value="<%= options[:precision] %>"<% end %> - <% if options[:step].present? %>data-money-field-step-value="<%= options[:step] %>"<% end %>> + <% if options[:precision].present? %> data-money-field-precision-value="<%= options[:precision] %>"<% end %> + <% if options[:step].present? %> data-money-field-step-value="<%= options[:step] %>"<% end %>> <% if options[:label_tooltip] %>
<%= form.label options[:label] || t(".label"), class: "form-field__label" do %> diff --git a/app/views/trades/_form.html.erb b/app/views/trades/_form.html.erb index 600dde3b9..31038f259 100644 --- a/app/views/trades/_form.html.erb +++ b/app/views/trades/_form.html.erb @@ -25,7 +25,7 @@ }} %> <% if %w[buy sell].include?(type) %> - <% if Security.provider.present? %> + <% if Security.providers.any? %>
<%= form.combobox :ticker, securities_path(country_code: Current.family.country), diff --git a/app/views/trades/show.html.erb b/app/views/trades/show.html.erb index 7e1c222c7..e581d60e3 100644 --- a/app/views/trades/show.html.erb +++ b/app/views/trades/show.html.erb @@ -8,6 +8,14 @@ <% trade = @entry.trade %> <% dialog.with_body do %> <%= render "entries/protection_indicator", entry: @entry, unlock_path: unlock_trade_path(trade) %> + <% if trade.security&.provider_status == :provider_unavailable %> +
+ <%= render DS::Alert.new( + message: t(".provider_disabled_warning", provider: trade.security.price_provider&.humanize || "Unknown"), + variant: :warning + ) %> +
+ <% end %> <% dialog.with_section(title: t(".details"), open: true) do %>
<%= styled_form_with model: @entry, diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index 071d2d3e3..8366388cb 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -26,7 +26,7 @@ <% end %>
-
+
"> <%= content_tag :div, class: ["flex items-center gap-3 lg:gap-4"] do %> -
+
"> <%# Protection indicator - shows on hover when entry is protected from sync %> <% if entry.protected_from_sync? && !entry.excluded? %> <%= link_to entry_path(entry), diff --git a/app/views/transactions/convert_to_trade.html.erb b/app/views/transactions/convert_to_trade.html.erb index 631a5c9ab..5c0f97270 100644 --- a/app/views/transactions/convert_to_trade.html.erb +++ b/app/views/transactions/convert_to_trade.html.erb @@ -33,7 +33,7 @@
<%= f.label :security_id, t(".security_label"), class: "font-medium text-sm text-primary block" %> - <% if Security.provider.present? %> + <% if Security.providers.any? %> <%# Always use searchable combobox when provider available - prevents picking wrong similar tickers %>
<%= f.combobox :ticker, @@ -157,7 +157,7 @@
<%# Only show exchange field when no provider - combobox selections already include exchange %> - <% unless Security.provider.present? %> + <% unless Security.providers.any? %>
<%= f.label :exchange_operating_mic, t(".exchange_label"), class: "font-medium text-sm text-primary block" %> <%= f.text_field :exchange_operating_mic, diff --git a/config/locales/views/holdings/en.yml b/config/locales/views/holdings/en.yml index 4fe40c9e6..6ccfc36c6 100644 --- a/config/locales/views/holdings/en.yml +++ b/config/locales/views/holdings/en.yml @@ -71,6 +71,10 @@ en: search_security_placeholder: Search by ticker or name cancel: Cancel remap_security: Save + provider_disabled_warning: "Price updates paused — %{provider} provider is disabled. Switch to another provider below or re-enable it in Settings." + switch_provider_label: Switch provider + switch_provider_description: "%{provider} is disabled. Search for this security from another enabled provider." + switch_provider_button: Switch no_security_provider: Security provider not configured. Cannot search for securities. security_remapped_label: Security remapped provider_sent: "Provider sent: %{ticker}" diff --git a/config/locales/views/securities/en.yml b/config/locales/views/securities/en.yml index 078dfd0d0..6df2a3102 100644 --- a/config/locales/views/securities/en.yml +++ b/config/locales/views/securities/en.yml @@ -4,3 +4,10 @@ en: combobox: display: "%{symbol} - %{name} (%{exchange})" exchange_label: "%{symbol} (%{exchange})" + providers: + twelve_data: Twelve Data + yahoo_finance: Yahoo Finance + tiingo: Tiingo + eodhd: EODHD + alpha_vantage: Alpha Vantage + mfapi: MFAPI.in diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index 814d0b13c..4446e3353 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -31,14 +31,25 @@ en: title: Clear data cache? body: Are you sure you want to clear the data cache? This will remove all exchange rates, security prices, account balances, and other data. This action cannot be undone. provider_selection: - title: Provider Selection - description: Choose which service to use for fetching exchange rates and security prices. Yahoo Finance is free and doesn't require an API key. Twelve Data requires a free API key but may offer more data coverage. + exchange_rate_title: Exchange Rate Provider + exchange_rate_description: Select a single provider for fetching currency exchange rates. exchange_rate_provider_label: Exchange Rate Provider - securities_provider_label: Securities (Stock Prices) Provider - env_configured_message: Provider selection is disabled because environment variables (EXCHANGE_RATE_PROVIDER or SECURITIES_PROVIDER) are set. To enable selection here, remove these environment variables from your configuration. + securities_title: Securities Providers + securities_description: Enable one or more providers for fetching stock, ETF, and mutual fund prices. When searching, all enabled providers are queried and results are merged. + env_configured_message: Provider selection is disabled because environment variables are set. To enable selection here, remove these environment variables from your configuration. + twelve_data_hint: requires API key, 800 credits/day + yahoo_finance_hint: free, no API key needed + requires_api_key: requires API key + requires_api_key_eodhd: requires API key, 20 calls/day limit + requires_api_key_alpha_vantage: requires API key, 25 calls/day limit + mfapi_hint: free, no API key -- Indian mutual funds only providers: twelve_data: Twelve Data yahoo_finance: Yahoo Finance + tiingo: Tiingo + eodhd: EODHD + alpha_vantage: Alpha Vantage + mfapi: MFAPI.in assistant_settings: title: AI Assistant description: Choose how the chat assistant responds. Builtin uses your configured LLM provider directly. External delegates to a remote AI agent that can call back to Sure's financial tools via MCP. @@ -95,12 +106,48 @@ en: status_inactive: Yahoo Finance connection failed connection_failed: Unable to connect to Yahoo Finance troubleshooting: Check your internet connection and firewall settings. Yahoo Finance may be temporarily unavailable. + tiingo_settings: + title: Tiingo + description: Enter the API token provided by Tiingo. Free tier supports 50 unique symbols per hour with 30+ years of historical data. + env_configured_message: Successfully configured through the TIINGO_API_KEY environment variable. + label: API Token + placeholder: Enter your Tiingo API token here + show_details: "(show details)" + step_1_html: 'Visit tiingo.com and create a free account.' + step_2_html: 'Go to the API Token page.' + step_3: Copy your API token and paste it below. + eodhd_settings: + title: EODHD + description: Enter the API token provided by EODHD. Supports EU ETFs on LSE, XETRA, and other international exchanges. + env_configured_message: Successfully configured through the EODHD_API_KEY environment variable. + label: API Token + placeholder: Enter your EODHD API token here + show_details: "(show details)" + step_1_html: 'Visit eodhd.com and create a free account.' + step_2_html: 'Go to your Dashboard to find your API token.' + step_3: Copy your API token and paste it below. + rate_limit_warning: "EODHD free tier is limited to 20 API calls per day. Best used as a supplementary provider for EU ETFs not available from other providers." + alpha_vantage_settings: + title: Alpha Vantage + description: Enter the API key from Alpha Vantage. Supports EU ETFs on London Stock Exchange, XETRA, and other exchanges. + env_configured_message: Successfully configured through the ALPHA_VANTAGE_API_KEY environment variable. + label: API Key + placeholder: Enter your Alpha Vantage API key here + show_details: "(show details)" + step_1_html: 'Visit alphavantage.co and claim your free API key.' + step_2: Copy the API key and paste it below. + rate_limit_warning: "Alpha Vantage free tier is limited to 25 API calls per day. Best used as a supplementary provider for EU ETFs not available from other providers." + no_health_check_note: "Connection health check is unavailable for this provider due to the strict rate limit." twelve_data_settings: api_calls_used: "%{used} / %{limit} API daily calls used (%{percentage})" description: Enter the API key provided by Twelve Data env_configured_message: Successfully configured through the TWELVE_DATA_API_KEY environment variable. label: API Key placeholder: Enter your API key here + show_details: "(show details)" + step_1_html: 'Visit twelvedata.com and create a free Twelve Data Developer account.' + step_2_html: 'Go to the API Keys page.' + step_3: Reveal your Secret Key and paste it below. plan: "%{plan} plan" plan_upgrade_warning_title: Some tickers require a paid plan plan_upgrade_warning_description: The following tickers in your portfolio cannot sync prices with your current Twelve Data plan. diff --git a/config/locales/views/trades/en.yml b/config/locales/views/trades/en.yml index b2257c26f..f8e1fb418 100644 --- a/config/locales/views/trades/en.yml +++ b/config/locales/views/trades/en.yml @@ -45,6 +45,7 @@ en: delete_subtitle: This action cannot be undone delete_title: Delete Trade details: Details + provider_disabled_warning: "Price updates paused — %{provider} provider is disabled. Re-enable it in Settings or remap the holding to another provider." exclude_subtitle: This trade will not be included in reports and calculations exclude_title: Exclude from analytics no_category: No category diff --git a/db/migrate/20260408120000_add_price_provider_to_securities.rb b/db/migrate/20260408120000_add_price_provider_to_securities.rb new file mode 100644 index 000000000..7e6da11b9 --- /dev/null +++ b/db/migrate/20260408120000_add_price_provider_to_securities.rb @@ -0,0 +1,8 @@ +class AddPriceProviderToSecurities < ActiveRecord::Migration[7.2] + disable_ddl_transaction! + + def change + add_column :securities, :price_provider, :string + add_index :securities, :price_provider, algorithm: :concurrently + end +end diff --git a/db/migrate/20260408151837_add_offline_reason_to_securities.rb b/db/migrate/20260408151837_add_offline_reason_to_securities.rb new file mode 100644 index 000000000..2e5edfe02 --- /dev/null +++ b/db/migrate/20260408151837_add_offline_reason_to_securities.rb @@ -0,0 +1,8 @@ +class AddOfflineReasonToSecurities < ActiveRecord::Migration[7.2] + disable_ddl_transaction! + + def change + add_column :securities, :offline_reason, :string + add_index :securities, [ :price_provider, :offline_reason ], algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 3e83968f7..24b067211 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_03_30_050801) do +ActiveRecord::Schema[7.2].define(version: 2026_04_08_151837) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -1215,10 +1215,14 @@ ActiveRecord::Schema[7.2].define(version: 2026_03_30_050801) do t.datetime "last_health_check_at" t.string "website_url" t.string "kind", default: "standard", null: false + t.string "price_provider" + t.string "offline_reason" t.index "upper((ticker)::text), COALESCE(upper((exchange_operating_mic)::text), ''::text)", name: "index_securities_on_ticker_and_exchange_operating_mic_unique", unique: true t.index ["country_code"], name: "index_securities_on_country_code" t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic" t.index ["kind"], name: "index_securities_on_kind" + t.index ["price_provider", "offline_reason"], name: "index_securities_on_price_provider_and_offline_reason" + t.index ["price_provider"], name: "index_securities_on_price_provider" t.check_constraint "kind::text = ANY (ARRAY['standard'::character varying, 'cash'::character varying]::text[])", name: "chk_securities_kind" end diff --git a/test/controllers/settings/hostings_controller_test.rb b/test/controllers/settings/hostings_controller_test.rb index f4706c07c..c38585bc3 100644 --- a/test/controllers/settings/hostings_controller_test.rb +++ b/test/controllers/settings/hostings_controller_test.rb @@ -247,4 +247,71 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest assert_equal I18n.t("settings.hostings.not_authorized"), flash[:alert] end end + + # --- Securities provider toggle --- + + test "can update securities providers" do + with_self_hosting do + patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "yahoo_finance" ] } } + + assert_redirected_to settings_hosting_url + assert_equal "twelve_data,yahoo_finance", Setting.securities_providers + end + ensure + Setting.securities_providers = "" + end + + test "filters out invalid provider names" do + with_self_hosting do + patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "fake_provider", "hacked" ] } } + + assert_redirected_to settings_hosting_url + # Only valid providers are stored + enabled = Setting.enabled_securities_providers + assert_includes enabled, "twelve_data" + refute_includes enabled, "fake_provider" + refute_includes enabled, "hacked" + end + ensure + Setting.securities_providers = "" + end + + test "removing a provider marks linked securities offline" do + with_self_hosting do + security = Security.create!(ticker: "CSPX", exchange_operating_mic: "XLON", price_provider: "tiingo", offline: false) + + # First enable tiingo + Setting.securities_providers = "twelve_data,tiingo" + + # Then remove tiingo + patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data" ] } } + + security.reload + assert security.offline?, "Security should be marked offline when its provider is removed" + assert_equal "provider_disabled", security.offline_reason + end + ensure + Setting.securities_providers = "" + end + + test "re-adding a provider brings securities back online" do + with_self_hosting do + security = Security.create!( + ticker: "CSPX2", exchange_operating_mic: "XLON", + price_provider: "tiingo", offline: true, offline_reason: "provider_disabled" + ) + + # Start without tiingo + Setting.securities_providers = "twelve_data" + + # Re-add tiingo + patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "tiingo" ] } } + + security.reload + refute security.offline?, "Security should come back online when its provider is re-added" + assert_nil security.offline_reason + end + ensure + Setting.securities_providers = "" + end end diff --git a/test/models/security/health_checker_test.rb b/test/models/security/health_checker_test.rb index a8efe8f7d..aef6a6c7a 100644 --- a/test/models/security/health_checker_test.rb +++ b/test/models/security/health_checker_test.rb @@ -11,7 +11,7 @@ class Security::HealthCheckerTest < ActiveSupport::TestCase Security.delete_all @provider = mock - Security.stubs(:provider).returns(@provider) + Security.any_instance.stubs(:price_data_provider).returns(@provider) # Brand new, no health check has been run yet @new_security = Security.create!( diff --git a/test/models/security/price_test.rb b/test/models/security/price_test.rb index 6e8e49da1..8214831ac 100644 --- a/test/models/security/price_test.rb +++ b/test/models/security/price_test.rb @@ -6,9 +6,8 @@ class Security::PriceTest < ActiveSupport::TestCase setup do @provider = mock - Security.stubs(:provider).returns(@provider) - @security = securities(:aapl) + @security.stubs(:price_data_provider).returns(@provider) end test "finds single security price in DB" do diff --git a/test/models/security/provided_test.rb b/test/models/security/provided_test.rb new file mode 100644 index 000000000..521a249ba --- /dev/null +++ b/test/models/security/provided_test.rb @@ -0,0 +1,295 @@ +require "test_helper" + +class Security::ProvidedTest < ActiveSupport::TestCase + include ProviderTestHelper + + setup do + @security = securities(:aapl) + end + + # --- search_provider --- + + test "search_provider returns results from multiple providers" do + provider_a = mock("provider_a") + provider_b = mock("provider_b") + + result_a = Provider::SecurityConcept::Security.new( + symbol: "AAPL", name: "Apple Inc", logo_url: nil, + exchange_operating_mic: "XNAS", country_code: "US", currency: "USD" + ) + result_b = Provider::SecurityConcept::Security.new( + symbol: "AAPL", name: "Apple Inc", logo_url: nil, + exchange_operating_mic: "XNAS", country_code: "US", currency: "USD" + ) + + provider_a.stubs(:class).returns(Provider::TwelveData) + provider_b.stubs(:class).returns(Provider::YahooFinance) + + provider_a.expects(:search_securities).with("AAPL").returns( + provider_success_response([ result_a ]) + ) + provider_b.expects(:search_securities).with("AAPL").returns( + provider_success_response([ result_b ]) + ) + + Security.stubs(:providers).returns([ provider_a, provider_b ]) + + results = Security.search_provider("AAPL") + + # Same ticker+exchange but different providers → both appear (dedup includes provider key) + assert_equal 2, results.size + assert results.all? { |s| s.ticker == "AAPL" } + providers_in_results = results.map(&:price_provider).sort + assert_includes providers_in_results, "twelve_data" + assert_includes providers_in_results, "yahoo_finance" + end + + test "search_provider deduplicates same ticker+exchange+provider" do + provider = mock("provider") + provider.stubs(:class).returns(Provider::TwelveData) + + dup_result = Provider::SecurityConcept::Security.new( + symbol: "MSFT", name: "Microsoft", logo_url: nil, + exchange_operating_mic: "XNAS", country_code: "US", currency: "USD" + ) + + provider.expects(:search_securities).with("MSFT").returns( + provider_success_response([ dup_result, dup_result ]) + ) + + Security.stubs(:providers).returns([ provider ]) + + results = Security.search_provider("MSFT") + assert_equal 1, results.size + end + + test "search_provider returns empty array for blank symbol" do + assert_equal [], Security.search_provider("") + assert_equal [], Security.search_provider(nil) + end + + test "search_provider returns empty array when no providers configured" do + Security.stubs(:providers).returns([]) + assert_equal [], Security.search_provider("AAPL") + end + + test "search_provider keeps fast provider results when slow provider times out" do + fast_provider = mock("fast_provider") + slow_provider = mock("slow_provider") + + fast_provider.stubs(:class).returns(Provider::TwelveData) + slow_provider.stubs(:class).returns(Provider::YahooFinance) + + fast_result = Provider::SecurityConcept::Security.new( + symbol: "SPY", name: "SPDR S&P 500", logo_url: nil, + exchange_operating_mic: "XNAS", country_code: "US", currency: "USD" + ) + + fast_provider.expects(:search_securities).with("SPY").returns( + provider_success_response([ fast_result ]) + ) + slow_provider.expects(:search_securities).with("SPY").returns( + provider_success_response([]) + ) + + Security.stubs(:providers).returns([ fast_provider, slow_provider ]) + + results = Security.search_provider("SPY") + + assert results.size >= 1, "Fast provider results should be returned even if slow provider returns nothing" + assert_equal "SPY", results.first.ticker + end + + test "search_provider handles provider error gracefully" do + good_provider = mock("good_provider") + bad_provider = mock("bad_provider") + + good_provider.stubs(:class).returns(Provider::TwelveData) + bad_provider.stubs(:class).returns(Provider::YahooFinance) + + good_result = Provider::SecurityConcept::Security.new( + symbol: "GOOG", name: "Alphabet", logo_url: nil, + exchange_operating_mic: "XNAS", country_code: "US", currency: "USD" + ) + + good_provider.expects(:search_securities).with("GOOG").returns( + provider_success_response([ good_result ]) + ) + bad_provider.expects(:search_securities).with("GOOG").raises(StandardError, "API down") + + Security.stubs(:providers).returns([ good_provider, bad_provider ]) + + results = Security.search_provider("GOOG") + + assert_equal 1, results.size + assert_equal "GOOG", results.first.ticker + end + + # --- price_data_provider --- + + test "price_data_provider returns assigned provider" do + provider = mock("tiingo_provider") + Security.stubs(:provider_for).with("tiingo").returns(provider) + + @security.update!(price_provider: "tiingo") + + assert_equal provider, @security.price_data_provider + end + + test "price_data_provider returns nil when assigned provider is disabled" do + Security.stubs(:provider_for).with("tiingo").returns(nil) + + @security.update!(price_provider: "tiingo") + + assert_nil @security.price_data_provider + end + + test "price_data_provider falls back to first provider when none assigned" do + fallback_provider = mock("fallback") + Security.stubs(:providers).returns([ fallback_provider ]) + + @security.update!(price_provider: nil) + + assert_equal fallback_provider, @security.price_data_provider + end + + # --- provider_status --- + + test "provider_status returns provider_unavailable when assigned provider disabled" do + Security.stubs(:provider_for).with("tiingo").returns(nil) + + @security.update!(price_provider: "tiingo") + + assert_equal :provider_unavailable, @security.provider_status + end + + test "provider_status returns ok for healthy security" do + provider = mock("provider") + Security.stubs(:provider_for).with("twelve_data").returns(provider) + + @security.update!(price_provider: "twelve_data", offline: false, failed_fetch_count: 0) + + assert_equal :ok, @security.provider_status + end + + # --- rank_search_results --- + + # Helper to build unsaved Security objects for ranking tests + def build_result(ticker:, name: nil, country_code: nil, exchange_operating_mic: nil) + Security.new( + ticker: ticker, + name: name || ticker, + country_code: country_code, + exchange_operating_mic: exchange_operating_mic + ) + end + + def rank(results, query, country_code = nil) + Security.send(:rank_search_results, results, query, country_code) + end + + test "ranking: AAPL exact match ranks above AAPL-prefixed and unrelated" do + results = [ + build_result(ticker: "AAPLX", name: "Some AAPL Fund"), + build_result(ticker: "AAPL", name: "Apple Inc", country_code: "US", exchange_operating_mic: "XNAS"), + build_result(ticker: "AAPL", name: "Apple Inc", country_code: "GB", exchange_operating_mic: "XLON"), + build_result(ticker: "AAPLD", name: "AAPL Dividend ETF") + ] + + ranked = rank(results, "AAPL", "US") + + # Exact matches first, US preferred over GB + assert_equal "AAPL", ranked[0].ticker + assert_equal "US", ranked[0].country_code + assert_equal "AAPL", ranked[1].ticker + assert_equal "GB", ranked[1].country_code + # Prefix matches after + assert ranked[2..].all? { |s| s.ticker.start_with?("AAPL") && s.ticker != "AAPL" } + end + + test "ranking: Apple name search surfaces Apple Inc above unrelated" do + results = [ + build_result(ticker: "PINEAPPLE", name: "Pineapple Corp"), + build_result(ticker: "AAPL", name: "Apple Inc", country_code: "US"), + build_result(ticker: "APLE", name: "Apple Hospitality REIT"), + build_result(ticker: "APPL", name: "Appell Petroleum") + ] + + ranked = rank(results, "Apple", "US") + + # No ticker matches "APPLE", so all fall to name-contains or worse. + # "Apple Inc" and "Apple Hospitality" and "Pineapple" contain "APPLE" in name. + # "Appell Petroleum" does not contain "APPLE". + # Among name matches, alphabetical ticker breaks ties. + name_matches = ranked.select { |s| s.name.upcase.include?("APPLE") } + non_matches = ranked.reject { |s| s.name.upcase.include?("APPLE") } + assert name_matches.size >= 2 + assert_equal non_matches, ranked.last(non_matches.size) + end + + test "ranking: SPX exact match first, then SPX-prefixed" do + results = [ + build_result(ticker: "SPXL", name: "Direxion Daily S&P 500 Bull 3X"), + build_result(ticker: "SPXS", name: "Direxion Daily S&P 500 Bear 3X"), + build_result(ticker: "SPX", name: "S&P 500 Index", country_code: "US"), + build_result(ticker: "SPXU", name: "ProShares UltraPro Short S&P 500") + ] + + ranked = rank(results, "SPX", "US") + + assert_equal "SPX", ranked[0].ticker, "Exact match should be first" + assert ranked[1..].all? { |s| s.ticker.start_with?("SPX") } + end + + test "ranking: VTTI exact match first regardless of country" do + results = [ + build_result(ticker: "VTI", name: "Vanguard Total Stock Market ETF", country_code: "US"), + build_result(ticker: "VTTI", name: "VTTI Energy Partners", country_code: "US"), + build_result(ticker: "VTTIX", name: "Vanguard Target 2060 Fund") + ] + + ranked = rank(results, "VTTI", "US") + + assert_equal "VTTI", ranked[0].ticker, "Exact match should be first" + assert_equal "VTTIX", ranked[1].ticker, "Prefix match second" + assert_equal "VTI", ranked[2].ticker, "Non-matching ticker last" + end + + test "ranking: iShares S&P multi-word query is contiguous substring match" do + results = [ + build_result(ticker: "IVV", name: "iShares S&P 500 ETF", country_code: "US"), + build_result(ticker: "CSPX", name: "iShares Core S&P 500 UCITS ETF", country_code: "GB"), + build_result(ticker: "IJH", name: "iShares S&P Mid-Cap ETF", country_code: "US"), + build_result(ticker: "UNRELATED", name: "Something Else Corp") + ] + + ranked = rank(results, "iShares S&P", "US") + + # Only names containing the exact substring "iShares S&P" match tier 2. + # "iShares Core S&P" does NOT match (word "Core" breaks contiguity). + contiguous_matches = ranked.select { |s| s.name.upcase.include?("ISHARES S&P") } + assert_equal 2, contiguous_matches.size, "Only IVV and IJH contain the exact substring" + # US contiguous matches should come first + assert_equal "IJH", ranked[0].ticker # US, name match, alphabetically before IVV? No... + assert_includes [ "IVV", "IJH" ], ranked[0].ticker + assert_includes [ "IVV", "IJH" ], ranked[1].ticker + # Non-contiguous and unrelated should be last + assert_equal "UNRELATED", ranked.last.ticker + end + + test "ranking: tesla name search finds TSLA" do + results = [ + build_result(ticker: "TSLA", name: "Tesla Inc", country_code: "US"), + build_result(ticker: "TSLA", name: "Tesla Inc", country_code: "DE"), + build_result(ticker: "TL0", name: "Tesla Inc", country_code: "DE", exchange_operating_mic: "XETR"), + build_result(ticker: "TELSA", name: "Telsa Mining Ltd") + ] + + ranked = rank(results, "tesla", "US") + + # No ticker matches "TESLA", so all go to name matching + # "Tesla Inc" contains "TESLA" → tier 2, US preferred + assert_equal "TSLA", ranked[0].ticker + assert_equal "US", ranked[0].country_code, "US Tesla should rank first for US user" + end +end diff --git a/test/models/security/resolver_test.rb b/test/models/security/resolver_test.rb index 1a557f221..fee3b9fd7 100644 --- a/test/models/security/resolver_test.rb +++ b/test/models/security/resolver_test.rb @@ -1,11 +1,6 @@ require "test_helper" class Security::ResolverTest < ActiveSupport::TestCase - setup do - @provider = mock - Security.stubs(:provider).returns(@provider) - end - test "resolves DB security" do # Given an existing security in the DB that exactly matches the lookup params db_security = Security.create!(ticker: "TSLA", exchange_operating_mic: "XNAS", country_code: "US") @@ -75,4 +70,73 @@ class Security::ResolverTest < ActiveSupport::TestCase assert_raises(ArgumentError) { Security::Resolver.new(nil).resolve } assert_raises(ArgumentError) { Security::Resolver.new("").resolve } end + + test "persists explicit price_provider on DB match" do + db_security = Security.create!(ticker: "CSPX", exchange_operating_mic: "XLON", country_code: "GB") + + Security.expects(:search_provider).never + Setting.stubs(:enabled_securities_providers).returns([ "tiingo" ]) + + resolved = Security::Resolver.new( + "CSPX", + exchange_operating_mic: "XLON", + country_code: "GB", + price_provider: "tiingo" + ).resolve + + assert_equal db_security, resolved + assert_equal "tiingo", resolved.reload.price_provider + end + + test "persists price_provider on provider match" do + match = Security.new(ticker: "VWCE", exchange_operating_mic: "XETR", country_code: "DE", price_provider: "eodhd") + + Security.expects(:search_provider) + .with("VWCE", exchange_operating_mic: "XETR") + .returns([ match ]) + + Setting.stubs(:enabled_securities_providers).returns([ "eodhd" ]) + + resolved = Security::Resolver.new( + "VWCE", + exchange_operating_mic: "XETR", + price_provider: "eodhd" + ).resolve + + assert resolved.persisted? + assert_equal "eodhd", resolved.price_provider + end + + test "rejects unknown price_provider" do + db_security = Security.create!(ticker: "AAPL2", exchange_operating_mic: "XNAS", country_code: "US") + + Security.expects(:search_provider).never + + resolved = Security::Resolver.new( + "AAPL2", + exchange_operating_mic: "XNAS", + country_code: "US", + price_provider: "fake_provider" + ).resolve + + assert_equal db_security, resolved + assert_nil resolved.reload.price_provider, "Unknown providers should be rejected" + end + + test "rejects disabled price_provider" do + db_security = Security.create!(ticker: "GOOG2", exchange_operating_mic: "XNAS", country_code: "US") + + Security.expects(:search_provider).never + Setting.stubs(:enabled_securities_providers).returns([ "twelve_data" ]) + + resolved = Security::Resolver.new( + "GOOG2", + exchange_operating_mic: "XNAS", + country_code: "US", + price_provider: "tiingo" + ).resolve + + assert_equal db_security, resolved + assert_nil resolved.reload.price_provider, "Disabled providers should be rejected" + end end From 814f2df5704cec135b524df35a468f02d58cf143 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 21:49:17 +0000 Subject: [PATCH 012/277] Bump version to next iteration after v0.7.0-alpha.3 release --- charts/sure/Chart.yaml | 4 ++-- config/initializers/version.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sure/Chart.yaml b/charts/sure/Chart.yaml index cee4ff9b6..edb5a8983 100644 --- a/charts/sure/Chart.yaml +++ b/charts/sure/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: sure description: Official Helm chart for deploying the Sure Rails app (web + Sidekiq) on Kubernetes with optional HA PostgreSQL (CloudNativePG) and Redis. type: application -version: 0.7.0-alpha.3 -appVersion: "0.7.0-alpha.3" +version: 0.7.0-alpha.4 +appVersion: "0.7.0-alpha.4" kubeVersion: ">=1.25.0-0" diff --git a/config/initializers/version.rb b/config/initializers/version.rb index fa7f79b90..1d009d3d1 100644 --- a/config/initializers/version.rb +++ b/config/initializers/version.rb @@ -16,7 +16,7 @@ module Sure private def semver - "0.7.0-alpha.3" + "0.7.0-alpha.4" end end end From 6551aaee0faeff7edadbd5f47097b4618dfbdcae Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 10 Apr 2026 15:40:13 +0200 Subject: [PATCH 013/277] fix(binance): fix hmac signature by using same parameter order in request and sign (#1425) --- app/models/provider/binance.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/models/provider/binance.rb b/app/models/provider/binance.rb index 498084882..d1b8c018c 100644 --- a/app/models/provider/binance.rb +++ b/app/models/provider/binance.rb @@ -91,11 +91,11 @@ class Provider::Binance def signed_get(path, extra_params: {}) params = timestamp_params.merge(extra_params) - params["signature"] = sign(params) + query_string = URI.encode_www_form(params.sort) response = self.class.get( path, - query: params, + query: "#{query_string}&signature=#{sign(query_string)}", headers: auth_headers ) @@ -106,9 +106,10 @@ class Provider::Binance { "timestamp" => (Time.current.to_f * 1000).to_i.to_s, "recvWindow" => "5000" } end - # HMAC-SHA256 of the query string + # HMAC-SHA256 of the query string. + # Accepts either a Hash of params or a pre-built query string. def sign(params) - query_string = URI.encode_www_form(params.sort) + query_string = params.is_a?(Hash) ? URI.encode_www_form(params.sort) : params OpenSSL::HMAC.hexdigest("sha256", api_secret, query_string) end From 0aca297e9cd152286444240e389555a625218979 Mon Sep 17 00:00:00 2001 From: soky srm Date: Fri, 10 Apr 2026 15:43:22 +0200 Subject: [PATCH 014/277] Add binance security provider for crypto (#1424) * Binance as securities provider * Disable twelve data crypto results * Add logo support and new currency pairs * FIX importer fallback * Add price clamping and optiimize retrieval * Review * Update adding-a-securities-provider.md * day gap miss fix * New fixes * Brandfetch doesn't support crypto. add new CDN * Update _investment_performance.html.erb --- AGENTS.md | 4 + app/models/provider/binance_public.rb | 320 +++++++++ app/models/provider/registry.rb | 6 +- app/models/provider/twelve_data.rb | 12 +- app/models/security.rb | 42 +- app/models/security/price/importer.rb | 89 ++- app/models/security/resolver.rb | 15 +- app/views/holdings/_holding.html.erb | 6 +- app/views/holdings/show.html.erb | 16 +- .../reports/_investment_performance.html.erb | 6 +- .../hostings/_provider_selection.html.erb | 1 + config/exchanges.yml | 5 + config/locales/views/holdings/en.yml | 1 + config/locales/views/securities/en.yml | 1 + config/locales/views/settings/hostings/en.yml | 2 + ...d_first_provider_price_on_to_securities.rb | 5 + db/schema.rb | 3 +- .../adding-a-securities-provider.md | 493 ++++++++++++++ test/models/provider/binance_public_test.rb | 608 ++++++++++++++++++ test/models/provider/twelve_data_test.rb | 71 ++ test/models/security/price/importer_test.rb | 239 +++++++ test/models/security/resolver_test.rb | 69 ++ test/models/security_test.rb | 102 +++ 23 files changed, 2091 insertions(+), 25 deletions(-) create mode 100644 app/models/provider/binance_public.rb create mode 100644 db/migrate/20260410114435_add_first_provider_price_on_to_securities.rb create mode 100644 docs/llm-guides/adding-a-securities-provider.md create mode 100644 test/models/provider/binance_public_test.rb diff --git a/AGENTS.md b/AGENTS.md index d3e52c168..30637862d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,10 @@ When adding or modifying API endpoints in `app/controllers/api/v1/`, you **MUST* ### Post-commit API consistency (LLM checklist) After every API endpoint commit, ensure: (1) **Minitest** behavioral coverage in `test/controllers/api/v1/{resource}_controller_test.rb` (no behavioral assertions in rswag); (2) **rswag** remains docs-only (no `expect`/`assert_*` in `spec/requests/api/v1/`); (3) **rswag auth** uses the same API key pattern everywhere (`X-Api-Key`, not OAuth/Bearer). Full checklist: [.cursor/rules/api-endpoint-consistency.mdc](.cursor/rules/api-endpoint-consistency.mdc). +## Securities Providers + +If you need to add a new securities price provider (Tiingo, EODHD, Binance-style crypto, etc.), see [adding-a-securities-provider.md](./docs/llm-guides/adding-a-securities-provider.md) for the full walkthrough — provider class, registry wiring, MIC handling, settings UI, locales, and tests. + ## Providers: Pending Transactions and FX Metadata (SimpleFIN/Plaid/Lunchflow) - Pending detection diff --git a/app/models/provider/binance_public.rb b/app/models/provider/binance_public.rb new file mode 100644 index 000000000..37c263f38 --- /dev/null +++ b/app/models/provider/binance_public.rb @@ -0,0 +1,320 @@ +class Provider::BinancePublic < Provider + include SecurityConcept, RateLimitable + extend SslConfigurable + + Error = Class.new(Provider::Error) + InvalidSecurityPriceError = Class.new(Error) + RateLimitError = Class.new(Error) + + MIN_REQUEST_INTERVAL = 0.1 + + # Binance's official ISO 10383 operating MIC (assigned Jan 2026, country AE). + # Crypto is not tied to a national jurisdiction, so we intentionally do NOT + # propagate the ISO-assigned country code to search results — the resolver + # treats a nil candidate country as a wildcard, letting any family resolve + # a Binance pick regardless of their own country. + BINANCE_MIC = "BNCX".freeze + + # Quote assets we expose in search results. Order = preference when multiple + # quote variants exist for the same base asset. USDT is Binance's dominant + # dollar quote and is surfaced to users as USD. GBP is absent because + # Binance has zero GBP trading pairs today; GBP-family users fall back to + # USDT->USD via the app's FX conversion, same as HUF/CZK/PLN users. + SUPPORTED_QUOTES = %w[USDT EUR JPY BRL TRY].freeze + + # Binance quote asset -> user-facing currency & ticker suffix. + QUOTE_TO_CURRENCY = { + "USDT" => "USD", + "EUR" => "EUR", + "JPY" => "JPY", + "BRL" => "BRL", + "TRY" => "TRY" + }.freeze + + # Per-asset logo PNGs served via jsDelivr from a GitHub repo that tracks the + # full Binance-listed asset set. We originally used bin.bnbstatic.com directly + # — Binance's own CDN — but that host enforces Referer-based hotlink + # protection at CloudFront: any request with a non-Binance Referer returns + # 403. A server-side HEAD from Faraday (no Referer) succeeds, which masked + # the breakage until the URL hit an actual tag in the browser. jsDelivr + # is CORS-open and hotlink-friendly, so the URL we persist is the URL the + # browser can actually load. File names are uppercase PNGs (BTC.png, ETH.png). + LOGO_CDN_BASE = "https://cdn.jsdelivr.net/gh/lindomar-oliveira/binance-data-plus/assets/img".freeze + + KLINE_MAX_LIMIT = 1000 + MS_PER_DAY = 24 * 60 * 60 * 1000 + SEARCH_LIMIT = 25 + + def initialize + # No API key required — public market data only. + end + + def healthy? + with_provider_response do + client.get("#{base_url}/api/v3/ping") + true + end + end + + def usage + with_provider_response do + UsageData.new(used: nil, limit: nil, utilization: nil, plan: "Free (no key required)") + end + end + + # ================================ + # Securities + # ================================ + + def search_securities(symbol, country_code: nil, exchange_operating_mic: nil) + with_provider_response do + query = symbol.to_s.strip.upcase + next [] if query.empty? + + symbols = exchange_info_symbols + + matches = symbols.select do |s| + base = s["baseAsset"].to_s.upcase + quote = s["quoteAsset"].to_s.upcase + symbol = s["symbol"].to_s.upcase + + next false unless SUPPORTED_QUOTES.include?(quote) + + # Match on either the base asset (so "BTC" surfaces every BTC pair) or + # the full Binance pair symbol (so users pasting their own portfolio + # ticker like "BTCEUR" or "BTCUSD" — which prefixes Binance's raw + # "BTCUSDT" — also hit a result). + base.include?(query) || symbol == query || symbol.start_with?(query) + end + + ranked = matches.sort_by do |s| + base = s["baseAsset"].to_s.upcase + quote = s["quoteAsset"].to_s.upcase + symbol = s["symbol"].to_s.upcase + quote_index = SUPPORTED_QUOTES.index(quote) || 99 + + relevance = if symbol == query + 0 # exact full-ticker match — highest priority + elsif symbol.start_with?(query) + 1 # ticker prefix match (e.g. "BTCUSD" against "BTCUSDT") + elsif base == query + 2 # exact base-asset match (e.g. "BTC") + elsif base.start_with?(query) + 3 + else + 4 + end + + [ relevance, quote_index, base ] + end + + ranked.first(SEARCH_LIMIT).map do |s| + base = s["baseAsset"].to_s.upcase + quote = s["quoteAsset"].to_s.upcase + display_currency = QUOTE_TO_CURRENCY[quote] + + Security.new( + symbol: "#{base}#{display_currency}", + name: base, + logo_url: "#{LOGO_CDN_BASE}/#{base}.png", + exchange_operating_mic: BINANCE_MIC, + country_code: nil, + currency: display_currency + ) + end + end + end + + def fetch_security_info(symbol:, exchange_operating_mic:) + with_provider_response do + parsed = parse_ticker(symbol) + raise Error, "Unsupported Binance ticker: #{symbol}" if parsed.nil? + + SecurityInfo.new( + symbol: symbol, + name: parsed[:base], + links: "https://www.binance.com/en/trade/#{parsed[:binance_pair]}", + logo_url: verified_logo_url(parsed[:base]), + description: nil, + kind: "crypto", + exchange_operating_mic: exchange_operating_mic + ) + end + end + + def fetch_security_price(symbol:, exchange_operating_mic:, date:) + with_provider_response do + historical = fetch_security_prices( + symbol: symbol, + exchange_operating_mic: exchange_operating_mic, + start_date: date, + end_date: date + ) + + raise historical.error if historical.error.present? + raise InvalidSecurityPriceError, "No price found for #{symbol} on #{date}" if historical.data.blank? + + historical.data.first + end + end + + def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:) + with_provider_response do + parsed = parse_ticker(symbol) + raise InvalidSecurityPriceError, "Unsupported Binance ticker: #{symbol}" if parsed.nil? + + binance_pair = parsed[:binance_pair] + display_currency = parsed[:display_currency] + prices = [] + cursor = start_date + seen_data = false + + while cursor <= end_date + window_end = [ cursor + (KLINE_MAX_LIMIT - 1).days, end_date ].min + + throttle_request + response = client.get("#{base_url}/api/v3/klines") do |req| + req.params["symbol"] = binance_pair + req.params["interval"] = "1d" + req.params["startTime"] = date_to_ms(cursor) + req.params["endTime"] = date_to_ms(window_end) + MS_PER_DAY - 1 + req.params["limit"] = KLINE_MAX_LIMIT + end + + batch = JSON.parse(response.body) + + if batch.empty? + # Empty window. Two cases: + # 1. cursor is before the pair's listing date — keep advancing + # until we hit the first window containing valid klines. + # Critical for long-range imports (e.g. account sync from a + # trade start date that predates the Binance listing). + # 2. We have already collected prices and this window is past + # the end of available history — stop to avoid wasted calls + # on delisted pairs. + break if seen_data + else + seen_data = true + batch.each do |row| + open_time_ms = row[0].to_i + close_price = row[4].to_f + next if close_price <= 0 + + prices << Price.new( + symbol: symbol, + date: Time.at(open_time_ms / 1000).utc.to_date, + price: close_price, + currency: display_currency, + exchange_operating_mic: exchange_operating_mic + ) + end + end + + # Note: we intentionally do NOT break on a short (non-empty) batch. + # A window that straddles the pair's listing date legitimately returns + # fewer than KLINE_MAX_LIMIT rows while there is still valid data in + # subsequent windows. + cursor = window_end + 1.day + end + + prices + end + end + + private + def base_url + ENV["BINANCE_PUBLIC_URL"] || "https://data-api.binance.vision" + end + + def client + @client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday| + # Explicit timeouts so a hanging Binance endpoint can't stall a Sidekiq + # worker or Puma thread indefinitely. Values are deliberately generous + # enough for a full 1000-row klines response but capped to bound the + # worst-case retry chain (3 attempts * 20s + backoff ~= 65s). + faraday.options.open_timeout = 5 + faraday.options.timeout = 20 + + faraday.request(:retry, { + max: 3, + interval: 0.5, + interval_randomness: 0.5, + backoff_factor: 2, + exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ] + }) + + faraday.request :json + faraday.response :raise_error + faraday.headers["Accept"] = "application/json" + end + end + + # Maps a user-visible ticker (e.g. "BTCUSD", "ETHEUR") to the Binance pair + # symbol, base asset, and display currency. Returns nil if the ticker does + # not end with a supported quote currency. + def parse_ticker(ticker) + ticker_up = ticker.to_s.upcase + SUPPORTED_QUOTES.each do |quote| + display_currency = QUOTE_TO_CURRENCY[quote] + next unless ticker_up.end_with?(display_currency) + + base = ticker_up.delete_suffix(display_currency) + next if base.empty? + + return { binance_pair: "#{base}#{quote}", base: base, display_currency: display_currency } + end + nil + end + + # Cached for 24h — exchangeInfo returns the full symbol universe (thousands + # of rows, weight 10) and rarely changes. + def exchange_info_symbols + Rails.cache.fetch("binance_public:exchange_info", expires_in: 24.hours) do + throttle_request + response = client.get("#{base_url}/api/v3/exchangeInfo") + parsed = JSON.parse(response.body) + (parsed["symbols"] || []).select { |s| s["status"] == "TRADING" } + end + end + + def date_to_ms(date) + Time.utc(date.year, date.month, date.day).to_i * 1000 + end + + # Returns the asset-specific jsDelivr logo URL if the HEAD succeeds, else + # nil. Returning nil (rather than a hard-coded fallback URL) lets + # Security#display_logo_url swap in a Brandfetch binance.com URL at render + # time — a config-dependent path that can't be baked into a constant here. + # Cached per base asset for 30 days so we HEAD at most once per coin and + # only when Security#import_provider_details runs (never during search, + # which must stay fast). + def verified_logo_url(base_asset) + Rails.cache.fetch("binance_public:logo:#{base_asset}", expires_in: 30.days) do + candidate = "#{LOGO_CDN_BASE}/#{base_asset}.png" + logo_client.head(candidate) + candidate + rescue Faraday::Error + nil + end + end + + # Dedicated Faraday client for the logo CDN host (jsdelivr.net is a + # different origin from data-api.binance.vision). HEAD-only with a tight + # timeout so CDN hiccups can't stall Security info imports. + def logo_client + @logo_client ||= Faraday.new(url: LOGO_CDN_BASE, ssl: self.class.faraday_ssl_options) do |faraday| + faraday.options.timeout = 3 + faraday.options.open_timeout = 2 + faraday.response :raise_error + end + end + + # Preserve BinancePublic::Error subclasses (e.g. InvalidSecurityPriceError) + # through with_provider_response. The inherited RateLimitable transformer + # only preserves RateLimitError and would otherwise downcast our custom + # errors to the generic Error class. + def default_error_transformer(error) + return error if error.is_a?(self.class::Error) + super + end +end diff --git a/app/models/provider/registry.rb b/app/models/provider/registry.rb index efbe4395e..4782c1ee1 100644 --- a/app/models/provider/registry.rb +++ b/app/models/provider/registry.rb @@ -109,6 +109,10 @@ class Provider::Registry def mfapi Provider::Mfapi.new end + + def binance_public + Provider::BinancePublic.new + end end def initialize(concept) @@ -141,7 +145,7 @@ class Provider::Registry when :exchange_rates %i[twelve_data yahoo_finance] when :securities - %i[twelve_data yahoo_finance tiingo eodhd alpha_vantage mfapi] + %i[twelve_data yahoo_finance tiingo eodhd alpha_vantage mfapi binance_public] when :llm %i[openai] else diff --git a/app/models/provider/twelve_data.rb b/app/models/provider/twelve_data.rb index 670414fef..ba332dfe5 100644 --- a/app/models/provider/twelve_data.rb +++ b/app/models/provider/twelve_data.rb @@ -149,7 +149,7 @@ class Provider::TwelveData < Provider raise Error, "API error (code: #{error_code}): #{error_message}" end - data.map do |security| + data.reject { |row| crypto_row?(row) }.map do |security| country = ISO3166::Country.find_country_by_any_name(security.dig("country")) Security.new( @@ -250,6 +250,16 @@ class Provider::TwelveData < Provider private attr_reader :api_key + # TwelveData tags crypto symbols with `instrument_type: "Digital Currency"` and + # `mic_code: "DIGITAL_CURRENCY"`, and returns an empty `currency` field for them. + # We exclude them so crypto is handled exclusively by Provider::BinancePublic — + # TD's empty currency would otherwise cascade into Security::Price rows defaulting + # to USD, silently mispricing EUR/GBP crypto holdings. + def crypto_row?(row) + row["instrument_type"].to_s.casecmp?("Digital Currency") || + row["mic_code"].to_s.casecmp?("DIGITAL_CURRENCY") + end + def base_url ENV["TWELVE_DATA_URL"] || "https://api.twelvedata.com" end diff --git a/app/models/security.rb b/app/models/security.rb index 46c70493c..af3518d9e 100644 --- a/app/models/security.rb +++ b/app/models/security.rb @@ -20,6 +20,7 @@ class Security < ApplicationRecord before_validation :upcase_symbols before_save :generate_logo_url_from_brandfetch, if: :should_generate_logo? + before_save :reset_first_provider_price_on_if_provider_changed has_many :trades, dependent: :nullify, class_name: "Trade" has_many :prices, dependent: :destroy @@ -52,6 +53,31 @@ class Security < ApplicationRecord kind == "cash" end + # True when this security represents a crypto asset. Today the only signal + # is the Binance ISO MIC — when we add a second crypto provider, extend + # this check rather than duplicating the test at every call site. + def crypto? + exchange_operating_mic == Provider::BinancePublic::BINANCE_MIC + end + + # Single source of truth for which logo URL the UI should render. The order + # differs by asset class: + # + # - Crypto: prefer the verified per-asset jsDelivr logo (set during + # import by Provider::BinancePublic#verified_logo_url). On a miss, fall + # back to Brandfetch with a forced `binance.com` identifier so the + # generic Binance brand mark shows instead of a ticker lettermark. + # + # - Everything else: Brandfetch first (domain-derived or ticker lettermark), + # then any provider-set logo_url. + def display_logo_url + if crypto? + logo_url.presence || brandfetch_icon_url(identifier: "binance.com") + else + brandfetch_icon_url.presence || logo_url.presence + end + end + # Returns user-friendly exchange name for a MIC code def self.exchange_name_for(mic) return nil if mic.blank? @@ -80,13 +106,13 @@ class Security < ApplicationRecord ) end - def brandfetch_icon_url(width: nil, height: nil) + def brandfetch_icon_url(width: nil, height: nil, identifier: nil) return nil unless Setting.brand_fetch_client_id.present? w = width || Setting.brand_fetch_logo_size h = height || Setting.brand_fetch_logo_size - identifier = extract_domain(website_url) if website_url.present? + identifier ||= extract_domain(website_url) if website_url.present? identifier ||= ticker return nil unless identifier.present? @@ -123,4 +149,16 @@ class Security < ApplicationRecord def generate_logo_url_from_brandfetch self.logo_url = brandfetch_icon_url end + + # When a user remaps a security to a different provider (via the holdings + # remap combobox or Security::Resolver), the previously-discovered + # first_provider_price_on belongs to the OLD provider and may no longer + # reflect what the new provider can serve. Reset it so the next sync's + # fallback rediscovers the correct earliest date for the new provider. + # Skip when the caller explicitly set both columns in the same save. + def reset_first_provider_price_on_if_provider_changed + return unless price_provider_changed? + return if first_provider_price_on_changed? + self.first_provider_price_on = nil + end end diff --git a/app/models/security/price/importer.rb b/app/models/security/price/importer.rb index cd2bb0688..e0e323ccc 100644 --- a/app/models/security/price/importer.rb +++ b/app/models/security/price/importer.rb @@ -30,6 +30,37 @@ class Security::Price::Importer prev_price_value = start_price_value prev_currency = prev_price_currency || db_price_currency || "USD" + # Fallback for holdings that predate the asset's listing on the provider + # (e.g. a 2018 BTCEUR trade vs. Binance's 2020-01-03 listing date, or a + # 2023 RDDT trade vs. the 2024-03-21 IPO on Yahoo/Twelve Data). We can't + # anchor a price on or before start_date, but provider_prices has real + # data later in the range — advance fill_start_date to the earliest + # available provider date and use that price as the LOCF anchor. Days + # before that are intentionally left out of the DB (honest gap) rather + # than backfilled from a future price. + advanced_first_price_on = nil + + if prev_price_value.blank? + # Filter for valid rows BEFORE picking the earliest — otherwise a + # single listing-day / halt-day row with a nil or zero price would + # cause us to fall through to the MissingStartPriceError bail even + # when plenty of valid prices exist later in the window. + earliest_provider_price = provider_prices.values + .select { |p| p.price.present? && p.price.to_f > 0 } + .min_by(&:date) + + if earliest_provider_price + Rails.logger.info( + "#{security.ticker}: no provider price on or before #{start_date}; " \ + "advancing gapfill start to earliest valid provider date #{earliest_provider_price.date}" + ) + prev_price_value = earliest_provider_price.price + prev_currency = earliest_provider_price.currency || prev_currency + @fill_start_date = earliest_provider_price.date + advanced_first_price_on = earliest_provider_price.date + end + end + unless prev_price_value.present? Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{fill_start_date}") @@ -95,7 +126,26 @@ class Security::Price::Importer } end - upsert_rows(gapfilled_prices) + result = upsert_rows(gapfilled_prices) + + # Persist the advanced start date so subsequent syncs can clamp + # expected_count and short-circuit via all_prices_exist? instead of + # re-iterating the full (start_date..end_date) range every time. + # + # Update when the column is currently blank, OR when we've discovered + # an EARLIER date than the stored one — the latter covers the + # clear_cache-driven case where a provider has extended its backward + # coverage (e.g. Binance backfilling older BTCEUR history) and we + # want subsequent syncs to reflect the new earlier clamp. We never + # move the column forward from a previously-discovered earlier value, + # since that would silently hide older rows already in the DB. + if advanced_first_price_on.present? && + (security.first_provider_price_on.blank? || + advanced_first_price_on < security.first_provider_price_on) + security.update_column(:first_provider_price_on, advanced_first_price_on) + end + + result end private @@ -166,7 +216,16 @@ class Security::Price::Importer def all_prices_exist? return false if has_refetchable_provisional_prices? - db_prices.count == expected_count + + # Count only prices in the clamped range so pre-listing / pre-IPO gaps + # don't perpetually trip the "expected_count mismatch" re-sync. Query + # directly rather than via db_prices (which stays at the full range to + # preserve any user-entered rows pre-listing). + persisted_count = Security::Price + .where(security_id: security.id, date: clamped_start_date..end_date) + .count + + persisted_count == expected_count end def has_refetchable_provisional_prices? @@ -176,20 +235,38 @@ class Security::Price::Importer end def expected_count - (start_date..end_date).count + (clamped_start_date..end_date).count + end + + # Effective start date after clamping to the security's known first + # provider-available price date. Unlike start_date, this shrinks when the + # provider's history (e.g. Binance BTCEUR listed 2020-01-03, RDDT IPO + # 2024-03-21) begins after the user's original start_date. Falls through + # to start_date for any security that has never tripped the fallback. + def clamped_start_date + @clamped_start_date ||= begin + listed = security.first_provider_price_on + listed.present? && listed > start_date ? listed : start_date + end end # Skip over ranges that already exist unless clearing cache - # Also includes dates with refetchable provisional prices + # Also includes dates with refetchable provisional prices. + # + # Iterates from clamped_start_date (not start_date) so pre-listing / + # pre-IPO gaps don't perpetually trip "first missing date = start_date" + # and cause every incremental sync to re-fetch + re-upsert the full + # post-listing range. clear_cache bypasses the clamp so a user-triggered + # refresh can rediscover earlier provider history. def effective_start_date return start_date if clear_cache - refetchable_dates = Security::Price.where(security_id: security.id, date: start_date..end_date) + refetchable_dates = Security::Price.where(security_id: security.id, date: clamped_start_date..end_date) .refetchable_provisional(lookback_days: PROVISIONAL_LOOKBACK_DAYS) .pluck(:date) .to_set - (start_date..end_date).detect do |d| + (clamped_start_date..end_date).detect do |d| !db_prices.key?(d) || refetchable_dates.include?(d) end || end_date end diff --git a/app/models/security/resolver.rb b/app/models/security/resolver.rb index 426b6937b..4d0521b2d 100644 --- a/app/models/security/resolver.rb +++ b/app/models/security/resolver.rb @@ -86,7 +86,7 @@ class Security::Resolver exchange_matches = s.exchange_operating_mic&.upcase.to_s == exchange_operating_mic.upcase.to_s if country_code && exchange_operating_mic - ticker_matches && exchange_matches && s.country_code&.upcase.to_s == country_code.upcase.to_s + ticker_matches && exchange_matches && country_matches?(s.country_code) else ticker_matches && exchange_matches end @@ -101,8 +101,10 @@ class Security::Resolver filtered_candidates = provider_search_result # If a country code is specified, we MUST find a match with the same code + # — but nil candidate country is treated as a wildcard (e.g. crypto from + # Binance, which isn't tied to a jurisdiction). if country_code.present? - filtered_candidates = filtered_candidates.select { |s| s.country_code&.upcase.to_s == country_code.upcase.to_s } + filtered_candidates = filtered_candidates.select { |s| country_matches?(s.country_code) } end # 1. Prefer exact ticker matches (MSTR before MSTRX when searching for "MSTR") @@ -161,6 +163,15 @@ class Security::Resolver security.update!(offline: false, offline_reason: nil, failed_fetch_count: 0, failed_fetch_at: nil) end + # Candidate country matches when it equals the resolver's country OR when + # the provider didn't report a country at all (e.g. crypto from Binance). + # A nil candidate country is a legitimate "no jurisdiction" signal, not a + # missing field, so we trust the user's provider + exchange pick. + def country_matches?(candidate_country) + return true if candidate_country.blank? + candidate_country.upcase == country_code.upcase + end + def provider_search_result params = { exchange_operating_mic: exchange_operating_mic, diff --git a/app/views/holdings/_holding.html.erb b/app/views/holdings/_holding.html.erb index c0ba88e64..8d6adbfdb 100644 --- a/app/views/holdings/_holding.html.erb +++ b/app/views/holdings/_holding.html.erb @@ -3,10 +3,8 @@ <%= turbo_frame_tag dom_id(holding) do %>
- <% if holding.security.brandfetch_icon_url.present? %> - <%= image_tag holding.security.brandfetch_icon_url, class: "w-9 h-9 rounded-full", loading: "lazy" %> - <% elsif holding.security.logo_url.present? %> - <%= image_tag holding.security.logo_url, class: "w-9 h-9 rounded-full", loading: "lazy" %> + <% if (logo = holding.security.display_logo_url).present? %> + <%= image_tag logo, class: "w-9 h-9 rounded-full", loading: "lazy" %> <% else %> <%= render DS::FilledIcon.new(variant: :text, text: holding.name, size: "md", rounded: true) %> <% end %> diff --git a/app/views/holdings/show.html.erb b/app/views/holdings/show.html.erb index fb4dc3b66..37c7c86d4 100644 --- a/app/views/holdings/show.html.erb +++ b/app/views/holdings/show.html.erb @@ -6,10 +6,8 @@ <%= tag.p @holding.ticker, class: "text-sm text-secondary" %>
- <% if @holding.security.brandfetch_icon_url.present? %> - <%= image_tag @holding.security.brandfetch_icon_url, loading: "lazy", class: "w-9 h-9 rounded-full" %> - <% elsif @holding.security.logo_url.present? %> - <%= image_tag @holding.security.logo_url, loading: "lazy", class: "w-9 h-9 rounded-full" %> + <% if (logo = @holding.security.display_logo_url).present? %> + <%= image_tag logo, loading: "lazy", class: "w-9 h-9 rounded-full" %> <% else %> <%= render DS::FilledIcon.new(variant: :text, text: @holding.name, size: "md", rounded: true) %> <% end %> @@ -28,6 +26,16 @@ ) %>
<% end %> + <% if (first_on = @holding.security.first_provider_price_on).present? && + (earliest_trade_date = @holding.trades.minimum(:date)) && + earliest_trade_date < first_on %> +
+ <%= render DS::Alert.new( + message: t(".truncated_history_warning", date: l(first_on, format: :long)), + variant: :warning + ) %> +
+ <% end %>
diff --git a/app/views/reports/_investment_performance.html.erb b/app/views/reports/_investment_performance.html.erb index 7434ab923..4f5e579d8 100644 --- a/app/views/reports/_investment_performance.html.erb +++ b/app/views/reports/_investment_performance.html.erb @@ -75,10 +75,8 @@ ">
- <% if holding.security.brandfetch_icon_url.present? %> - <%= holding.ticker %> - <% elsif holding.security.logo_url.present? %> - <%= holding.ticker %> + <% if (logo = holding.security.display_logo_url).present? %> + <%= holding.ticker %> <% else %>
<%= holding.ticker[0..1] %> diff --git a/app/views/settings/hostings/_provider_selection.html.erb b/app/views/settings/hostings/_provider_selection.html.erb index 1ca2da305..147d082ab 100644 --- a/app/views/settings/hostings/_provider_selection.html.erb +++ b/app/views/settings/hostings/_provider_selection.html.erb @@ -50,6 +50,7 @@ ["eodhd", t(".providers.eodhd"), t(".requires_api_key_eodhd")], ["alpha_vantage", t(".providers.alpha_vantage"), t(".requires_api_key_alpha_vantage")], ["mfapi", t(".providers.mfapi"), t(".mfapi_hint")], + ["binance_public", t(".providers.binance_public"), t(".binance_public_hint")], ].each do |value, label, hint| %>