Expand financial providers (#1407)

* Initial implementation

* Tiingo fixes

* Adds 2 providers, remove 2

* Add  extra checks

* FIX a big hotwire race condition

// Fix hotwire_combobox race condition: when typing quickly, a slow response for
// an early query (e.g. "A") can overwrite the correct results for the final query
// (e.g. "AAPL"). We abort the previous in-flight request whenever a new one fires,
// so stale Turbo Stream responses never reach the DOM.

* pipelock

* Update price_test.rb

* Reviews

* i8n

* fixes

* fixes

* Update tiingo.rb

* fixes

* Improvements

* Big revamp

* optimisations

* Update 20260408151837_add_offline_reason_to_securities.rb

* Add missing tests, fixes

* small rank tests

* FIX tests

* Update show.html.erb

* Update resolver.rb

* Update usd_converter.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update _yahoo_finance_settings.html.erb
This commit is contained in:
soky srm
2026-04-09 18:33:59 +02:00
committed by GitHub
parent ab13093634
commit 7908f7d8a4
50 changed files with 2553 additions and 206 deletions

View File

@@ -52,36 +52,44 @@ class HoldingsController < ApplicationController
end
def remap_security
# Combobox returns "TICKER|EXCHANGE" format
ticker, exchange = params[:security_id].to_s.split("|")
# Combobox returns "TICKER|EXCHANGE|PROVIDER" format
parsed = Security.parse_combobox_id(params[:security_id])
# Validate ticker is present (form has required: true, but can be bypassed)
if ticker.blank?
flash[:alert] = t(".security_not_found")
redirect_to account_path(@holding.account, tab: "holdings")
return
end
new_security = Security::Resolver.new(
ticker,
exchange_operating_mic: exchange,
country_code: Current.family.country
).resolve
if new_security.nil?
if parsed[:ticker].blank?
flash[:alert] = t(".security_not_found")
redirect_to account_path(@holding.account, tab: "holdings")
return
end
# The user explicitly selected this security from provider search results,
# so we know the provider can handle it. Bring it back online if it was
# previously marked offline (e.g. by a failed QIF import resolution).
if new_security.offline?
new_security.update!(offline: false, failed_fetch_count: 0, failed_fetch_at: nil)
end
# so we use the combobox data directly — no need to re-resolve via provider APIs.
new_security = Security.find_or_initialize_by(
ticker: parsed[:ticker],
exchange_operating_mic: parsed[:exchange_operating_mic]
)
# Honor the user's provider choice (validated by model inclusion check on save)
new_security.price_provider = parsed[:price_provider] if parsed[:price_provider].present?
# Bring it online — user explicitly selected it from provider search results,
# so we know the provider can handle it.
new_security.offline = false
new_security.failed_fetch_count = 0
new_security.failed_fetch_at = nil
new_security.save!
@holding.remap_security!(new_security)
# Re-materialize holdings with the new security's prices.
# Reload account to avoid stale associations from remap_security!.
# The around_action :switch_timezone already sets the family timezone
# for this request, so Date.current is correct here.
account = Account.find(@holding.account_id)
strategy = account.linked? ? :reverse : :forward
Balance::Materializer.new(account, strategy: strategy, security_ids: [ new_security.id ]).materialize_balances
flash[:notice] = t(".success")
respond_to do |format|

View File

@@ -14,13 +14,14 @@ class Settings::HostingsController < ApplicationController
# Determine which providers are currently selected
exchange_rate_provider = ENV["EXCHANGE_RATE_PROVIDER"].presence || Setting.exchange_rate_provider
securities_provider = ENV["SECURITIES_PROVIDER"].presence || Setting.securities_provider
enabled_securities = Setting.enabled_securities_providers
# Show Twelve Data settings if either provider is set to twelve_data
@show_twelve_data_settings = exchange_rate_provider == "twelve_data" || securities_provider == "twelve_data"
# Show Yahoo Finance settings if either provider is set to yahoo_finance
@show_yahoo_finance_settings = exchange_rate_provider == "yahoo_finance" || securities_provider == "yahoo_finance"
# Show provider settings if used for FX or enabled for securities
@show_twelve_data_settings = exchange_rate_provider == "twelve_data" || enabled_securities.include?("twelve_data")
@show_yahoo_finance_settings = exchange_rate_provider == "yahoo_finance" || enabled_securities.include?("yahoo_finance")
@show_tiingo_settings = enabled_securities.include?("tiingo")
@show_eodhd_settings = enabled_securities.include?("eodhd")
@show_alpha_vantage_settings = enabled_securities.include?("alpha_vantage")
# Only fetch provider data if we're showing the section
if @show_twelve_data_settings
@@ -57,9 +58,7 @@ class Settings::HostingsController < ApplicationController
Setting.brand_fetch_high_res_logos = hosting_params[:brand_fetch_high_res_logos] == "1"
end
if hosting_params.key?(:twelve_data_api_key)
Setting.twelve_data_api_key = hosting_params[:twelve_data_api_key]
end
update_encrypted_setting(:twelve_data_api_key)
if hosting_params.key?(:exchange_rate_provider)
Setting.exchange_rate_provider = hosting_params[:exchange_rate_provider]
@@ -69,6 +68,40 @@ class Settings::HostingsController < ApplicationController
Setting.securities_provider = hosting_params[:securities_provider]
end
if hosting_params.key?(:securities_providers)
new_providers = Array(hosting_params[:securities_providers]).reject(&:blank?) & Security.valid_price_providers
old_providers = Setting.enabled_securities_providers
Setting.securities_providers = new_providers.join(",")
# Clear the legacy singular setting so the fallback in
# enabled_securities_providers doesn't re-enable a provider
# the user just unchecked.
Setting.securities_provider = nil if new_providers.empty?
# Mark securities linked to removed providers as offline so they aren't
# silently queried against an incompatible fallback provider (e.g. MFAPI
# scheme codes sent to TwelveData). The price_provider is preserved so
# provider_status can report :provider_unavailable.
removed = old_providers - new_providers
removed.each do |removed_provider|
Security.where(price_provider: removed_provider, offline: false)
.in_batches.update_all(offline: true, offline_reason: "provider_disabled")
end
# Bring securities back online when their provider is re-enabled — but only
# those that were taken offline by a provider toggle, not by health checks.
added = new_providers - old_providers
added.each do |added_provider|
Security.where(price_provider: added_provider, offline: true, offline_reason: "provider_disabled")
.in_batches.update_all(offline: false, offline_reason: nil, failed_fetch_count: 0, failed_fetch_at: nil)
end
end
update_encrypted_setting(:tiingo_api_key)
update_encrypted_setting(:eodhd_api_key)
update_encrypted_setting(:alpha_vantage_api_key)
if hosting_params.key?(:syncs_include_pending)
Setting.syncs_include_pending = hosting_params[:syncs_include_pending] == "1"
end
@@ -166,7 +199,7 @@ class Settings::HostingsController < ApplicationController
private
def hosting_params
return ActionController::Parameters.new unless params.key?(:setting)
params.require(:setting).permit(:onboarding_state, :require_email_confirmation, :invite_only_default_family_id, :brand_fetch_client_id, :brand_fetch_high_res_logos, :twelve_data_api_key, :openai_access_token, :openai_uri_base, :openai_model, :openai_json_mode, :exchange_rate_provider, :securities_provider, :syncs_include_pending, :auto_sync_enabled, :auto_sync_time, :external_assistant_url, :external_assistant_token, :external_assistant_agent_id)
params.require(:setting).permit(:onboarding_state, :require_email_confirmation, :invite_only_default_family_id, :brand_fetch_client_id, :brand_fetch_high_res_logos, :twelve_data_api_key, :tiingo_api_key, :eodhd_api_key, :alpha_vantage_api_key, :openai_access_token, :openai_uri_base, :openai_model, :openai_json_mode, :exchange_rate_provider, :securities_provider, :syncs_include_pending, :auto_sync_enabled, :auto_sync_time, :external_assistant_url, :external_assistant_token, :external_assistant_agent_id, securities_providers: [])
end
def update_assistant_type
@@ -195,6 +228,12 @@ class Settings::HostingsController < ApplicationController
flash[:alert] = t(".scheduler_sync_failed")
end
def update_encrypted_setting(param_key)
return unless hosting_params.key?(param_key)
value = hosting_params[param_key].to_s.strip
Setting.public_send(:"#{param_key}=", value) unless value.blank? || value == "********"
end
def current_user_timezone
Current.family&.timezone.presence || "UTC"
end

View File

@@ -524,12 +524,18 @@ class TransactionsController < ApplicationController
if params[:security_id] == "__custom__"
# User selected "Enter custom ticker" - check for combobox selection or manual entry
if params[:ticker].present?
# Combobox selection: format is "SYMBOL|EXCHANGE"
ticker_symbol, exchange_operating_mic = params[:ticker].split("|")
# Combobox selection: format is "SYMBOL|EXCHANGE|PROVIDER"
parsed = Security.parse_combobox_id(params[:ticker])
if parsed[:ticker].blank?
flash[:alert] = t("transactions.convert_to_trade.errors.enter_ticker")
redirect_back_or_to transactions_path
return nil
end
Security::Resolver.new(
ticker_symbol.strip,
exchange_operating_mic: exchange_operating_mic.presence || params[:exchange_operating_mic].presence,
country_code: user_country
parsed[:ticker].strip,
exchange_operating_mic: parsed[:exchange_operating_mic] || params[:exchange_operating_mic].presence,
country_code: user_country,
price_provider: parsed[:price_provider]
).resolve
elsif params[:custom_ticker].present?
# Manual entry from combobox's name_when_new or fallback text field
@@ -552,12 +558,18 @@ class TransactionsController < ApplicationController
end
found
elsif params[:ticker].present?
# Direct combobox (no existing holdings) - format is "SYMBOL|EXCHANGE"
ticker_symbol, exchange_operating_mic = params[:ticker].split("|")
# Direct combobox (no existing holdings) - format is "SYMBOL|EXCHANGE|PROVIDER"
parsed = Security.parse_combobox_id(params[:ticker])
if parsed[:ticker].blank?
flash[:alert] = t("transactions.convert_to_trade.errors.enter_ticker")
redirect_back_or_to transactions_path
return nil
end
Security::Resolver.new(
ticker_symbol.strip,
exchange_operating_mic: exchange_operating_mic.presence || params[:exchange_operating_mic].presence,
country_code: user_country
parsed[:ticker].strip,
exchange_operating_mic: parsed[:exchange_operating_mic] || params[:exchange_operating_mic].presence,
country_code: user_country,
price_provider: parsed[:price_provider]
).resolve
elsif params[:custom_ticker].present?
# Manual entry from combobox's name_when_new (no existing holdings path)

View File

@@ -1,6 +1,49 @@
// Configure your import map in config/importmap.rb. Read more: https://github.com/rails/importmap-rails
import "@hotwired/turbo-rails";
import "controllers";
import HwComboboxController from "controllers/hw_combobox_controller";
// Fix hotwire_combobox race condition: when typing quickly, a slow response for
// an early query (e.g. "A") can overwrite the correct results for the final query
// (e.g. "AAPL"). We abort the previous in-flight request whenever a new one fires,
// so stale Turbo Stream responses never reach the DOM.
const originalFilterAsync = HwComboboxController.prototype._filterAsync;
HwComboboxController.prototype._filterAsync = async function(inputType) {
if (this._searchAbortController) {
this._searchAbortController.abort();
}
this._searchAbortController = new AbortController();
const query = {
q: this._fullQuery,
input_type: inputType,
for_id: this.element.dataset.asyncId,
callback_id: this._enqueueCallback()
};
const url = new URL(this.asyncSrcValue, window.location.origin);
Object.entries(query).forEach(([k, v]) => {
if (v != null) url.searchParams.set(k, v);
});
try {
const response = await fetch(url.toString(), {
headers: {
"Accept": "text/vnd.turbo-stream.html, text/html, application/xhtml+xml",
"X-Requested-With": "XMLHttpRequest",
"X-CSRF-Token": document.querySelector('meta[name="csrf-token"]')?.content
},
signal: this._searchAbortController.signal,
credentials: "same-origin"
});
if (response.ok) {
await Turbo.renderStreamMessage(await response.text());
}
} catch (e) {
if (e.name !== "AbortError") throw e;
}
};
Turbo.StreamActions.redirect = function () {
// Use "replace" to avoid adding form submission to browser history

View File

@@ -155,7 +155,7 @@ class Assistant::Function::ImportBankStatement < Assistant::Function
account_holder: result[:account_holder],
message: "Successfully extracted #{result[:transactions].size} transactions. Import created with ID: #{import.id}. Review and publish when ready."
}
rescue Provider::ProviderError, Faraday::Error, Timeout::Error, RuntimeError => e
rescue Provider::Error, Faraday::Error, Timeout::Error, RuntimeError => e
Rails.logger.error("ImportBankStatement error: #{e.class.name} - #{e.message}")
Rails.logger.error(e.backtrace.first(10).join("\n"))
{

View File

@@ -17,7 +17,7 @@ class MarketDataImporter
# Syncs historical security prices (and details)
def import_security_prices
unless Security.provider
unless Security.providers.any?
Rails.logger.warn("No provider configured for MarketDataImporter.import_security_prices, skipping sync")
return
end

View File

@@ -0,0 +1,340 @@
class Provider::AlphaVantage < Provider
include SecurityConcept, RateLimitable
extend SslConfigurable
# Subclass so errors caught in this provider are raised as Provider::AlphaVantage::Error
Error = Class.new(Provider::Error)
InvalidSecurityPriceError = Class.new(Error)
RateLimitError = Class.new(Error)
# Minimum delay between requests to avoid rate limiting (in seconds)
MIN_REQUEST_INTERVAL = 1.0
# Maximum requests per day (Alpha Vantage free tier limit)
MAX_REQUESTS_PER_DAY = 25
# Free tier "compact" returns ~100 trading days (~140 calendar days).
# "full" requires a paid plan.
def max_history_days
140
end
# MIC code to Alpha Vantage symbol suffix mapping
MIC_TO_AV_SUFFIX = {
"XNYS" => "", "XNAS" => "", "XASE" => "",
"XLON" => ".LON",
"XETR" => ".DEX",
"XTSE" => ".TRT",
"XPAR" => ".PAR",
"XAMS" => ".AMS",
"XSWX" => ".SWX",
"XHKG" => ".HKG",
"XASX" => ".ASX",
"XMIL" => ".MIL",
"XMAD" => ".BME",
"XOSL" => ".OSL",
"XSTO" => ".STO",
"XCSE" => ".CPH",
"XHEL" => ".HEL"
}.freeze
# Alpha Vantage symbol suffix to MIC code mapping (auto-generated from forward map)
AV_SUFFIX_TO_MIC = MIC_TO_AV_SUFFIX
.reject { |_, suffix| suffix.empty? }
.each_with_object({}) { |(mic, suffix), h| h[suffix.delete(".")] = mic }
.merge("FRK" => "XFRA") # FRK is not in the forward map (no MIC→FRK entry)
.freeze
# Alpha Vantage region names to ISO country codes
AV_REGION_TO_COUNTRY = {
"United States" => "US", "United Kingdom" => "GB",
"Frankfurt" => "DE", "XETRA" => "DE",
"Amsterdam" => "NL", "Paris/Brussels" => "FR",
"Switzerland" => "CH", "Toronto" => "CA",
"Brazil/Sao Paolo" => "BR",
"India/Bombay" => "IN", "Hong Kong" => "HK",
"Milan" => "IT", "Madrid" => "ES",
"Oslo" => "NO", "Helsinki" => "FI",
"Copenhagen" => "DK", "Stockholm" => "SE",
"Australia" => "AU", "Japan" => "JP"
}.freeze
def initialize(api_key)
@api_key = api_key # pipelock:ignore
end
# Alpha Vantage has no non-quota endpoint — every API call counts against
# the 25/day free-tier limit. Rather than burn a call, we just check that
# the API key is configured.
def healthy?
with_provider_response do
api_key.present?
end
end
def usage
with_provider_response do
day_key = "alpha_vantage:daily:#{Date.current}"
used = Rails.cache.read(day_key).to_i
UsageData.new(
used: used,
limit: max_requests_per_day,
utilization: (used.to_f / max_requests_per_day * 100).round(1),
plan: "Free"
)
end
end
# ================================
# Securities
# ================================
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
with_provider_response do
throttle_request
response = client.get("#{base_url}/query") do |req|
req.params["function"] = "SYMBOL_SEARCH"
req.params["keywords"] = symbol
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
data = parsed.dig("bestMatches")
if data.nil?
raise Error, "No data returned from search endpoint"
end
data.first(25).map do |match|
av_ticker = match["1. symbol"]
region = match["4. region"]
currency = match["8. currency"]
# Cache the API-returned currency so fetch_security_prices can use it
# instead of relying solely on the hardcoded suffix→currency fallback
if currency.present?
cache_key = "alpha_vantage:currency:#{av_ticker.upcase}"
Rails.cache.write(cache_key, currency, expires_in: 24.hours)
end
Security.new(
symbol: strip_av_suffix(av_ticker),
name: match["2. name"],
logo_url: nil,
exchange_operating_mic: extract_mic_from_symbol(av_ticker),
country_code: AV_REGION_TO_COUNTRY[region],
currency: currency
)
end
end
end
def fetch_security_info(symbol:, exchange_operating_mic:)
with_provider_response do
av_symbol = to_av_symbol(symbol, exchange_operating_mic)
throttle_request
response = client.get("#{base_url}/query") do |req|
req.params["function"] = "OVERVIEW"
req.params["symbol"] = av_symbol
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
name = parsed["Name"]
if name.blank?
raise Error, "No metadata returned for symbol #{av_symbol}"
end
SecurityInfo.new(
symbol: parsed["Symbol"] || symbol,
name: name,
links: parsed["OfficialSite"].presence,
logo_url: nil,
description: parsed["Description"].presence,
kind: parsed["AssetType"]&.downcase,
exchange_operating_mic: exchange_operating_mic
)
end
end
def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
with_provider_response do
historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
raise historical_data.error if historical_data.error.present?
raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?
historical_data.data.first
end
end
def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
with_provider_response do
av_symbol = to_av_symbol(symbol, exchange_operating_mic)
throttle_request
response = client.get("#{base_url}/query") do |req|
req.params["function"] = "TIME_SERIES_DAILY"
req.params["symbol"] = av_symbol
req.params["outputsize"] = "compact"
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
time_series = parsed.dig("Time Series (Daily)")
if time_series.nil?
raise InvalidSecurityPriceError, "No time series data returned for symbol #{av_symbol}"
end
currency = infer_currency_from_symbol(av_symbol)
time_series.filter_map do |date_str, values|
date = Date.parse(date_str)
next unless date >= start_date && date <= end_date
price = values["4. close"]
if price.nil? || price.to_f <= 0
Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date_str}. Price data: #{price.inspect}")
next
end
Price.new(
symbol: symbol,
date: date,
price: price,
currency: currency,
exchange_operating_mic: exchange_operating_mic
)
end
end
end
private
attr_reader :api_key
def base_url
ENV["ALPHA_VANTAGE_URL"] || "https://www.alphavantage.co"
end
def client
@client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
faraday.request(:retry, {
max: 3,
interval: 1.0,
interval_randomness: 0.5,
backoff_factor: 2,
exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
})
faraday.request :json
faraday.response :raise_error
faraday.params["apikey"] = api_key
end
end
# Adds daily request counter on top of the interval throttle from RateLimitable.
def throttle_request
super
# Global per-day request counter via cache (Redis).
# Atomic increment-then-check avoids the TOCTOU of read-check-increment.
day_key = "alpha_vantage:daily:#{Date.current}"
new_count = Rails.cache.increment(day_key, 1, expires_in: 24.hours).to_i
if new_count > max_requests_per_day
Rails.logger.warn("AlphaVantage: daily request limit reached (#{new_count}/#{max_requests_per_day})")
raise RateLimitError, "Alpha Vantage daily request limit reached (#{max_requests_per_day} per day)"
end
end
def max_requests_per_day
ENV.fetch("ALPHA_VANTAGE_MAX_REQUESTS_PER_DAY", MAX_REQUESTS_PER_DAY).to_i
end
# Converts a symbol + MIC code to Alpha Vantage's ticker format
def to_av_symbol(symbol, exchange_operating_mic)
return symbol if exchange_operating_mic.blank?
suffix = MIC_TO_AV_SUFFIX[exchange_operating_mic]
return symbol if suffix.nil?
return symbol if suffix.empty?
# Avoid double-suffixing if the symbol already has the correct suffix
return symbol if symbol.end_with?(suffix)
"#{symbol}#{suffix}"
end
# Strips the Alpha Vantage exchange suffix to get the canonical ticker
# e.g., "CSPX.LON" -> "CSPX", "AAPL" -> "AAPL"
def strip_av_suffix(symbol)
return symbol unless symbol.include?(".")
parts = symbol.split(".", 2)
AV_SUFFIX_TO_MIC.key?(parts.last) ? parts.first : symbol
end
# Extracts MIC code from Alpha Vantage symbol suffix (e.g., "CSPX.LON" -> "XLON")
def extract_mic_from_symbol(symbol)
return nil unless symbol.include?(".")
suffix = symbol.split(".").last
AV_SUFFIX_TO_MIC[suffix]
end
# Infers currency from the exchange suffix of an Alpha Vantage symbol.
# Falls back to cached currency from search results if available.
def infer_currency_from_symbol(av_symbol)
cache_key = "alpha_vantage:currency:#{av_symbol.upcase}"
cached = Rails.cache.read(cache_key)
return cached if cached.present?
# Default currency based on exchange suffix
suffix = av_symbol.include?(".") ? av_symbol.split(".").last : nil
currency = case suffix
when "LON" then "GBP"
when "DEX", "FRK" then "EUR"
when "PAR", "AMS", "MIL", "BME", "HEL" then "EUR"
when "TRT" then "CAD"
when "SWX" then "CHF"
when "HKG" then "HKD"
when "ASX" then "AUD"
when "STO" then "SEK"
when "CPH" then "DKK"
when "OSL" then "NOK"
else "USD"
end
Rails.cache.write(cache_key, currency, expires_in: 24.hours)
currency
end
# Checks for Alpha Vantage-specific error responses.
# Alpha Vantage returns errors as JSON keys rather than HTTP status codes.
def check_api_error!(parsed)
return unless parsed.is_a?(Hash)
# Rate limit: Alpha Vantage returns a "Note" key when rate-limited
if parsed["Note"].present?
Rails.logger.warn("AlphaVantage rate limit: #{parsed["Note"]}")
raise RateLimitError, parsed["Note"]
end
# General info/limit messages
if parsed["Information"].present?
Rails.logger.warn("AlphaVantage info: #{parsed["Information"]}")
raise RateLimitError, parsed["Information"]
end
# Explicit error messages for invalid parameters
if parsed["Error Message"].present?
raise Error, "API error: #{parsed["Error Message"]}"
end
end
end

View File

@@ -0,0 +1,304 @@
class Provider::Eodhd < Provider
include SecurityConcept, RateLimitable
extend SslConfigurable
# Subclass so errors caught in this provider are raised as Provider::Eodhd::Error
Error = Class.new(Provider::Error)
InvalidSecurityPriceError = Class.new(Error)
RateLimitError = Class.new(Error)
# Minimum delay between requests to avoid rate limiting (in seconds)
MIN_REQUEST_INTERVAL = 0.5
# Maximum API calls per day (EODHD free/basic plans are very restrictive)
MAX_REQUESTS_PER_DAY = 20
# EODHD free tier provides ~1 year of EOD data
def max_history_days
365
end
# EODHD uses {SYMBOL}.{EXCHANGE} ticker format with its own exchange codes
MIC_TO_EODHD_EXCHANGE = {
"XNYS" => "US", "XNAS" => "US", "XASE" => "US",
"XLON" => "LSE",
"XETR" => "XETRA",
"XTSE" => "TO",
"XPAR" => "PA",
"XAMS" => "AS",
"XSWX" => "SW",
"XHKG" => "HK",
"XASX" => "AU",
"XTKS" => "TSE",
"XMIL" => "MI",
"XMAD" => "MC",
"XOSL" => "OL",
"XHEL" => "HE",
"XCSE" => "CO",
"XSTO" => "ST",
"XKRX" => "KS",
"XBOM" => "BSE",
"XNSE" => "NSE"
}.freeze
EODHD_EXCHANGE_TO_MIC = {
"US" => "XNYS", "LSE" => "XLON", "XETRA" => "XETR",
"TO" => "XTSE", "PA" => "XPAR", "AS" => "XAMS",
"SW" => "XSWX", "HK" => "XHKG", "AU" => "XASX",
"TSE" => "XTKS", "MI" => "XMIL", "MC" => "XMAD",
"OL" => "XOSL", "HE" => "XHEL", "CO" => "XCSE",
"ST" => "XSTO", "KS" => "XKRX", "BSE" => "XBOM",
"NSE" => "XNSE"
}.freeze
EODHD_COUNTRY_TO_CODE = {
"USA" => "US", "UK" => "GB", "Germany" => "DE", "France" => "FR",
"Netherlands" => "NL", "Switzerland" => "CH", "Canada" => "CA",
"Japan" => "JP", "Australia" => "AU", "Hong Kong" => "HK",
"Italy" => "IT", "Spain" => "ES", "Norway" => "NO",
"Finland" => "FI", "Denmark" => "DK", "Sweden" => "SE",
"South Korea" => "KR", "India" => "IN"
}.freeze
EXCHANGE_CURRENCY = {
"US" => "USD", "LSE" => "GBP", "XETRA" => "EUR", "TO" => "CAD",
"PA" => "EUR", "AS" => "EUR", "SW" => "CHF", "HK" => "HKD",
"AU" => "AUD", "TSE" => "JPY", "MI" => "EUR", "MC" => "EUR",
"OL" => "NOK", "HE" => "EUR", "CO" => "DKK",
"ST" => "SEK", "KS" => "KRW", "BSE" => "INR",
"NSE" => "INR"
}.freeze
def initialize(api_key)
@api_key = api_key # pipelock:ignore
end
def healthy?
with_provider_response do
response = client.get("#{base_url}/api/user") do |req|
req.params["api_token"] = api_key
req.params["fmt"] = "json"
end
JSON.parse(response.body).dig("name").present?
end
end
def usage
with_provider_response do
response = client.get("#{base_url}/api/user") do |req|
req.params["api_token"] = api_key
req.params["fmt"] = "json"
end
parsed = JSON.parse(response.body)
limit = parsed.dig("apiRequests").to_i
daily_limit = parsed.dig("dailyRateLimit").to_i
daily_key = daily_cache_key
used = Rails.cache.read(daily_key).to_i
UsageData.new(
used: used,
limit: daily_limit > 0 ? daily_limit : MAX_REQUESTS_PER_DAY,
utilization: daily_limit > 0 ? (used.to_f / daily_limit * 100) : (used.to_f / MAX_REQUESTS_PER_DAY * 100),
plan: parsed.dig("subscriptionType") || "unknown"
)
end
end
# ================================
# Securities
# ================================
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
with_provider_response do
enforce_daily_limit!
throttle_request
response = client.get("#{base_url}/api/search/#{CGI.escape(symbol)}") do |req|
req.params["api_token"] = api_key
req.params["fmt"] = "json"
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
unless parsed.is_a?(Array)
raise Error, "Unexpected response format from search API"
end
parsed.first(25).map do |security|
eodhd_exchange = security.dig("Exchange")
mic = EODHD_EXCHANGE_TO_MIC[eodhd_exchange]
country = EODHD_COUNTRY_TO_CODE[security.dig("Country")]
code = security.dig("Code")
currency = security.dig("Currency")
# Cache the API-returned currency so fetch_security_prices can use it
if currency.present? && mic.present?
cache_key = "eodhd:currency:#{code.upcase}:#{mic}"
Rails.cache.write(cache_key, currency, expires_in: 24.hours)
end
Security.new(
symbol: code,
name: security.dig("Name"),
logo_url: nil,
exchange_operating_mic: mic,
country_code: country,
currency: currency
)
end
end
end
def fetch_security_info(symbol:, exchange_operating_mic:)
with_provider_response do
enforce_daily_limit!
throttle_request
ticker = eodhd_symbol(symbol, exchange_operating_mic)
response = client.get("#{base_url}/api/fundamentals/#{CGI.escape(ticker)}") do |req|
req.params["api_token"] = api_key
req.params["fmt"] = "json"
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
general = parsed.dig("General") || {}
SecurityInfo.new(
symbol: symbol,
name: general.dig("Name"),
links: general.dig("WebURL"),
logo_url: general.dig("LogoURL"),
description: general.dig("Description"),
kind: general.dig("Type"),
exchange_operating_mic: exchange_operating_mic
)
end
end
def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
with_provider_response do
historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
raise historical_data.error if historical_data.error.present?
raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?
historical_data.data.first
end
end
def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
with_provider_response do
enforce_daily_limit!
throttle_request
ticker = eodhd_symbol(symbol, exchange_operating_mic)
response = client.get("#{base_url}/api/eod/#{CGI.escape(ticker)}") do |req|
req.params["api_token"] = api_key
req.params["fmt"] = "json"
req.params["from"] = start_date.to_s
req.params["to"] = end_date.to_s
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
unless parsed.is_a?(Array)
raise InvalidSecurityPriceError, "Unexpected response format from EOD API"
end
# Prefer cached currency from search results; fall back to hardcoded map
cache_key = "eodhd:currency:#{symbol.upcase}:#{exchange_operating_mic}"
eodhd_exchange = MIC_TO_EODHD_EXCHANGE[exchange_operating_mic]
currency = Rails.cache.read(cache_key) || EXCHANGE_CURRENCY[eodhd_exchange]
parsed.map do |resp|
price = resp.dig("close")
date = resp.dig("date")
if price.nil? || price.to_f <= 0
Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}. Price data: #{price.inspect}")
next
end
Price.new(
symbol: symbol,
date: date.to_date,
price: price,
currency: currency,
exchange_operating_mic: exchange_operating_mic
)
end.compact
end
end
private
attr_reader :api_key
def base_url
ENV["EODHD_URL"] || "https://eodhd.com"
end
def client
@client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
faraday.request(:retry, {
max: 3,
interval: 1.0,
interval_randomness: 0.5,
backoff_factor: 2,
exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
})
faraday.request :json
faraday.response :raise_error
end
end
# Builds the EODHD ticker format: {SYMBOL}.{EXCHANGE}
def eodhd_symbol(symbol, exchange_operating_mic)
eodhd_exchange = MIC_TO_EODHD_EXCHANGE[exchange_operating_mic] if exchange_operating_mic.present?
if eodhd_exchange.present?
"#{symbol}.#{eodhd_exchange}"
elsif exchange_operating_mic.present?
"#{symbol}.#{exchange_operating_mic}"
else
"#{symbol}.US"
end
end
# Cache key for tracking daily API usage
def daily_cache_key
"eodhd:daily:#{Date.current}"
end
# Enforces the daily rate limit. Raises RateLimitError if the limit is exhausted.
# Uses atomic increment-then-check to avoid TOCTOU races between concurrent workers.
def enforce_daily_limit!
new_count = Rails.cache.increment(daily_cache_key, 1, expires_in: 24.hours).to_i
if new_count > max_requests_per_day
raise RateLimitError, "EODHD daily rate limit of #{max_requests_per_day} requests exhausted"
end
end
# throttle_request and min_request_interval provided by RateLimitable
def max_requests_per_day
ENV.fetch("EODHD_MAX_REQUESTS_PER_DAY", MAX_REQUESTS_PER_DAY).to_i
end
def check_api_error!(parsed)
return unless parsed.is_a?(Hash) && parsed["error"].present?
raise Error, "API error: #{parsed["error"]}"
end
end

View File

@@ -0,0 +1,168 @@
class Provider::Mfapi < Provider
include SecurityConcept, RateLimitable
extend SslConfigurable
Error = Class.new(Provider::Error)
InvalidSecurityPriceError = Class.new(Error)
RateLimitError = Class.new(Error)
# Minimum delay between requests
MIN_REQUEST_INTERVAL = 0.5
def initialize
# No API key required
end
def healthy?
with_provider_response do
response = client.get("#{base_url}/mf/125497/latest")
parsed = JSON.parse(response.body)
parsed.dig("meta", "scheme_name").present?
end
end
def usage
with_provider_response do
UsageData.new(
used: nil,
limit: nil,
utilization: nil,
plan: "Free (no key required)"
)
end
end
# ================================
# Securities
# ================================
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
with_provider_response do
throttle_request
response = client.get("#{base_url}/mf/search") do |req|
req.params["q"] = symbol
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
unless parsed.is_a?(Array)
raise Error, "Unexpected response format from search endpoint"
end
parsed.first(25).map do |fund|
Security.new(
symbol: fund["schemeCode"].to_s,
name: fund["schemeName"],
logo_url: nil,
exchange_operating_mic: "XBOM",
country_code: "IN",
currency: "INR"
)
end
end
end
def fetch_security_info(symbol:, exchange_operating_mic:)
with_provider_response do
throttle_request
response = client.get("#{base_url}/mf/#{CGI.escape(symbol)}/latest")
parsed = JSON.parse(response.body)
check_api_error!(parsed)
meta = parsed["meta"] || {}
SecurityInfo.new(
symbol: symbol,
name: meta["scheme_name"],
links: nil,
logo_url: nil,
description: [ meta["fund_house"], meta["scheme_category"] ].compact.join(" - "),
kind: "mutual fund",
exchange_operating_mic: exchange_operating_mic
)
end
end
def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
with_provider_response do
historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date - 7.days, end_date: date)
raise historical_data.error if historical_data.error.present?
raise InvalidSecurityPriceError, "No NAV found for scheme #{symbol} on or before #{date}" if historical_data.data.blank?
# Find exact date or closest previous
historical_data.data.select { |p| p.date <= date }.max_by(&:date) || historical_data.data.first
end
end
def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
with_provider_response do
throttle_request
response = client.get("#{base_url}/mf/#{CGI.escape(symbol)}") do |req|
req.params["startDate"] = start_date.to_s
req.params["endDate"] = end_date.to_s
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
nav_data = parsed["data"]
if nav_data.nil? || !nav_data.is_a?(Array)
raise InvalidSecurityPriceError, "No NAV data returned for scheme #{symbol}"
end
nav_data.filter_map do |entry|
nav = entry["nav"]
date_str = entry["date"]
next if nav.nil? || nav.to_f <= 0 || date_str.blank?
# MFAPI returns dates as DD-MM-YYYY
date = Date.strptime(date_str, "%d-%m-%Y")
Price.new(
symbol: symbol,
date: date,
price: nav.to_f,
currency: "INR",
exchange_operating_mic: exchange_operating_mic
)
end
end
end
private
def base_url
ENV["MFAPI_URL"] || "https://api.mfapi.in"
end
def client
@client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
faraday.request(:retry, {
max: 3,
interval: 1.0,
interval_randomness: 0.5,
backoff_factor: 2,
exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
})
faraday.request :json
faraday.response :raise_error
faraday.headers["Accept"] = "application/json"
end
end
# throttle_request and min_request_interval provided by RateLimitable
def check_api_error!(parsed)
return unless parsed.is_a?(Hash)
if parsed["status"] == "ERROR" || parsed["status"] == "FAIL"
raise Error, "API error: #{parsed['message'] || parsed['status']}"
end
end
end

View File

@@ -0,0 +1,59 @@
# Shared concern for providers that need interval-based request throttling
# and a standard error transformation pattern.
#
# Providers that include this concern get:
# - `throttle_request`: sleeps to enforce MIN_REQUEST_INTERVAL between calls
# - `min_request_interval`: reads from ENV with fallback to the class constant
# - `default_error_transformer`: maps Faraday/rate-limit errors to provider-scoped types
#
# The including class MUST define:
# - `MIN_REQUEST_INTERVAL` (Float) — default seconds between requests
# - `Error` (Class) — provider-scoped error class
# - `RateLimitError` (Class) — provider-scoped rate-limit error class
#
# And MAY define a `PROVIDER_ENV_PREFIX` constant (e.g. "ALPHA_VANTAGE") used
# to derive the ENV key for the min request interval override. When omitted
# the prefix is derived from the class name (Provider::AlphaVantage → "ALPHA_VANTAGE").
module Provider::RateLimitable
extend ActiveSupport::Concern
private
# Enforces a minimum interval between consecutive requests on this instance.
# Subclasses that need additional rate-limit layers (daily counters, hourly
# counters) should call `super` or invoke this via `throttle_interval` and
# add their own checks.
def throttle_request
@last_request_time ||= Time.at(0)
elapsed = Time.current - @last_request_time
sleep_time = min_request_interval - elapsed
sleep(sleep_time) if sleep_time > 0
@last_request_time = Time.current
end
def min_request_interval
ENV.fetch("#{provider_env_prefix}_MIN_REQUEST_INTERVAL", self.class::MIN_REQUEST_INTERVAL).to_f
end
def provider_env_prefix
self.class.const_defined?(:PROVIDER_ENV_PREFIX) ? self.class::PROVIDER_ENV_PREFIX : self.class.name.demodulize.underscore.upcase
end
# Standard error transformation: maps common Faraday errors to provider-scoped
# error classes. Providers with extra error types (e.g. AuthenticationError)
# should override and call `super` for the default cases.
def default_error_transformer(error)
case error
when self.class::RateLimitError
error
when Faraday::TooManyRequestsError
self.class::RateLimitError.new(
"#{self.class.name.demodulize} rate limit exceeded",
details: error.response&.dig(:body)
)
when Faraday::Error
self.class::Error.new(error.message, details: error.response&.dig(:body))
else
self.class::Error.new(error.message)
end
end
end

View File

@@ -81,6 +81,34 @@ class Provider::Registry
def yahoo_finance
Provider::YahooFinance.new
end
def tiingo
api_key = ENV["TIINGO_API_KEY"].presence || Setting.tiingo_api_key # pipelock:ignore
return nil unless api_key.present?
Provider::Tiingo.new(api_key)
end
def eodhd
api_key = ENV["EODHD_API_KEY"].presence || Setting.eodhd_api_key # pipelock:ignore
return nil unless api_key.present?
Provider::Eodhd.new(api_key)
end
def alpha_vantage
api_key = ENV["ALPHA_VANTAGE_API_KEY"].presence || Setting.alpha_vantage_api_key # pipelock:ignore
return nil unless api_key.present?
Provider::AlphaVantage.new(api_key)
end
def mfapi
Provider::Mfapi.new
end
end
def initialize(concept)
@@ -92,6 +120,11 @@ class Provider::Registry
available_providers.map { |p| self.class.send(p) }.compact
end
# Returns the list of provider key names (symbols) registered for this concept.
def provider_keys
available_providers
end
def get_provider(name)
provider_method = available_providers.find { |p| p == name.to_sym }
@@ -108,7 +141,7 @@ class Provider::Registry
when :exchange_rates
%i[twelve_data yahoo_finance]
when :securities
%i[twelve_data yahoo_finance]
%i[twelve_data yahoo_finance tiingo eodhd alpha_vantage mfapi]
when :llm
%i[openai]
else

View File

@@ -1,7 +1,14 @@
module Provider::SecurityConcept
extend ActiveSupport::Concern
Security = Data.define(:symbol, :name, :logo_url, :exchange_operating_mic, :country_code)
# NOTE: This `Security` is a lightweight Data value object used for search results.
# Inside provider classes that `include SecurityConcept`, unqualified `Security`
# resolves to this Data class — NOT to `::Security` (the ActiveRecord model).
Security = Data.define(:symbol, :name, :logo_url, :exchange_operating_mic, :country_code, :currency) do
def initialize(symbol:, name:, logo_url:, exchange_operating_mic:, country_code:, currency: nil)
super
end
end
SecurityInfo = Data.define(:symbol, :name, :links, :logo_url, :description, :kind, :exchange_operating_mic)
Price = Data.define(:symbol, :date, :price, :currency, :exchange_operating_mic)
@@ -20,4 +27,11 @@ module Provider::SecurityConcept
def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:)
raise NotImplementedError, "Subclasses must implement #fetch_security_prices"
end
# Maximum number of calendar days of historical data the provider can return.
# Callers should clamp start_date to avoid requesting data beyond this window.
# Override in subclasses with provider-specific limits.
def max_history_days
nil # nil means no known limit
end
end

View File

@@ -0,0 +1,294 @@
class Provider::Tiingo < Provider
include SecurityConcept, RateLimitable
extend SslConfigurable
# Subclass so errors caught in this provider are raised as Provider::Tiingo::Error
Error = Class.new(Provider::Error)
InvalidSecurityPriceError = Class.new(Error)
RateLimitError = Class.new(Error)
# Minimum delay between requests to avoid rate limiting (in seconds)
MIN_REQUEST_INTERVAL = 1.5
# Maximum unique symbols per month (Tiingo free tier limit)
MAX_SYMBOLS_PER_MONTH = 500
# Maximum requests per hour
MAX_REQUESTS_PER_HOUR = 1000
# Tiingo exchange names to MIC codes
TIINGO_EXCHANGE_TO_MIC = {
"NASDAQ" => "XNAS",
"NYSE" => "XNYS",
"NYSE ARCA" => "XARC",
"NYSE MKT" => "XASE",
"BATS" => "BATS",
"LSE" => "XLON",
"SHE" => "XSHE",
"SHG" => "XSHG",
"OTCMKTS" => "XOTC",
"OTCD" => "XOTC",
"PINK" => "XOTC"
}.freeze
# Tiingo asset types to normalized kinds
TIINGO_ASSET_TYPE_MAP = {
"Stock" => "common stock",
"ETF" => "etf",
"Mutual Fund" => "mutual fund"
}.freeze
def initialize(api_key)
@api_key = api_key # pipelock:ignore
end
def healthy?
with_provider_response do
response = client.get("#{base_url}/tiingo/daily/AAPL")
parsed = JSON.parse(response.body)
parsed.dig("ticker").present?
end
end
def usage
with_provider_response do
count_key = "tiingo:symbol_count:#{Date.current.strftime('%Y-%m')}"
symbols_used = Rails.cache.read(count_key).to_i
UsageData.new(
used: symbols_used,
limit: MAX_SYMBOLS_PER_MONTH,
utilization: (symbols_used.to_f / MAX_SYMBOLS_PER_MONTH * 100).round(1),
plan: "Free"
)
end
end
# ================================
# Securities
# ================================
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
with_provider_response do
throttle_request
response = client.get("#{base_url}/tiingo/utilities/search") do |req|
req.params["query"] = symbol
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
unless parsed.is_a?(Array)
raise Error, "Unexpected response format from search endpoint"
end
parsed.first(25).map do |security|
ticker = security["ticker"]
currency = security["priceCurrency"]
# Cache the API-returned currency so fetch_security_prices can use it
# without making a second search request
if currency.present? && ticker.present?
Rails.cache.write("tiingo:currency:#{ticker.upcase}", currency, expires_in: 24.hours)
end
Security.new(
symbol: ticker,
name: security["name"],
logo_url: nil,
exchange_operating_mic: map_exchange_to_mic(security["exchange"]),
country_code: security["countryCode"].presence || country_code,
currency: currency
)
end
end
end
def fetch_security_info(symbol:, exchange_operating_mic:)
with_provider_response do
throttle_request
track_symbol(symbol)
response = client.get("#{base_url}/tiingo/daily/#{CGI.escape(symbol)}")
parsed = JSON.parse(response.body)
check_api_error!(parsed)
# The daily metadata endpoint returns exchangeCode (e.g., "NYSE ARCA", "OTCD")
resolved_mic = exchange_operating_mic.presence || map_exchange_to_mic(parsed["exchangeCode"])
SecurityInfo.new(
symbol: parsed["ticker"] || symbol,
name: parsed["name"],
links: nil,
logo_url: nil,
description: parsed["description"],
kind: nil,
exchange_operating_mic: resolved_mic
)
end
end
def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
with_provider_response do
historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
raise historical_data.error if historical_data.error.present?
raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?
historical_data.data.first
end
end
def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
with_provider_response do
throttle_request
track_symbol(symbol)
response = client.get("#{base_url}/tiingo/daily/#{CGI.escape(symbol)}/prices") do |req|
req.params["startDate"] = start_date.to_s
req.params["endDate"] = end_date.to_s
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
unless parsed.is_a?(Array)
error_message = parsed.is_a?(Hash) ? (parsed["detail"] || "Unexpected response format") : "Unexpected response format"
raise InvalidSecurityPriceError, "API error: #{error_message}"
end
# Prefer cached currency from search results to avoid a second API call
cache_key = "tiingo:currency:#{symbol.upcase}"
currency = Rails.cache.read(cache_key) || fetch_currency_for_symbol(symbol)
parsed.map do |resp|
price = resp["close"]
date = resp["date"]
if price.nil? || price.to_f <= 0
Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}. Price data: #{price.inspect}")
next
end
Price.new(
symbol: symbol,
date: Date.parse(date),
price: price,
currency: currency,
exchange_operating_mic: exchange_operating_mic
)
end.compact
end
end
private
attr_reader :api_key
def base_url
ENV["TIINGO_URL"] || "https://api.tiingo.com"
end
def client
@client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
faraday.request(:retry, {
max: 3,
interval: 1.0,
interval_randomness: 0.5,
backoff_factor: 2,
exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
})
faraday.request :json
faraday.response :raise_error
faraday.headers["Authorization"] = "Token #{api_key}"
faraday.headers["Content-Type"] = "application/json"
end
end
# Adds hourly request counter on top of the interval throttle from RateLimitable.
def throttle_request
super
# Global per-hour request counter via cache (Redis).
# Atomic increment-then-check avoids the TOCTOU of read-check-increment.
hour_key = "tiingo:requests:#{Time.current.to_i / 3600}"
new_count = Rails.cache.increment(hour_key, 1, expires_in: 7200.seconds).to_i
if new_count >= max_requests_per_hour
raise RateLimitError, "Tiingo hourly request limit reached (#{new_count}/#{max_requests_per_hour})"
end
end
# Tracks unique symbols queried per month to stay within Tiingo's 500 symbols/month limit.
# Uses atomic set-if-absent (Redis SETNX) to eliminate the read-then-write race
# where two concurrent workers could both see the symbol as untracked and both
# increment the counter.
def track_symbol(symbol)
symbol_key = "tiingo:symbol:#{Date.current.strftime('%Y-%m')}:#{symbol.upcase}"
count_key = "tiingo:symbol_count:#{Date.current.strftime('%Y-%m')}"
# Atomic write-if-absent: returns false when the key already exists (Redis SETNX).
# Only the first worker to claim this symbol will proceed to increment the counter.
return unless Rails.cache.write(symbol_key, true, expires_in: 35.days, unless_exist: true)
new_count = Rails.cache.increment(count_key, 1, expires_in: 35.days).to_i
if new_count >= MAX_SYMBOLS_PER_MONTH
Rails.cache.decrement(count_key, 1)
Rails.cache.delete(symbol_key)
raise RateLimitError, "Tiingo unique symbol limit reached (#{MAX_SYMBOLS_PER_MONTH} per month)"
end
end
# min_request_interval provided by RateLimitable
def max_requests_per_hour
ENV.fetch("TIINGO_MAX_REQUESTS_PER_HOUR", MAX_REQUESTS_PER_HOUR).to_i
end
# Fetches the price currency for a symbol via the search endpoint.
# Only called as a fallback when the cache (populated by search_securities)
# doesn't have the currency. Raises on failure to avoid silently mislabeling
# non-USD instruments as USD.
def fetch_currency_for_symbol(symbol)
throttle_request
response = client.get("#{base_url}/tiingo/utilities/search") do |req|
req.params["query"] = symbol
end
parsed = JSON.parse(response.body)
check_api_error!(parsed)
if parsed.is_a?(Array)
match = parsed.find { |s| s["ticker"]&.upcase == symbol.upcase }
currency = match&.dig("priceCurrency")
if currency.present?
Rails.cache.write("tiingo:currency:#{symbol.upcase}", currency, expires_in: 24.hours)
return currency
end
end
raise Error, "Could not determine currency for #{symbol} from Tiingo search"
end
def map_exchange_to_mic(exchange_name)
return nil if exchange_name.blank?
TIINGO_EXCHANGE_TO_MIC[exchange_name.strip] || exchange_name.strip
end
def check_api_error!(parsed)
return unless parsed.is_a?(Hash) && parsed["detail"].present?
detail = parsed["detail"]
if detail.downcase.include?("rate limit") || detail.downcase.include?("too many")
raise RateLimitError, detail
end
raise Error, "API error: #{detail}"
end
end

View File

@@ -157,7 +157,8 @@ class Provider::TwelveData < Provider
name: security.dig("instrument_name"),
logo_url: nil,
exchange_operating_mic: security.dig("mic_code"),
country_code: country ? country.alpha2 : nil
country_code: country ? country.alpha2 : nil,
currency: security.dig("currency")
)
end
end
@@ -199,7 +200,8 @@ class Provider::TwelveData < Provider
with_provider_response do
historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
raise ProviderError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.empty?
raise historical_data.error if historical_data.error.present?
raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?
historical_data.data.first
end

View File

@@ -20,6 +20,10 @@ class Provider::YahooFinance < Provider
# Maximum lookback window for historical data (configurable)
MAX_LOOKBACK_WINDOW = 10.years
def max_history_days
(MAX_LOOKBACK_WINDOW / 1.day).to_i
end
# Minimum delay between requests to avoid rate limiting (in seconds)
MIN_REQUEST_INTERVAL = 0.5

View File

@@ -1,6 +1,9 @@
class Security < ApplicationRecord
include Provided, PlanRestrictionTracker
# Transient attribute for search results -- not persisted
attr_accessor :search_currency
# ISO 10383 MIC codes mapped to user-friendly exchange names
# Source: https://www.iso20022.org/market-identifier-codes
# Data stored in config/exchanges.yml
@@ -8,6 +11,13 @@ class Security < ApplicationRecord
KINDS = %w[standard cash].freeze
# Known securities provider keys — derived from the registry so adding a new
# provider to Registry#available_providers automatically allows it here.
# Evaluated at runtime (not boot) so runtime-enabled providers are accepted.
def self.valid_price_providers
Provider::Registry.for_concept(:securities).provider_keys.map(&:to_s)
end
before_validation :upcase_symbols
before_save :generate_logo_url_from_brandfetch, if: :should_generate_logo?
@@ -17,10 +27,17 @@ class Security < ApplicationRecord
validates :ticker, presence: true
validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }
validates :kind, inclusion: { in: KINDS }
validates :price_provider, inclusion: { in: ->(_) { Security.valid_price_providers } }, allow_nil: true
scope :online, -> { where(offline: false) }
scope :standard, -> { where(kind: "standard") }
# Parses the combobox ID format "SYMBOL|EXCHANGE|PROVIDER" into a hash.
def self.parse_combobox_id(value)
parts = value.to_s.split("|", 3)
{ ticker: parts[0].presence, exchange_operating_mic: parts[1].presence, price_provider: parts[2].presence }
end
# Lazily finds or creates a synthetic cash security for an account.
# Used as fallback when creating an interest Trade without a user-selected security.
def self.cash_for(account)
@@ -57,7 +74,9 @@ class Security < ApplicationRecord
name: name,
logo_url: logo_url,
exchange_operating_mic: exchange_operating_mic,
country_code: country_code
country_code: country_code,
price_provider: price_provider,
currency: search_currency
)
end

View File

@@ -1,10 +1,10 @@
class Security::ComboboxOption
include ActiveModel::Model
attr_accessor :symbol, :name, :logo_url, :exchange_operating_mic, :country_code
attr_accessor :symbol, :name, :logo_url, :exchange_operating_mic, :country_code, :price_provider, :currency
def id
"#{symbol}|#{exchange_operating_mic}"
"#{symbol}|#{exchange_operating_mic}|#{price_provider}"
end
def exchange_name

View File

@@ -66,11 +66,21 @@ class Security::HealthChecker
attr_reader :security
def provider
Security.provider
security.price_data_provider
end
# Some providers (e.g., Alpha Vantage) have very low daily limits and no
# lightweight endpoint — each health check burns a full API call that
# fetches ~100 data points. Skip health checks for those providers to
# avoid exhausting their quota on monitoring alone.
def skip_health_check?
provider.present? && provider.respond_to?(:max_history_days) &&
provider.is_a?(Provider::AlphaVantage)
end
def latest_provider_price
return nil unless provider.present?
return true if skip_health_check? # treat as healthy — quota too precious
response = provider.fetch_security_price(
symbol: security.ticker,
@@ -111,6 +121,7 @@ class Security::HealthChecker
Security.transaction do
security.update!(
offline: true,
offline_reason: "health_check_failed",
failed_fetch_count: MAX_CONSECUTIVE_FAILURES + 1,
failed_fetch_at: Time.current
)

View File

@@ -31,20 +31,20 @@ class Security::Price::Importer
prev_currency = prev_price_currency || db_price_currency || "USD"
unless prev_price_value.present?
Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{start_date}")
Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{fill_start_date}")
Sentry.capture_exception(MissingStartPriceError.new("Could not determine start price for ticker")) do |scope|
scope.set_tags(security_id: security.id)
scope.set_context("security", {
id: security.id,
start_date: start_date
start_date: fill_start_date
})
end
return 0
end
gapfilled_prices = effective_start_date.upto(end_date).map do |date|
gapfilled_prices = fill_start_date.upto(end_date).map do |date|
db_price = db_prices[date]
db_price_value = db_price&.price
provider_price = provider_prices[date]
@@ -101,15 +101,34 @@ class Security::Price::Importer
private
attr_reader :security, :security_provider, :start_date, :end_date, :clear_cache
# The start date sent to the provider API, clamped to the provider's max
# lookback window when applicable. Computed independently of provider_prices
# so fill_start_date can reference it without relying on method call order.
def provider_fetch_start_date
@provider_fetch_start_date ||= begin
base = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days
max_days = security_provider.respond_to?(:max_history_days) ? security_provider.max_history_days : nil
if max_days && (end_date - base).to_i > max_days
clamped = end_date - max_days.days
Rails.logger.info(
"#{security_provider.class.name} max history is #{max_days} days; " \
"clamping #{security.ticker} start_date from #{base} to #{clamped}"
)
clamped
else
base
end
end
end
def provider_prices
@provider_prices ||= begin
provider_fetch_start_date = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days
response = security_provider.fetch_security_prices(
symbol: security.ticker,
exchange_operating_mic: security.exchange_operating_mic,
start_date: provider_fetch_start_date,
end_date: end_date
end_date: end_date
)
if response.success?
@@ -175,9 +194,17 @@ class Security::Price::Importer
end || end_date
end
# The date the gap-fill loop starts from. When the provider's history was
# clamped (e.g. Alpha Vantage 140 days), we start from the clamped window
# instead of the original effective_start_date to avoid writing hundreds of
# LOCF-filled prices for dates the provider can't actually serve.
def fill_start_date
@fill_start_date ||= [ provider_fetch_start_date, effective_start_date ].max
end
def start_price_value
# When processing full range (first sync), use original behavior
if effective_start_date == start_date
if fill_start_date == start_date
provider_price_value = provider_prices.select { |date, _| date <= start_date }
.max_by { |date, _| date }
&.last&.price
@@ -188,9 +215,8 @@ class Security::Price::Importer
return nil
end
# For partial range (effective_start_date > start_date), use recent data
# This prevents stale prices from old trade dates propagating to current gap-fills
cutoff_date = effective_start_date
# For partial range or clamped range, use the most recent data before fill_start_date
cutoff_date = fill_start_date
# First try provider prices (most recent before cutoff)
provider_price_value = provider_prices

View File

@@ -4,50 +4,187 @@ module Security::Provided
SecurityInfoMissingError = Class.new(StandardError)
class_methods do
def provider
provider = ENV["SECURITIES_PROVIDER"].presence || Setting.securities_provider
registry = Provider::Registry.for_concept(:securities)
registry.get_provider(provider.to_sym)
# Returns all enabled and configured securities providers
def providers
Setting.enabled_securities_providers.filter_map do |name|
Provider::Registry.for_concept(:securities).get_provider(name.to_sym)
rescue Provider::Registry::Error
nil
end
end
# Backward compat: first enabled provider
def provider
providers.first
end
# Get a specific provider by key name (e.g., "finnhub", "twelve_data")
# Returns nil if the provider is disabled in settings or not configured.
def provider_for(name)
return nil if name.blank?
return nil unless Setting.enabled_securities_providers.include?(name.to_s)
Provider::Registry.for_concept(:securities).get_provider(name.to_sym)
rescue Provider::Registry::Error
nil
end
# Cache duration for search results (avoids burning through provider rate limits)
SEARCH_CACHE_TTL = 5.minutes
# Maximum number of results returned to the combobox dropdown
MAX_SEARCH_RESULTS = 30
# Per-provider timeout so one slow provider can't stall the entire search
PROVIDER_SEARCH_TIMEOUT = 8.seconds
def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)
return [] if provider.nil? || symbol.blank?
return [] if symbol.blank?
active_providers = providers.compact
return [] if active_providers.empty?
params = {
country_code: country_code,
exchange_operating_mic: exchange_operating_mic
}.compact_blank
response = provider.search_securities(symbol, **params)
if response.success?
securities = response.data.map do |provider_security|
# Need to map to domain model so Combobox can display via to_combobox_option
Security.new(
ticker: provider_security.symbol,
name: provider_security.name,
logo_url: provider_security.logo_url,
exchange_operating_mic: provider_security.exchange_operating_mic,
country_code: provider_security.country_code
)
# Query all providers concurrently so the total wall time is max(provider
# latencies) instead of sum. Each future runs in the concurrent-ruby thread
# pool, keeping Puma threads unblocked during individual provider sleeps.
futures = active_providers.map do |prov|
Concurrent::Promises.future(prov) do |provider|
fetch_provider_results(provider, symbol, params)
end
# Sort results to prioritize user's country if provided
if country_code.present?
user_country = country_code.upcase
securities.sort_by do |s|
[
s.country_code&.upcase == user_country ? 0 : 1, # User's country first
s.ticker.upcase == symbol.upcase ? 0 : 1 # Exact ticker match second
]
end
else
securities
end
else
[]
end
# Collect results from each future individually with a shared deadline.
# Unlike zip (which is all-or-nothing), this keeps results from fast
# providers even when a slow one times out.
deadline = Time.current + PROVIDER_SEARCH_TIMEOUT
results_array = futures.map do |future|
remaining = [ (deadline - Time.current), 0 ].max
future.value(remaining)
end
all_results = []
seen_keys = Set.new
results_array.each_with_index do |provider_results, idx|
next if provider_results.nil?
provider_key = provider_key_for(active_providers[idx])
provider_results.each do |ps|
# Dedup key includes provider so the same ticker on the same exchange can
# appear once per provider — the user picks which provider's price feed
# they want and that choice is stored in price_provider.
dedup_key = "#{ps[:symbol]}|#{ps[:exchange_operating_mic]}|#{provider_key}".upcase
next if seen_keys.include?(dedup_key)
seen_keys.add(dedup_key)
security = Security.new(
ticker: ps[:symbol],
name: ps[:name],
logo_url: ps[:logo_url],
exchange_operating_mic: ps[:exchange_operating_mic],
country_code: ps[:country_code],
search_currency: ps[:currency],
price_provider: provider_key
)
all_results << security
end
end
if all_results.empty? && active_providers.any?
Rails.logger.warn("Security search: all #{active_providers.size} providers returned no results for '#{symbol}'")
end
rank_search_results(all_results, symbol, country_code).first(MAX_SEARCH_RESULTS)
end
private
def provider_key_for(provider_instance)
provider_instance.class.name.demodulize.underscore
end
# Fetches (or reads from cache) search results for a single provider.
# Designed to run inside a Concurrent::Promises.future.
def fetch_provider_results(prov, symbol, params)
provider_key = provider_key_for(prov)
cache_key = "security_search:#{provider_key}:#{symbol.upcase}:#{Digest::SHA256.hexdigest(params.sort_by { |k, _| k }.to_json)}"
Rails.cache.fetch(cache_key, expires_in: SEARCH_CACHE_TTL, skip_nil: true) do
response = prov.search_securities(symbol, **params)
next nil unless response.success?
response.data.map do |ps|
{ symbol: ps.symbol, name: ps.name, logo_url: ps.logo_url,
exchange_operating_mic: ps.exchange_operating_mic, country_code: ps.country_code,
currency: ps.respond_to?(:currency) ? ps.currency : nil }
end
end
rescue => e
Rails.logger.warn("Security search failed for #{provider_key}: #{e.message}")
nil
end
# Scores and sorts search results so the most relevant matches appear first.
# Scoring criteria (lower = better):
# 0: exact ticker match
# 1: ticker starts with query
# 2: name contains query
# 3: everything else
# Within the same relevance tier, user's country is preferred.
def rank_search_results(results, symbol, country_code)
query = symbol.upcase
user_country = country_code&.upcase
results.sort_by do |s|
ticker_up = s.ticker.upcase
relevance = if ticker_up == query
0
elsif ticker_up.start_with?(query)
1
elsif s.name&.upcase&.include?(query)
2
else
3
end
country_match = (user_country.present? && s.country_code&.upcase == user_country) ? 0 : 1
[ relevance, country_match, ticker_up ]
end
end
end
# Public method: resolves the provider for this specific security.
# Uses the security's assigned price_provider if available and configured.
# Falls back to the first enabled provider only when no specific provider
# was ever assigned. When an assigned provider becomes unavailable, returns
# nil so the security is skipped rather than queried against an incompatible
# provider (e.g. MFAPI scheme codes sent to TwelveData).
def price_data_provider
if price_provider.present?
assigned = self.class.provider_for(price_provider)
return assigned if assigned.present?
return nil # assigned provider is unavailable — don't silently fall back
end
self.class.providers.first
end
# Returns the health status of this security's provider link.
# Delegates to price_data_provider to avoid duplicating provider lookup logic.
def provider_status
resolved = price_data_provider
# Had a specific provider assigned but it's now unavailable
return :provider_unavailable if resolved.nil? && price_provider.present?
return :offline if offline?
return :no_provider if resolved.nil?
return :stale if failed_fetch_count.to_i > 0
:ok
end
def find_or_fetch_price(date: Date.current, cache: true)
@@ -59,8 +196,8 @@ module Security::Provided
return nil if offline?
# Make sure we have a data provider before fetching
return nil unless provider.present?
response = provider.fetch_security_price(
return nil unless price_data_provider.present?
response = price_data_provider.fetch_security_price(
symbol: ticker,
exchange_operating_mic: exchange_operating_mic,
date: date
@@ -79,7 +216,7 @@ module Security::Provided
end
def import_provider_details(clear_cache: false)
unless provider.present?
unless price_data_provider.present?
Rails.logger.warn("No provider configured for Security.import_provider_details")
return
end
@@ -88,19 +225,21 @@ module Security::Provided
return
end
response = provider.fetch_security_info(
response = price_data_provider.fetch_security_info(
symbol: ticker,
exchange_operating_mic: exchange_operating_mic
)
if response.success?
update(
name: response.data.name,
logo_url: response.data.logo_url,
website_url: response.data.links
)
# Only overwrite fields the provider actually returned, so providers that
# don't support metadata (e.g. Alpha Vantage) won't blank existing values.
attrs = {}
attrs[:name] = response.data.name if response.data.name.present?
attrs[:logo_url] = response.data.logo_url if response.data.logo_url.present?
attrs[:website_url] = response.data.links if response.data.links.present?
update(attrs) if attrs.any?
else
Rails.logger.warn("Failed to fetch security info for #{ticker} from #{provider.class.name}: #{response.error.message}")
Rails.logger.warn("Failed to fetch security info for #{ticker} from #{price_data_provider.class.name}: #{response.error.message}")
Sentry.capture_exception(SecurityInfoMissingError.new("Failed to get security info"), level: :warning) do |scope|
scope.set_tags(security_id: self.id)
scope.set_context("security", { id: self.id, provider_error: response.error.message })
@@ -109,23 +248,18 @@ module Security::Provided
end
def import_provider_prices(start_date:, end_date:, clear_cache: false)
unless provider.present?
unless price_data_provider.present?
Rails.logger.warn("No provider configured for Security.import_provider_prices")
return 0
end
importer = Security::Price::Importer.new(
security: self,
security_provider: provider,
security_provider: price_data_provider,
start_date: start_date,
end_date: end_date,
clear_cache: clear_cache
)
[ importer.import_provider_prices, importer.provider_error ]
end
private
def provider
self.class.provider
end
end

View File

@@ -1,8 +1,9 @@
class Security::Resolver
def initialize(symbol, exchange_operating_mic: nil, country_code: nil)
def initialize(symbol, exchange_operating_mic: nil, country_code: nil, price_provider: nil)
@symbol = validate_symbol!(symbol)
@exchange_operating_mic = exchange_operating_mic
@country_code = country_code
@price_provider = validated_price_provider(price_provider)
end
# Attempts several paths to resolve a security:
@@ -20,13 +21,22 @@ class Security::Resolver
end
private
attr_reader :symbol, :exchange_operating_mic, :country_code
attr_reader :symbol, :exchange_operating_mic, :country_code, :price_provider
def validate_symbol!(symbol)
raise ArgumentError, "Symbol is required and cannot be blank" if symbol.blank?
symbol.strip.upcase
end
# Only accept price_provider values that are known and currently enabled.
# Prevents tampered combobox values from persisting invalid provider names.
def validated_price_provider(value)
return nil if value.blank?
return nil unless Security.valid_price_providers.include?(value.to_s)
return nil unless Setting.enabled_securities_providers.include?(value.to_s)
value.to_s
end
def offline_security
security = Security.find_or_initialize_by(
ticker: symbol,
@@ -44,13 +54,26 @@ class Security::Resolver
end
def exact_match_from_db
Security.find_by(
security = Security.find_by(
{
ticker: symbol,
exchange_operating_mic: exchange_operating_mic,
country_code: country_code.presence
}.compact
)
return nil unless security
# When the caller provides an explicit provider (e.g. user selected from
# search results), honor that choice. Automated syncs (Plaid, SimpleFIN)
# pass price_provider: nil and will not overwrite.
if price_provider.present? && security.price_provider != price_provider
security.update!(price_provider: price_provider)
end
reactivate_if_provider_available!(security)
security
end
# If provided a ticker + exchange (and optionally, a country code), we can find exact matches
@@ -59,8 +82,8 @@ class Security::Resolver
return nil unless exchange_operating_mic.present?
match = provider_search_result.find do |s|
ticker_matches = s.ticker.upcase.to_s == symbol.upcase.to_s
exchange_matches = s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s
ticker_matches = s.ticker&.upcase.to_s == symbol.upcase.to_s
exchange_matches = s.exchange_operating_mic&.upcase.to_s == exchange_operating_mic.upcase.to_s
if country_code && exchange_operating_mic
ticker_matches && exchange_matches && s.country_code&.upcase.to_s == country_code.upcase.to_s
@@ -88,8 +111,8 @@ class Security::Resolver
# 4. Rank by exchange_operating_mic relevance (lower index in the list is more relevant)
sorted_candidates = filtered_candidates.sort_by do |s|
[
s.ticker.upcase.to_s == symbol.upcase.to_s ? 0 : 1,
exchange_operating_mic.present? && s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
s.ticker&.upcase.to_s == symbol.upcase.to_s ? 0 : 1,
exchange_operating_mic.present? && s.exchange_operating_mic&.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
sorted_country_codes_by_relevance.index(s.country_code&.upcase.to_s) || sorted_country_codes_by_relevance.length,
sorted_exchange_operating_mics_by_relevance.index(s.exchange_operating_mic&.upcase.to_s) || sorted_exchange_operating_mics_by_relevance.length
]
@@ -109,11 +132,35 @@ class Security::Resolver
)
security.country_code = match.country_code
# Set provider when explicitly provided (user selection) or when the
# record is new / has no provider yet. Automated syncs pass nil and
# will not overwrite an existing choice.
effective_provider = price_provider.presence ||
(match.respond_to?(:price_provider) ? match.price_provider.presence : nil)
if effective_provider.present?
security.price_provider = effective_provider
end
security.save!
reactivate_if_provider_available!(security)
security
end
# If a security was marked offline (e.g. its provider was temporarily
# removed in settings) but now has a valid, enabled provider, bring it
# back online so the MarketDataImporter picks it up again.
def reactivate_if_provider_available!(security)
return unless security.offline?
return unless security.offline_reason == "provider_disabled"
return unless security.price_data_provider.present?
security.update!(offline: false, offline_reason: nil, failed_fetch_count: 0, failed_fetch_at: nil)
end
def provider_search_result
params = {
exchange_operating_mic: exchange_operating_mic,

View File

@@ -36,6 +36,83 @@ class Setting < RailsSettings::Base
field :exchange_rate_provider, type: :string, default: ENV.fetch("EXCHANGE_RATE_PROVIDER", "twelve_data")
field :securities_provider, type: :string, default: ENV.fetch("SECURITIES_PROVIDER", "twelve_data")
# Multi-provider: comma-separated list of enabled securities providers
field :securities_providers, type: :string, default: ENV.fetch("SECURITIES_PROVIDERS", "")
# New provider API keys (encrypted at rest — see EncryptedSettingFields below)
field :tiingo_api_key, type: :string, default: ENV["TIINGO_API_KEY"]
field :eodhd_api_key, type: :string, default: ENV["EODHD_API_KEY"]
field :alpha_vantage_api_key, type: :string, default: ENV["ALPHA_VANTAGE_API_KEY"]
# Transparent encryption for API key fields. The `field` macro defines the
# raw getter/setter on the class. By prepending this module we intercept
# reads (decrypt) and writes (encrypt) while `super` delegates to the
# original getter/setter generated by rails-settings-cached.
#
# Backward-compatible: if decryption fails (e.g. the value was stored before
# encryption was enabled) the raw value is returned as-is.
module EncryptedSettingFields
ENCRYPTED_FIELDS = %i[
twelve_data_api_key
tiingo_api_key
eodhd_api_key
alpha_vantage_api_key
openai_access_token
external_assistant_token
].freeze
ENCRYPTED_FIELDS.each do |field_name|
define_method(field_name) do
raw = super()
decrypt_setting(raw)
end
define_method(:"#{field_name}=") do |value|
super(encrypt_setting(value))
end
end
private
def setting_encryptor
@setting_encryptor ||= begin
key = ActiveSupport::KeyGenerator.new(
Rails.application.secret_key_base
).generate_key("setting_encryption", 32)
ActiveSupport::MessageEncryptor.new(key)
end
end
def encrypt_setting(value)
return value if value.blank?
setting_encryptor.encrypt_and_sign(value)
end
def decrypt_setting(value)
return value if value.blank?
setting_encryptor.decrypt_and_verify(value)
rescue ActiveSupport::MessageVerifier::InvalidSignature,
ActiveSupport::MessageEncryptor::InvalidMessage
# Value was stored before encryption was enabled — return as-is.
# It will be re-encrypted on next write.
value
end
end
class << self
prepend EncryptedSettingFields
end
def self.enabled_securities_providers
plural = ENV["SECURITIES_PROVIDERS"].presence || securities_providers.presence
if plural.present?
plural.to_s.split(",").map(&:strip).reject(&:blank?)
else
# Backward compat: fall back to singular setting
[ ENV["SECURITIES_PROVIDER"].presence || securities_provider ].compact
end
end
# Sync settings - check both provider env vars for default
# Only defaults to true if neither provider explicitly disables pending
SYNCS_INCLUDE_PENDING_DEFAULT = begin

View File

@@ -22,11 +22,13 @@ class Trade::CreateForm
private
# Users can either look up a ticker from a provider or enter a manual, "offline" ticker (that we won't fetch prices for)
def security
ticker_symbol, exchange_operating_mic = ticker.present? ? ticker.split("|") : [ manual_ticker, nil ]
parsed = ticker.present? ? Security.parse_combobox_id(ticker) : { ticker: manual_ticker }
return nil if parsed[:ticker].blank?
Security::Resolver.new(
ticker_symbol,
exchange_operating_mic: exchange_operating_mic
parsed[:ticker],
exchange_operating_mic: parsed[:exchange_operating_mic],
price_provider: parsed[:price_provider]
).resolve
end

View File

@@ -20,13 +20,21 @@
<% dialog.with_body do %>
<% dialog.with_section(title: t(".overview"), open: true) do %>
<div class="pb-4">
<% if @holding.security.provider_status == :provider_unavailable %>
<div class="px-3 pt-2">
<%= render DS::Alert.new(
message: t(".provider_disabled_warning", provider: @holding.security.price_provider&.humanize || "Unknown"),
variant: :warning
) %>
</div>
<% end %>
<dl class="space-y-3 px-3 py-2">
<div data-controller="holding-security-remap">
<div class="flex items-center justify-between text-sm">
<dt class="text-secondary"><%= t(".security_label") %></dt>
<dd class="text-primary flex items-center gap-2">
<%= @holding.ticker %>
<% if @holding.security_remapped? %>
<% if @holding.security_remapped? && @holding.account.linked? %>
<span class="text-xs text-secondary">
(<%= t(".originally", ticker: @holding.provider_security.ticker) %>)
</span>
@@ -41,7 +49,7 @@
</dd>
</div>
<div data-holding-security-remap-target="form" class="hidden mt-3 space-y-3">
<% if Security.provider.present? %>
<% if Security.providers.any? %>
<%= form_with url: remap_security_holding_path(@holding), method: :patch, class: "space-y-3" do |f| %>
<div class="form-field combobox">
<%= f.combobox :security_id,
@@ -229,10 +237,31 @@
</div>
</div>
<% end %>
<% if @holding.cost_basis_locked? || @holding.security_remapped? || @holding.account.can_delete_holdings? || !@holding.security.offline? %>
<% if @holding.cost_basis_locked? || (@holding.security_remapped? && @holding.account.linked?) || @holding.account.can_delete_holdings? || !@holding.security.offline? || @holding.security.provider_status == :provider_unavailable %>
<% dialog.with_section(title: t(".settings"), open: true) do %>
<div class="pb-4">
<% if @holding.security_remapped? %>
<% if @holding.security.provider_status == :provider_unavailable %>
<div class="flex items-center justify-between gap-2 p-3 border-b border-tertiary">
<div class="text-sm space-y-1 flex-1">
<h4 class="text-primary"><%= t(".switch_provider_label") %></h4>
<p class="text-secondary"><%= t(".switch_provider_description", provider: @holding.security.price_provider&.humanize || "Unknown") %></p>
<% if Security.providers.any? %>
<%= form_with url: remap_security_holding_path(@holding), method: :patch, data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "change" } do |f| %>
<div class="form-field combobox mt-2">
<%= f.combobox :security_id,
securities_path(country_code: Current.family.country),
id: "switch_provider_security_id",
placeholder: t(".search_security_placeholder"),
data: { "auto-submit-form-target": "auto" } %>
</div>
<% end %>
<% else %>
<p class="text-xs text-secondary mt-2"><%= t(".no_security_provider") %></p>
<% end %>
</div>
</div>
<% end %>
<% if @holding.security_remapped? && @holding.account.linked? %>
<div class="flex items-center justify-between gap-2 p-3 border-b border-tertiary">
<div class="text-sm space-y-1">
<h4 class="text-primary"><%= t(".security_remapped_label") %></h4>

View File

@@ -27,7 +27,7 @@
<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name" } %>
<% unless Security.provider %>
<% unless Security.providers.any? %>
<div class="alert alert-warning">
<p>
<strong>Note:</strong> The security prices provider is not configured. Your trade imports will work, but Sure will not backfill price history. Please go to your settings to configure this.

View File

@@ -9,18 +9,28 @@
</span>
<span class="text-xs text-secondary">
<%= t("securities.combobox.exchange_label", symbol: combobox_security.symbol, exchange: combobox_security.exchange_name) %>
<% if combobox_security.price_provider.present? %>
<span class="text-xs text-tertiary">&middot; <%= t("securities.providers.#{combobox_security.price_provider}", default: combobox_security.price_provider.humanize) %></span>
<% end %>
</span>
</div>
<% if combobox_security.country_code.present? %>
<div class="flex items-center bg-container-inset rounded-sm px-1.5 py-1 gap-1">
<%= image_tag("https://hatscripts.github.io/circle-flags/flags/#{combobox_security.country_code.downcase}.svg",
class: "h-4 rounded-sm",
alt: "#{combobox_security.country_code.upcase} flag",
title: combobox_security.country_code.upcase) %>
<span class="text-xs text-secondary">
<%= combobox_security.country_code.upcase %>
<div class="flex items-center gap-1.5">
<% if combobox_security.currency.present? %>
<span class="text-xs text-secondary bg-container-inset rounded-sm px-1.5 py-1">
<%= combobox_security.currency %>
</span>
</div>
<% end %>
<% end %>
<% if combobox_security.country_code.present? %>
<div class="flex items-center bg-container-inset rounded-sm px-1.5 py-1 gap-1">
<%= image_tag("https://hatscripts.github.io/circle-flags/flags/#{combobox_security.country_code.downcase}.svg",
class: "h-4 rounded-sm",
alt: "#{combobox_security.country_code.upcase} flag",
title: combobox_security.country_code.upcase) %>
<span class="text-xs text-secondary">
<%= combobox_security.country_code.upcase %>
</span>
</div>
<% end %>
</div>
</div>
</div>

View File

@@ -0,0 +1,46 @@
<div class="space-y-4">
<div>
<h2 class="font-medium mb-1"><%= t(".title") %></h2>
<% if ENV["ALPHA_VANTAGE_API_KEY"].present? %>
<p class="text-sm text-secondary"><%= t(".env_configured_message") %></p>
<% else %>
<div class="text-secondary text-sm mb-4">
<span><%= t(".description") %></span>
<details class="inline">
<summary class="cursor-pointer font-medium text-secondary underline inline"> <%= t(".show_details") %></summary>
<ol class="text-sm text-secondary mt-2 list-decimal ml-6 space-y-2">
<li><%= t(".step_1_html") %></li>
<li><%= t(".step_2") %></li>
</ol>
</details>
</div>
<% end %>
</div>
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "blur"
} do |form| %>
<% has_key = ENV["ALPHA_VANTAGE_API_KEY"].present? || Setting.alpha_vantage_api_key.present? %>
<%= form.text_field :alpha_vantage_api_key,
label: t(".label"),
type: "password",
placeholder: t(".placeholder"),
value: has_key ? "********" : "",
disabled: ENV["ALPHA_VANTAGE_API_KEY"].present?,
data: { "auto-submit-form-target": "auto" } %>
<% end %>
<div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
<div class="flex items-start gap-2">
<%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5") %>
<div class="text-sm text-amber-700">
<p><%= t(".rate_limit_warning") %></p>
<p class="mt-1"><%= t(".no_health_check_note") %></p>
</div>
</div>
</div>
</div>

View File

@@ -0,0 +1,46 @@
<div class="space-y-4">
<div>
<h2 class="font-medium mb-1"><%= t(".title") %></h2>
<% if ENV["EODHD_API_KEY"].present? %>
<p class="text-sm text-secondary"><%= t(".env_configured_message") %></p>
<% else %>
<div class="text-secondary text-sm mb-4">
<span><%= t(".description") %></span>
<details class="inline">
<summary class="cursor-pointer font-medium text-secondary underline inline"> <%= t(".show_details") %></summary>
<ol class="text-sm text-secondary mt-2 list-decimal ml-6 space-y-2">
<li><%= t(".step_1_html") %></li>
<li><%= t(".step_2_html") %></li>
<li><%= t(".step_3") %></li>
</ol>
</details>
</div>
<% end %>
</div>
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "blur"
} do |form| %>
<% has_key = ENV["EODHD_API_KEY"].present? || Setting.eodhd_api_key.present? %>
<%= form.text_field :eodhd_api_key,
label: t(".label"),
type: "password",
placeholder: t(".placeholder"),
value: has_key ? "********" : "",
disabled: ENV["EODHD_API_KEY"].present?,
data: { "auto-submit-form-target": "auto" } %>
<% end %>
<div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
<div class="flex items-start gap-2">
<%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5") %>
<p class="text-sm text-amber-700">
<%= t(".rate_limit_warning") %>
</p>
</div>
</div>
</div>

View File

@@ -1,17 +1,16 @@
<div class="space-y-4">
<div>
<h2 class="font-medium mb-1"><%= t(".title") %></h2>
<p class="text-secondary text-sm mb-4"><%= t(".description") %></p>
</div>
<div class="space-y-6">
<%# Exchange Rate Provider - single dropdown %>
<div class="space-y-2">
<h3 class="font-medium text-sm"><%= t(".exchange_rate_title") %></h3>
<p class="text-secondary text-xs mb-2"><%= t(".exchange_rate_description") %></p>
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "change"
} do |form| %>
<div class="space-y-4">
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "change"
} do |form| %>
<%= form.select :exchange_rate_provider,
[
[t(".providers.twelve_data"), "twelve_data"],
@@ -23,29 +22,61 @@
disabled: ENV["EXCHANGE_RATE_PROVIDER"].present?,
data: { "auto-submit-form-target": "auto" }
} %>
<% end %>
</div>
<%= form.select :securities_provider,
[
[t(".providers.twelve_data"), "twelve_data"],
[t(".providers.yahoo_finance"), "yahoo_finance"]
],
{ label: t(".securities_provider_label") },
{
value: ENV.fetch("SECURITIES_PROVIDER", Setting.securities_provider),
disabled: ENV["SECURITIES_PROVIDER"].present?,
data: { "auto-submit-form-target": "auto" }
} %>
</div>
<%# Securities Providers - multiple checkboxes %>
<div class="space-y-2">
<h3 class="font-medium text-sm"><%= t(".securities_title") %></h3>
<p class="text-secondary text-xs mb-2"><%= t(".securities_description") %></p>
<% if ENV["EXCHANGE_RATE_PROVIDER"].present? || ENV["SECURITIES_PROVIDER"].present? %>
<div class="mt-4 bg-warning-50 border border-warning-200 rounded-lg p-3">
<div class="flex items-start gap-2">
<%= icon("alert-circle", class: "w-5 h-5 text-warning-600 mt-0.5 shrink-0") %>
<p class="text-sm text-warning-800">
<%= t(".env_configured_message") %>
</p>
</div>
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form"
} do |form| %>
<% disabled = ENV["SECURITIES_PROVIDERS"].present? || ENV["SECURITIES_PROVIDER"].present? %>
<% enabled_providers = Setting.enabled_securities_providers %>
<div class="space-y-2">
<%# Hidden field to ensure empty array is submitted when all unchecked %>
<input type="hidden" name="setting[securities_providers][]" value="">
<% [
["twelve_data", t(".providers.twelve_data"), t(".twelve_data_hint")],
["yahoo_finance", t(".providers.yahoo_finance"), t(".yahoo_finance_hint")],
["tiingo", t(".providers.tiingo"), t(".requires_api_key")],
["eodhd", t(".providers.eodhd"), t(".requires_api_key_eodhd")],
["alpha_vantage", t(".providers.alpha_vantage"), t(".requires_api_key_alpha_vantage")],
["mfapi", t(".providers.mfapi"), t(".mfapi_hint")],
].each do |value, label, hint| %>
<label class="flex items-center gap-2 cursor-pointer">
<input type="checkbox"
name="setting[securities_providers][]"
value="<%= value %>"
class="rounded border-primary text-primary focus:ring-primary"
<%= "checked" if enabled_providers.include?(value) %>
<%= "disabled" if disabled %>
data-auto-submit-form-target="auto">
<span class="text-sm"><%= label %></span>
<% if hint %>
<span class="text-xs text-secondary">(<%= hint %>)</span>
<% end %>
</label>
<% end %>
</div>
<% end %>
</div>
<% if ENV["EXCHANGE_RATE_PROVIDER"].present? || ENV["SECURITIES_PROVIDERS"].present? || ENV["SECURITIES_PROVIDER"].present? %>
<div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
<div class="flex items-start gap-2">
<%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5 shrink-0") %>
<p class="text-sm text-amber-700">
<%= t(".env_configured_message") %>
</p>
</div>
</div>
<% end %>
</div>

View File

@@ -0,0 +1,37 @@
<div class="space-y-4">
<div>
<h2 class="font-medium mb-1"><%= t(".title") %></h2>
<% if ENV["TIINGO_API_KEY"].present? %>
<p class="text-sm text-secondary"><%= t(".env_configured_message") %></p>
<% else %>
<div class="text-secondary text-sm mb-4">
<span><%= t(".description") %></span>
<details class="inline">
<summary class="cursor-pointer font-medium text-secondary underline inline"> <%= t(".show_details") %></summary>
<ol class="text-sm text-secondary mt-2 list-decimal ml-6 space-y-2">
<li><%= t(".step_1_html") %></li>
<li><%= t(".step_2_html") %></li>
<li><%= t(".step_3") %></li>
</ol>
</details>
</div>
<% end %>
</div>
<%= styled_form_with model: Setting.new,
url: settings_hosting_path,
method: :patch,
data: {
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "blur"
} do |form| %>
<% has_key = ENV["TIINGO_API_KEY"].present? || Setting.tiingo_api_key.present? %>
<%= form.text_field :tiingo_api_key,
label: t(".label"),
type: "password",
placeholder: t(".placeholder"),
value: has_key ? "********" : "",
disabled: ENV["TIINGO_API_KEY"].present?,
data: { "auto-submit-form-target": "auto" } %>
<% end %>
</div>

View File

@@ -7,17 +7,11 @@
<div class="text-secondary text-sm mb-4">
<span><%= t(".description") %></span>
<details class="inline">
<summary class="cursor-pointer font-medium text-secondary underline inline"> (show details)</summary>
<summary class="cursor-pointer font-medium text-secondary underline inline"> <%= t(".show_details") %></summary>
<ol class="text-sm text-secondary mt-2 list-decimal ml-6 space-y-2">
<li>
Visit <a href="https://twelvedata.com/register" target="_blank" rel="noopener noreferrer" class="underline">twelvedata.com</a> and create a free Twelve Data Developer account.
</li>
<li>
Go to the <a href="https://twelvedata.com/account/api-keys" target="_blank" rel="noopener noreferrer" class="underline">API Keys</a> page.
</li>
<li>
Reveal your <strong>Secret Key</strong> and paste it below.
</li>
<li><%= t(".step_1_html") %></li>
<li><%= t(".step_2_html") %></li>
<li><%= t(".step_3") %></li>
</ol>
</details>
</div>
@@ -31,11 +25,12 @@
controller: "auto-submit-form",
"auto-submit-form-trigger-event-value": "blur"
} do |form| %>
<% has_key = ENV["TWELVE_DATA_API_KEY"].present? || Setting.twelve_data_api_key.present? %>
<%= form.text_field :twelve_data_api_key,
label: t(".label"),
type: "password",
placeholder: t(".placeholder"),
value: ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key),
value: has_key ? "********" : "",
disabled: ENV["TWELVE_DATA_API_KEY"].present?,
container_class: @twelve_data_usage.present? && !@twelve_data_usage.success? ? "border-red-500" : "",
data: { "auto-submit-form-target": "auto" } %>

View File

@@ -20,16 +20,12 @@
<%= t(".status_inactive") %>
</p>
</div>
<div class="bg-destructive-50 border border-destructive-200 rounded-lg p-3">
<div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
<div class="flex items-start gap-2">
<%= icon("alert-circle", class: "w-5 h-5 text-destructive-600 mt-0.5 shrink-0") %>
<%= icon("alert-triangle", size: "sm", class: "text-amber-600 mt-0.5 shrink-0") %>
<div>
<h3 class="text-sm font-medium text-destructive-800">
<%= t(".connection_failed") %>
</h3>
<div class="mt-2 text-sm text-destructive-700">
<p><%= t(".troubleshooting") %></p>
</div>
<h3 class="text-sm font-medium text-amber-700"><%= t(".connection_failed") %></h3>
<p class="text-sm text-amber-700 mt-1"><%= t(".troubleshooting") %></p>
</div>
</div>
</div>

View File

@@ -17,6 +17,15 @@
<% if @show_twelve_data_settings %>
<%= render "settings/hostings/twelve_data_settings" %>
<% end %>
<% if @show_tiingo_settings %>
<%= render "settings/hostings/tiingo_settings" %>
<% end %>
<% if @show_eodhd_settings %>
<%= render "settings/hostings/eodhd_settings" %>
<% end %>
<% if @show_alpha_vantage_settings %>
<%= render "settings/hostings/alpha_vantage_settings" %>
<% end %>
</div>
<% end %>
<%= settings_section title: t(".sync_settings") do %>

View File

@@ -9,8 +9,8 @@
<div class="form-field <%= options[:container_class] %>"
data-controller="money-field"
<% if options[:precision].present? %>data-money-field-precision-value="<%= options[:precision] %>"<% end %>
<% if options[:step].present? %>data-money-field-step-value="<%= options[:step] %>"<% end %>>
<% if options[:precision].present? %> data-money-field-precision-value="<%= options[:precision] %>"<% end %>
<% if options[:step].present? %> data-money-field-step-value="<%= options[:step] %>"<% end %>>
<% if options[:label_tooltip] %>
<div class="form-field__header">
<%= form.label options[:label] || t(".label"), class: "form-field__label" do %>

View File

@@ -25,7 +25,7 @@
}} %>
<% if %w[buy sell].include?(type) %>
<% if Security.provider.present? %>
<% if Security.providers.any? %>
<div class="form-field combobox">
<%= form.combobox :ticker,
securities_path(country_code: Current.family.country),

View File

@@ -8,6 +8,14 @@
<% trade = @entry.trade %>
<% dialog.with_body do %>
<%= render "entries/protection_indicator", entry: @entry, unlock_path: unlock_trade_path(trade) %>
<% if trade.security&.provider_status == :provider_unavailable %>
<div class="px-3 py-2">
<%= render DS::Alert.new(
message: t(".provider_disabled_warning", provider: trade.security.price_provider&.humanize || "Unknown"),
variant: :warning
) %>
</div>
<% end %>
<% dialog.with_section(title: t(".details"), open: true) do %>
<div class="pb-4">
<%= styled_form_with model: @entry,

View File

@@ -26,7 +26,7 @@
<% end %>
</div>
<div class="max-w-full <%= 'opacity-50 text-secondary' if entry.excluded %>">
<div class="max-w-full <%= "opacity-50 text-secondary" if entry.excluded %>">
<%= content_tag :div, class: ["flex items-center gap-3 lg:gap-4"] do %>
<div class="hidden lg:flex">
<% if transaction.merchant&.logo_url.present? %>
@@ -159,7 +159,7 @@
<% end %>
</div>
<div class="shrink-0 col-span-4 lg:col-span-2 ml-auto flex items-center justify-end gap-2 <%= 'opacity-50' if entry.excluded %>">
<div class="shrink-0 col-span-4 lg:col-span-2 ml-auto flex items-center justify-end gap-2 <%= "opacity-50" if entry.excluded %>">
<%# Protection indicator - shows on hover when entry is protected from sync %>
<% if entry.protected_from_sync? && !entry.excluded? %>
<%= link_to entry_path(entry),

View File

@@ -33,7 +33,7 @@
<div class="space-y-4">
<div class="space-y-1">
<%= f.label :security_id, t(".security_label"), class: "font-medium text-sm text-primary block" %>
<% if Security.provider.present? %>
<% if Security.providers.any? %>
<%# Always use searchable combobox when provider available - prevents picking wrong similar tickers %>
<div class="form-field combobox" style="--hw-handle-width: 0; --hw-handle-image: none;">
<%= f.combobox :ticker,
@@ -157,7 +157,7 @@
</div>
<%# Only show exchange field when no provider - combobox selections already include exchange %>
<% unless Security.provider.present? %>
<% unless Security.providers.any? %>
<div class="space-y-1">
<%= f.label :exchange_operating_mic, t(".exchange_label"), class: "font-medium text-sm text-primary block" %>
<%= f.text_field :exchange_operating_mic,

View File

@@ -71,6 +71,10 @@ en:
search_security_placeholder: Search by ticker or name
cancel: Cancel
remap_security: Save
provider_disabled_warning: "Price updates paused — %{provider} provider is disabled. Switch to another provider below or re-enable it in Settings."
switch_provider_label: Switch provider
switch_provider_description: "%{provider} is disabled. Search for this security from another enabled provider."
switch_provider_button: Switch
no_security_provider: Security provider not configured. Cannot search for securities.
security_remapped_label: Security remapped
provider_sent: "Provider sent: %{ticker}"

View File

@@ -4,3 +4,10 @@ en:
combobox:
display: "%{symbol} - %{name} (%{exchange})"
exchange_label: "%{symbol} (%{exchange})"
providers:
twelve_data: Twelve Data
yahoo_finance: Yahoo Finance
tiingo: Tiingo
eodhd: EODHD
alpha_vantage: Alpha Vantage
mfapi: MFAPI.in

View File

@@ -31,14 +31,25 @@ en:
title: Clear data cache?
body: Are you sure you want to clear the data cache? This will remove all exchange rates, security prices, account balances, and other data. This action cannot be undone.
provider_selection:
title: Provider Selection
description: Choose which service to use for fetching exchange rates and security prices. Yahoo Finance is free and doesn't require an API key. Twelve Data requires a free API key but may offer more data coverage.
exchange_rate_title: Exchange Rate Provider
exchange_rate_description: Select a single provider for fetching currency exchange rates.
exchange_rate_provider_label: Exchange Rate Provider
securities_provider_label: Securities (Stock Prices) Provider
env_configured_message: Provider selection is disabled because environment variables (EXCHANGE_RATE_PROVIDER or SECURITIES_PROVIDER) are set. To enable selection here, remove these environment variables from your configuration.
securities_title: Securities Providers
securities_description: Enable one or more providers for fetching stock, ETF, and mutual fund prices. When searching, all enabled providers are queried and results are merged.
env_configured_message: Provider selection is disabled because environment variables are set. To enable selection here, remove these environment variables from your configuration.
twelve_data_hint: requires API key, 800 credits/day
yahoo_finance_hint: free, no API key needed
requires_api_key: requires API key
requires_api_key_eodhd: requires API key, 20 calls/day limit
requires_api_key_alpha_vantage: requires API key, 25 calls/day limit
mfapi_hint: free, no API key -- Indian mutual funds only
providers:
twelve_data: Twelve Data
yahoo_finance: Yahoo Finance
tiingo: Tiingo
eodhd: EODHD
alpha_vantage: Alpha Vantage
mfapi: MFAPI.in
assistant_settings:
title: AI Assistant
description: Choose how the chat assistant responds. Builtin uses your configured LLM provider directly. External delegates to a remote AI agent that can call back to Sure's financial tools via MCP.
@@ -95,12 +106,48 @@ en:
status_inactive: Yahoo Finance connection failed
connection_failed: Unable to connect to Yahoo Finance
troubleshooting: Check your internet connection and firewall settings. Yahoo Finance may be temporarily unavailable.
tiingo_settings:
title: Tiingo
description: Enter the API token provided by Tiingo. Free tier supports 50 unique symbols per hour with 30+ years of historical data.
env_configured_message: Successfully configured through the TIINGO_API_KEY environment variable.
label: API Token
placeholder: Enter your Tiingo API token here
show_details: "(show details)"
step_1_html: 'Visit <a href="https://www.tiingo.com" target="_blank" rel="noopener noreferrer" class="underline">tiingo.com</a> and create a free account.'
step_2_html: 'Go to the <a href="https://www.tiingo.com/account/api/token" target="_blank" rel="noopener noreferrer" class="underline">API Token</a> page.'
step_3: Copy your API token and paste it below.
eodhd_settings:
title: EODHD
description: Enter the API token provided by EODHD. Supports EU ETFs on LSE, XETRA, and other international exchanges.
env_configured_message: Successfully configured through the EODHD_API_KEY environment variable.
label: API Token
placeholder: Enter your EODHD API token here
show_details: "(show details)"
step_1_html: 'Visit <a href="https://eodhd.com/register" target="_blank" rel="noopener noreferrer" class="underline">eodhd.com</a> and create a free account.'
step_2_html: 'Go to your <a href="https://eodhd.com/cp/dashboard" target="_blank" rel="noopener noreferrer" class="underline">Dashboard</a> to find your API token.'
step_3: Copy your API token and paste it below.
rate_limit_warning: "EODHD free tier is limited to 20 API calls per day. Best used as a supplementary provider for EU ETFs not available from other providers."
alpha_vantage_settings:
title: Alpha Vantage
description: Enter the API key from Alpha Vantage. Supports EU ETFs on London Stock Exchange, XETRA, and other exchanges.
env_configured_message: Successfully configured through the ALPHA_VANTAGE_API_KEY environment variable.
label: API Key
placeholder: Enter your Alpha Vantage API key here
show_details: "(show details)"
step_1_html: 'Visit <a href="https://www.alphavantage.co/support/#api-key" target="_blank" rel="noopener noreferrer" class="underline">alphavantage.co</a> and claim your free API key.'
step_2: Copy the API key and paste it below.
rate_limit_warning: "Alpha Vantage free tier is limited to 25 API calls per day. Best used as a supplementary provider for EU ETFs not available from other providers."
no_health_check_note: "Connection health check is unavailable for this provider due to the strict rate limit."
twelve_data_settings:
api_calls_used: "%{used} / %{limit} API daily calls used (%{percentage})"
description: Enter the API key provided by Twelve Data
env_configured_message: Successfully configured through the TWELVE_DATA_API_KEY environment variable.
label: API Key
placeholder: Enter your API key here
show_details: "(show details)"
step_1_html: 'Visit <a href="https://twelvedata.com/register" target="_blank" rel="noopener noreferrer" class="underline">twelvedata.com</a> and create a free Twelve Data Developer account.'
step_2_html: 'Go to the <a href="https://twelvedata.com/account/api-keys" target="_blank" rel="noopener noreferrer" class="underline">API Keys</a> page.'
step_3: Reveal your Secret Key and paste it below.
plan: "%{plan} plan"
plan_upgrade_warning_title: Some tickers require a paid plan
plan_upgrade_warning_description: The following tickers in your portfolio cannot sync prices with your current Twelve Data plan.

View File

@@ -45,6 +45,7 @@ en:
delete_subtitle: This action cannot be undone
delete_title: Delete Trade
details: Details
provider_disabled_warning: "Price updates paused — %{provider} provider is disabled. Re-enable it in Settings or remap the holding to another provider."
exclude_subtitle: This trade will not be included in reports and calculations
exclude_title: Exclude from analytics
no_category: No category

View File

@@ -0,0 +1,8 @@
class AddPriceProviderToSecurities < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
def change
add_column :securities, :price_provider, :string
add_index :securities, :price_provider, algorithm: :concurrently
end
end

View File

@@ -0,0 +1,8 @@
class AddOfflineReasonToSecurities < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
def change
add_column :securities, :offline_reason, :string
add_index :securities, [ :price_provider, :offline_reason ], algorithm: :concurrently
end
end

6
db/schema.rb generated
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2026_03_30_050801) do
ActiveRecord::Schema[7.2].define(version: 2026_04_08_151837) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -1215,10 +1215,14 @@ ActiveRecord::Schema[7.2].define(version: 2026_03_30_050801) do
t.datetime "last_health_check_at"
t.string "website_url"
t.string "kind", default: "standard", null: false
t.string "price_provider"
t.string "offline_reason"
t.index "upper((ticker)::text), COALESCE(upper((exchange_operating_mic)::text), ''::text)", name: "index_securities_on_ticker_and_exchange_operating_mic_unique", unique: true
t.index ["country_code"], name: "index_securities_on_country_code"
t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"
t.index ["kind"], name: "index_securities_on_kind"
t.index ["price_provider", "offline_reason"], name: "index_securities_on_price_provider_and_offline_reason"
t.index ["price_provider"], name: "index_securities_on_price_provider"
t.check_constraint "kind::text = ANY (ARRAY['standard'::character varying, 'cash'::character varying]::text[])", name: "chk_securities_kind"
end

View File

@@ -247,4 +247,71 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest
assert_equal I18n.t("settings.hostings.not_authorized"), flash[:alert]
end
end
# --- Securities provider toggle ---
test "can update securities providers" do
with_self_hosting do
patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "yahoo_finance" ] } }
assert_redirected_to settings_hosting_url
assert_equal "twelve_data,yahoo_finance", Setting.securities_providers
end
ensure
Setting.securities_providers = ""
end
test "filters out invalid provider names" do
with_self_hosting do
patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "fake_provider", "hacked" ] } }
assert_redirected_to settings_hosting_url
# Only valid providers are stored
enabled = Setting.enabled_securities_providers
assert_includes enabled, "twelve_data"
refute_includes enabled, "fake_provider"
refute_includes enabled, "hacked"
end
ensure
Setting.securities_providers = ""
end
test "removing a provider marks linked securities offline" do
with_self_hosting do
security = Security.create!(ticker: "CSPX", exchange_operating_mic: "XLON", price_provider: "tiingo", offline: false)
# First enable tiingo
Setting.securities_providers = "twelve_data,tiingo"
# Then remove tiingo
patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data" ] } }
security.reload
assert security.offline?, "Security should be marked offline when its provider is removed"
assert_equal "provider_disabled", security.offline_reason
end
ensure
Setting.securities_providers = ""
end
test "re-adding a provider brings securities back online" do
with_self_hosting do
security = Security.create!(
ticker: "CSPX2", exchange_operating_mic: "XLON",
price_provider: "tiingo", offline: true, offline_reason: "provider_disabled"
)
# Start without tiingo
Setting.securities_providers = "twelve_data"
# Re-add tiingo
patch settings_hosting_url, params: { setting: { securities_providers: [ "twelve_data", "tiingo" ] } }
security.reload
refute security.offline?, "Security should come back online when its provider is re-added"
assert_nil security.offline_reason
end
ensure
Setting.securities_providers = ""
end
end

View File

@@ -11,7 +11,7 @@ class Security::HealthCheckerTest < ActiveSupport::TestCase
Security.delete_all
@provider = mock
Security.stubs(:provider).returns(@provider)
Security.any_instance.stubs(:price_data_provider).returns(@provider)
# Brand new, no health check has been run yet
@new_security = Security.create!(

View File

@@ -6,9 +6,8 @@ class Security::PriceTest < ActiveSupport::TestCase
setup do
@provider = mock
Security.stubs(:provider).returns(@provider)
@security = securities(:aapl)
@security.stubs(:price_data_provider).returns(@provider)
end
test "finds single security price in DB" do

View File

@@ -0,0 +1,295 @@
require "test_helper"
class Security::ProvidedTest < ActiveSupport::TestCase
include ProviderTestHelper
setup do
@security = securities(:aapl)
end
# --- search_provider ---
test "search_provider returns results from multiple providers" do
provider_a = mock("provider_a")
provider_b = mock("provider_b")
result_a = Provider::SecurityConcept::Security.new(
symbol: "AAPL", name: "Apple Inc", logo_url: nil,
exchange_operating_mic: "XNAS", country_code: "US", currency: "USD"
)
result_b = Provider::SecurityConcept::Security.new(
symbol: "AAPL", name: "Apple Inc", logo_url: nil,
exchange_operating_mic: "XNAS", country_code: "US", currency: "USD"
)
provider_a.stubs(:class).returns(Provider::TwelveData)
provider_b.stubs(:class).returns(Provider::YahooFinance)
provider_a.expects(:search_securities).with("AAPL").returns(
provider_success_response([ result_a ])
)
provider_b.expects(:search_securities).with("AAPL").returns(
provider_success_response([ result_b ])
)
Security.stubs(:providers).returns([ provider_a, provider_b ])
results = Security.search_provider("AAPL")
# Same ticker+exchange but different providers → both appear (dedup includes provider key)
assert_equal 2, results.size
assert results.all? { |s| s.ticker == "AAPL" }
providers_in_results = results.map(&:price_provider).sort
assert_includes providers_in_results, "twelve_data"
assert_includes providers_in_results, "yahoo_finance"
end
test "search_provider deduplicates same ticker+exchange+provider" do
provider = mock("provider")
provider.stubs(:class).returns(Provider::TwelveData)
dup_result = Provider::SecurityConcept::Security.new(
symbol: "MSFT", name: "Microsoft", logo_url: nil,
exchange_operating_mic: "XNAS", country_code: "US", currency: "USD"
)
provider.expects(:search_securities).with("MSFT").returns(
provider_success_response([ dup_result, dup_result ])
)
Security.stubs(:providers).returns([ provider ])
results = Security.search_provider("MSFT")
assert_equal 1, results.size
end
test "search_provider returns empty array for blank symbol" do
assert_equal [], Security.search_provider("")
assert_equal [], Security.search_provider(nil)
end
test "search_provider returns empty array when no providers configured" do
Security.stubs(:providers).returns([])
assert_equal [], Security.search_provider("AAPL")
end
test "search_provider keeps fast provider results when slow provider times out" do
fast_provider = mock("fast_provider")
slow_provider = mock("slow_provider")
fast_provider.stubs(:class).returns(Provider::TwelveData)
slow_provider.stubs(:class).returns(Provider::YahooFinance)
fast_result = Provider::SecurityConcept::Security.new(
symbol: "SPY", name: "SPDR S&P 500", logo_url: nil,
exchange_operating_mic: "XNAS", country_code: "US", currency: "USD"
)
fast_provider.expects(:search_securities).with("SPY").returns(
provider_success_response([ fast_result ])
)
slow_provider.expects(:search_securities).with("SPY").returns(
provider_success_response([])
)
Security.stubs(:providers).returns([ fast_provider, slow_provider ])
results = Security.search_provider("SPY")
assert results.size >= 1, "Fast provider results should be returned even if slow provider returns nothing"
assert_equal "SPY", results.first.ticker
end
test "search_provider handles provider error gracefully" do
good_provider = mock("good_provider")
bad_provider = mock("bad_provider")
good_provider.stubs(:class).returns(Provider::TwelveData)
bad_provider.stubs(:class).returns(Provider::YahooFinance)
good_result = Provider::SecurityConcept::Security.new(
symbol: "GOOG", name: "Alphabet", logo_url: nil,
exchange_operating_mic: "XNAS", country_code: "US", currency: "USD"
)
good_provider.expects(:search_securities).with("GOOG").returns(
provider_success_response([ good_result ])
)
bad_provider.expects(:search_securities).with("GOOG").raises(StandardError, "API down")
Security.stubs(:providers).returns([ good_provider, bad_provider ])
results = Security.search_provider("GOOG")
assert_equal 1, results.size
assert_equal "GOOG", results.first.ticker
end
# --- price_data_provider ---
test "price_data_provider returns assigned provider" do
provider = mock("tiingo_provider")
Security.stubs(:provider_for).with("tiingo").returns(provider)
@security.update!(price_provider: "tiingo")
assert_equal provider, @security.price_data_provider
end
test "price_data_provider returns nil when assigned provider is disabled" do
Security.stubs(:provider_for).with("tiingo").returns(nil)
@security.update!(price_provider: "tiingo")
assert_nil @security.price_data_provider
end
test "price_data_provider falls back to first provider when none assigned" do
fallback_provider = mock("fallback")
Security.stubs(:providers).returns([ fallback_provider ])
@security.update!(price_provider: nil)
assert_equal fallback_provider, @security.price_data_provider
end
# --- provider_status ---
test "provider_status returns provider_unavailable when assigned provider disabled" do
Security.stubs(:provider_for).with("tiingo").returns(nil)
@security.update!(price_provider: "tiingo")
assert_equal :provider_unavailable, @security.provider_status
end
test "provider_status returns ok for healthy security" do
provider = mock("provider")
Security.stubs(:provider_for).with("twelve_data").returns(provider)
@security.update!(price_provider: "twelve_data", offline: false, failed_fetch_count: 0)
assert_equal :ok, @security.provider_status
end
# --- rank_search_results ---
# Helper to build unsaved Security objects for ranking tests
def build_result(ticker:, name: nil, country_code: nil, exchange_operating_mic: nil)
Security.new(
ticker: ticker,
name: name || ticker,
country_code: country_code,
exchange_operating_mic: exchange_operating_mic
)
end
def rank(results, query, country_code = nil)
Security.send(:rank_search_results, results, query, country_code)
end
test "ranking: AAPL exact match ranks above AAPL-prefixed and unrelated" do
results = [
build_result(ticker: "AAPLX", name: "Some AAPL Fund"),
build_result(ticker: "AAPL", name: "Apple Inc", country_code: "US", exchange_operating_mic: "XNAS"),
build_result(ticker: "AAPL", name: "Apple Inc", country_code: "GB", exchange_operating_mic: "XLON"),
build_result(ticker: "AAPLD", name: "AAPL Dividend ETF")
]
ranked = rank(results, "AAPL", "US")
# Exact matches first, US preferred over GB
assert_equal "AAPL", ranked[0].ticker
assert_equal "US", ranked[0].country_code
assert_equal "AAPL", ranked[1].ticker
assert_equal "GB", ranked[1].country_code
# Prefix matches after
assert ranked[2..].all? { |s| s.ticker.start_with?("AAPL") && s.ticker != "AAPL" }
end
test "ranking: Apple name search surfaces Apple Inc above unrelated" do
results = [
build_result(ticker: "PINEAPPLE", name: "Pineapple Corp"),
build_result(ticker: "AAPL", name: "Apple Inc", country_code: "US"),
build_result(ticker: "APLE", name: "Apple Hospitality REIT"),
build_result(ticker: "APPL", name: "Appell Petroleum")
]
ranked = rank(results, "Apple", "US")
# No ticker matches "APPLE", so all fall to name-contains or worse.
# "Apple Inc" and "Apple Hospitality" and "Pineapple" contain "APPLE" in name.
# "Appell Petroleum" does not contain "APPLE".
# Among name matches, alphabetical ticker breaks ties.
name_matches = ranked.select { |s| s.name.upcase.include?("APPLE") }
non_matches = ranked.reject { |s| s.name.upcase.include?("APPLE") }
assert name_matches.size >= 2
assert_equal non_matches, ranked.last(non_matches.size)
end
test "ranking: SPX exact match first, then SPX-prefixed" do
results = [
build_result(ticker: "SPXL", name: "Direxion Daily S&P 500 Bull 3X"),
build_result(ticker: "SPXS", name: "Direxion Daily S&P 500 Bear 3X"),
build_result(ticker: "SPX", name: "S&P 500 Index", country_code: "US"),
build_result(ticker: "SPXU", name: "ProShares UltraPro Short S&P 500")
]
ranked = rank(results, "SPX", "US")
assert_equal "SPX", ranked[0].ticker, "Exact match should be first"
assert ranked[1..].all? { |s| s.ticker.start_with?("SPX") }
end
test "ranking: VTTI exact match first regardless of country" do
results = [
build_result(ticker: "VTI", name: "Vanguard Total Stock Market ETF", country_code: "US"),
build_result(ticker: "VTTI", name: "VTTI Energy Partners", country_code: "US"),
build_result(ticker: "VTTIX", name: "Vanguard Target 2060 Fund")
]
ranked = rank(results, "VTTI", "US")
assert_equal "VTTI", ranked[0].ticker, "Exact match should be first"
assert_equal "VTTIX", ranked[1].ticker, "Prefix match second"
assert_equal "VTI", ranked[2].ticker, "Non-matching ticker last"
end
test "ranking: iShares S&P multi-word query is contiguous substring match" do
results = [
build_result(ticker: "IVV", name: "iShares S&P 500 ETF", country_code: "US"),
build_result(ticker: "CSPX", name: "iShares Core S&P 500 UCITS ETF", country_code: "GB"),
build_result(ticker: "IJH", name: "iShares S&P Mid-Cap ETF", country_code: "US"),
build_result(ticker: "UNRELATED", name: "Something Else Corp")
]
ranked = rank(results, "iShares S&P", "US")
# Only names containing the exact substring "iShares S&P" match tier 2.
# "iShares Core S&P" does NOT match (word "Core" breaks contiguity).
contiguous_matches = ranked.select { |s| s.name.upcase.include?("ISHARES S&P") }
assert_equal 2, contiguous_matches.size, "Only IVV and IJH contain the exact substring"
# US contiguous matches should come first
assert_equal "IJH", ranked[0].ticker # US, name match, alphabetically before IVV? No...
assert_includes [ "IVV", "IJH" ], ranked[0].ticker
assert_includes [ "IVV", "IJH" ], ranked[1].ticker
# Non-contiguous and unrelated should be last
assert_equal "UNRELATED", ranked.last.ticker
end
test "ranking: tesla name search finds TSLA" do
results = [
build_result(ticker: "TSLA", name: "Tesla Inc", country_code: "US"),
build_result(ticker: "TSLA", name: "Tesla Inc", country_code: "DE"),
build_result(ticker: "TL0", name: "Tesla Inc", country_code: "DE", exchange_operating_mic: "XETR"),
build_result(ticker: "TELSA", name: "Telsa Mining Ltd")
]
ranked = rank(results, "tesla", "US")
# No ticker matches "TESLA", so all go to name matching
# "Tesla Inc" contains "TESLA" → tier 2, US preferred
assert_equal "TSLA", ranked[0].ticker
assert_equal "US", ranked[0].country_code, "US Tesla should rank first for US user"
end
end

View File

@@ -1,11 +1,6 @@
require "test_helper"
class Security::ResolverTest < ActiveSupport::TestCase
setup do
@provider = mock
Security.stubs(:provider).returns(@provider)
end
test "resolves DB security" do
# Given an existing security in the DB that exactly matches the lookup params
db_security = Security.create!(ticker: "TSLA", exchange_operating_mic: "XNAS", country_code: "US")
@@ -75,4 +70,73 @@ class Security::ResolverTest < ActiveSupport::TestCase
assert_raises(ArgumentError) { Security::Resolver.new(nil).resolve }
assert_raises(ArgumentError) { Security::Resolver.new("").resolve }
end
test "persists explicit price_provider on DB match" do
db_security = Security.create!(ticker: "CSPX", exchange_operating_mic: "XLON", country_code: "GB")
Security.expects(:search_provider).never
Setting.stubs(:enabled_securities_providers).returns([ "tiingo" ])
resolved = Security::Resolver.new(
"CSPX",
exchange_operating_mic: "XLON",
country_code: "GB",
price_provider: "tiingo"
).resolve
assert_equal db_security, resolved
assert_equal "tiingo", resolved.reload.price_provider
end
test "persists price_provider on provider match" do
match = Security.new(ticker: "VWCE", exchange_operating_mic: "XETR", country_code: "DE", price_provider: "eodhd")
Security.expects(:search_provider)
.with("VWCE", exchange_operating_mic: "XETR")
.returns([ match ])
Setting.stubs(:enabled_securities_providers).returns([ "eodhd" ])
resolved = Security::Resolver.new(
"VWCE",
exchange_operating_mic: "XETR",
price_provider: "eodhd"
).resolve
assert resolved.persisted?
assert_equal "eodhd", resolved.price_provider
end
test "rejects unknown price_provider" do
db_security = Security.create!(ticker: "AAPL2", exchange_operating_mic: "XNAS", country_code: "US")
Security.expects(:search_provider).never
resolved = Security::Resolver.new(
"AAPL2",
exchange_operating_mic: "XNAS",
country_code: "US",
price_provider: "fake_provider"
).resolve
assert_equal db_security, resolved
assert_nil resolved.reload.price_provider, "Unknown providers should be rejected"
end
test "rejects disabled price_provider" do
db_security = Security.create!(ticker: "GOOG2", exchange_operating_mic: "XNAS", country_code: "US")
Security.expects(:search_provider).never
Setting.stubs(:enabled_securities_providers).returns([ "twelve_data" ])
resolved = Security::Resolver.new(
"GOOG2",
exchange_operating_mic: "XNAS",
country_code: "US",
price_provider: "tiingo"
).resolve
assert_equal db_security, resolved
assert_nil resolved.reload.price_provider, "Disabled providers should be rejected"
end
end