QT/sure
1
0
Fork 0
mirror of https://github.com/we-promise/sure.git synced 2026-04-18 19:44:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expand financial providers (#1407)

Browse Source 
* Initial implementation

* Tiingo fixes

* Adds 2 providers, remove 2

* Add  extra checks

* FIX a big hotwire race condition

// Fix hotwire_combobox race condition: when typing quickly, a slow response for
// an early query (e.g. "A") can overwrite the correct results for the final query
// (e.g. "AAPL"). We abort the previous in-flight request whenever a new one fires,
// so stale Turbo Stream responses never reach the DOM.

* pipelock

* Update price_test.rb

* Reviews

* i8n

* fixes

* fixes

* Update tiingo.rb

* fixes

* Improvements

* Big revamp

* optimisations

* Update 20260408151837_add_offline_reason_to_securities.rb

* Add missing tests, fixes

* small rank tests

* FIX tests

* Update show.html.erb

* Update resolver.rb

* Update usd_converter.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update holdings_controller.rb

* Update _yahoo_finance_settings.html.erb
This commit is contained in:
soky srm
2026-04-09 18:33:59 +02:00
committed by GitHub
parent ab13093634
commit 7908f7d8a4
50 changed files with 2553 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/security/combobox_option.rb
View File

@@ -1,10 +1,10 @@
class Security::ComboboxOption
include ActiveModel::Model
attr_accessor :symbol, :name, :logo_url, :exchange_operating_mic, :country_code
attr_accessor :symbol, :name, :logo_url, :exchange_operating_mic, :country_code, :price_provider, :currency
def id
"#{symbol}|#{exchange_operating_mic}"
"#{symbol}|#{exchange_operating_mic}|#{price_provider}"
end
def exchange_name

13
app/models/security/health_checker.rb
View File

@@ -66,11 +66,21 @@ class Security::HealthChecker
attr_reader :security
def provider
Security.provider
security.price_data_provider
end
# Some providers (e.g., Alpha Vantage) have very low daily limits and no
# lightweight endpoint — each health check burns a full API call that
# fetches ~100 data points. Skip health checks for those providers to
# avoid exhausting their quota on monitoring alone.
def skip_health_check?
provider.present? && provider.respond_to?(:max_history_days) &&
provider.is_a?(Provider::AlphaVantage)
end
def latest_provider_price
return nil unless provider.present?
return true if skip_health_check? # treat as healthy — quota too precious
response = provider.fetch_security_price(
symbol: security.ticker,
@@ -111,6 +121,7 @@ class Security::HealthChecker
Security.transaction do
security.update!(
offline: true,
offline_reason: "health_check_failed",
failed_fetch_count: MAX_CONSECUTIVE_FAILURES + 1,
failed_fetch_at: Time.current
)

46
app/models/security/price/importer.rb
View File

@@ -31,20 +31,20 @@ class Security::Price::Importer
prev_currency = prev_price_currency || db_price_currency || "USD"
unless prev_price_value.present?
Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{start_date}")
Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{fill_start_date}")
Sentry.capture_exception(MissingStartPriceError.new("Could not determine start price for ticker")) do |scope|
scope.set_tags(security_id: security.id)
scope.set_context("security", {
id: security.id,
start_date: start_date
start_date: fill_start_date
})
end
return 0
end
gapfilled_prices = effective_start_date.upto(end_date).map do |date|
gapfilled_prices = fill_start_date.upto(end_date).map do |date|
db_price = db_prices[date]
db_price_value = db_price&.price
provider_price = provider_prices[date]
@@ -101,15 +101,34 @@ class Security::Price::Importer
private
attr_reader :security, :security_provider, :start_date, :end_date, :clear_cache
# The start date sent to the provider API, clamped to the provider's max
# lookback window when applicable. Computed independently of provider_prices
# so fill_start_date can reference it without relying on method call order.
def provider_fetch_start_date
@provider_fetch_start_date ||= begin
base = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days
max_days = security_provider.respond_to?(:max_history_days) ? security_provider.max_history_days : nil
if max_days && (end_date - base).to_i > max_days
clamped = end_date - max_days.days
Rails.logger.info(
"#{security_provider.class.name} max history is #{max_days} days; " \
"clamping #{security.ticker} start_date from #{base} to #{clamped}"
)
clamped
else
base
end
end
end
def provider_prices
@provider_prices ||= begin
provider_fetch_start_date = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days
response = security_provider.fetch_security_prices(
symbol: security.ticker,
exchange_operating_mic: security.exchange_operating_mic,
start_date: provider_fetch_start_date,
end_date: end_date
end_date: end_date
)
if response.success?
@@ -175,9 +194,17 @@ class Security::Price::Importer
end || end_date
end
# The date the gap-fill loop starts from. When the provider's history was
# clamped (e.g. Alpha Vantage 140 days), we start from the clamped window
# instead of the original effective_start_date to avoid writing hundreds of
# LOCF-filled prices for dates the provider can't actually serve.
def fill_start_date
@fill_start_date ||= [ provider_fetch_start_date, effective_start_date ].max
end
def start_price_value
# When processing full range (first sync), use original behavior
if effective_start_date == start_date
if fill_start_date == start_date
provider_price_value = provider_prices.select { |date, _| date <= start_date }
.max_by { |date, _| date }
&.last&.price
@@ -188,9 +215,8 @@ class Security::Price::Importer
return nil
end
# For partial range (effective_start_date > start_date), use recent data
# This prevents stale prices from old trade dates propagating to current gap-fills
cutoff_date = effective_start_date
# For partial range or clamped range, use the most recent data before fill_start_date
cutoff_date = fill_start_date
# First try provider prices (most recent before cutoff)
provider_price_value = provider_prices

232
app/models/security/provided.rb
View File

@@ -4,50 +4,187 @@ module Security::Provided
SecurityInfoMissingError = Class.new(StandardError)
class_methods do
def provider
provider = ENV["SECURITIES_PROVIDER"].presence || Setting.securities_provider
registry = Provider::Registry.for_concept(:securities)
registry.get_provider(provider.to_sym)
# Returns all enabled and configured securities providers
def providers
Setting.enabled_securities_providers.filter_map do |name|
Provider::Registry.for_concept(:securities).get_provider(name.to_sym)
rescue Provider::Registry::Error
nil
end
end
# Backward compat: first enabled provider
def provider
providers.first
end
# Get a specific provider by key name (e.g., "finnhub", "twelve_data")
# Returns nil if the provider is disabled in settings or not configured.
def provider_for(name)
return nil if name.blank?
return nil unless Setting.enabled_securities_providers.include?(name.to_s)
Provider::Registry.for_concept(:securities).get_provider(name.to_sym)
rescue Provider::Registry::Error
nil
end
# Cache duration for search results (avoids burning through provider rate limits)
SEARCH_CACHE_TTL = 5.minutes
# Maximum number of results returned to the combobox dropdown
MAX_SEARCH_RESULTS = 30
# Per-provider timeout so one slow provider can't stall the entire search
PROVIDER_SEARCH_TIMEOUT = 8.seconds
def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)
return [] if provider.nil? || symbol.blank?
return [] if symbol.blank?
active_providers = providers.compact
return [] if active_providers.empty?
params = {
country_code: country_code,
exchange_operating_mic: exchange_operating_mic
}.compact_blank
response = provider.search_securities(symbol, **params)
if response.success?
securities = response.data.map do |provider_security|
# Need to map to domain model so Combobox can display via to_combobox_option
Security.new(
ticker: provider_security.symbol,
name: provider_security.name,
logo_url: provider_security.logo_url,
exchange_operating_mic: provider_security.exchange_operating_mic,
country_code: provider_security.country_code
)
# Query all providers concurrently so the total wall time is max(provider
# latencies) instead of sum. Each future runs in the concurrent-ruby thread
# pool, keeping Puma threads unblocked during individual provider sleeps.
futures = active_providers.map do |prov|
Concurrent::Promises.future(prov) do |provider|
fetch_provider_results(provider, symbol, params)
end
# Sort results to prioritize user's country if provided
if country_code.present?
user_country = country_code.upcase
securities.sort_by do |s|
[
s.country_code&.upcase == user_country ? 0 : 1, # User's country first
s.ticker.upcase == symbol.upcase ? 0 : 1 # Exact ticker match second
]
end
else
securities
end
else
[]
end
# Collect results from each future individually with a shared deadline.
# Unlike zip (which is all-or-nothing), this keeps results from fast
# providers even when a slow one times out.
deadline = Time.current + PROVIDER_SEARCH_TIMEOUT
results_array = futures.map do |future|
remaining = [ (deadline - Time.current), 0 ].max
future.value(remaining)
end
all_results = []
seen_keys = Set.new
results_array.each_with_index do |provider_results, idx|
next if provider_results.nil?
provider_key = provider_key_for(active_providers[idx])
provider_results.each do |ps|
# Dedup key includes provider so the same ticker on the same exchange can
# appear once per provider — the user picks which provider's price feed
# they want and that choice is stored in price_provider.
dedup_key = "#{ps[:symbol]}|#{ps[:exchange_operating_mic]}|#{provider_key}".upcase
next if seen_keys.include?(dedup_key)
seen_keys.add(dedup_key)
security = Security.new(
ticker: ps[:symbol],
name: ps[:name],
logo_url: ps[:logo_url],
exchange_operating_mic: ps[:exchange_operating_mic],
country_code: ps[:country_code],
search_currency: ps[:currency],
price_provider: provider_key
)
all_results << security
end
end
if all_results.empty? && active_providers.any?
Rails.logger.warn("Security search: all #{active_providers.size} providers returned no results for '#{symbol}'")
end
rank_search_results(all_results, symbol, country_code).first(MAX_SEARCH_RESULTS)
end
private
def provider_key_for(provider_instance)
provider_instance.class.name.demodulize.underscore
end
# Fetches (or reads from cache) search results for a single provider.
# Designed to run inside a Concurrent::Promises.future.
def fetch_provider_results(prov, symbol, params)
provider_key = provider_key_for(prov)
cache_key = "security_search:#{provider_key}:#{symbol.upcase}:#{Digest::SHA256.hexdigest(params.sort_by { |k, _| k }.to_json)}"
Rails.cache.fetch(cache_key, expires_in: SEARCH_CACHE_TTL, skip_nil: true) do
response = prov.search_securities(symbol, **params)
next nil unless response.success?
response.data.map do |ps|
{ symbol: ps.symbol, name: ps.name, logo_url: ps.logo_url,
exchange_operating_mic: ps.exchange_operating_mic, country_code: ps.country_code,
currency: ps.respond_to?(:currency) ? ps.currency : nil }
end
end
rescue => e
Rails.logger.warn("Security search failed for #{provider_key}: #{e.message}")
nil
end
# Scores and sorts search results so the most relevant matches appear first.
# Scoring criteria (lower = better):
# 0: exact ticker match
# 1: ticker starts with query
# 2: name contains query
# 3: everything else
# Within the same relevance tier, user's country is preferred.
def rank_search_results(results, symbol, country_code)
query = symbol.upcase
user_country = country_code&.upcase
results.sort_by do |s|
ticker_up = s.ticker.upcase
relevance = if ticker_up == query
0
elsif ticker_up.start_with?(query)
1
elsif s.name&.upcase&.include?(query)
2
else
3
end
country_match = (user_country.present? && s.country_code&.upcase == user_country) ? 0 : 1
[ relevance, country_match, ticker_up ]
end
end
end
# Public method: resolves the provider for this specific security.
# Uses the security's assigned price_provider if available and configured.
# Falls back to the first enabled provider only when no specific provider
# was ever assigned. When an assigned provider becomes unavailable, returns
# nil so the security is skipped rather than queried against an incompatible
# provider (e.g. MFAPI scheme codes sent to TwelveData).
def price_data_provider
if price_provider.present?
assigned = self.class.provider_for(price_provider)
return assigned if assigned.present?
return nil # assigned provider is unavailable — don't silently fall back
end
self.class.providers.first
end
# Returns the health status of this security's provider link.
# Delegates to price_data_provider to avoid duplicating provider lookup logic.
def provider_status
resolved = price_data_provider
# Had a specific provider assigned but it's now unavailable
return :provider_unavailable if resolved.nil? && price_provider.present?
return :offline if offline?
return :no_provider if resolved.nil?
return :stale if failed_fetch_count.to_i > 0
:ok
end
def find_or_fetch_price(date: Date.current, cache: true)
@@ -59,8 +196,8 @@ module Security::Provided
return nil if offline?
# Make sure we have a data provider before fetching
return nil unless provider.present?
response = provider.fetch_security_price(
return nil unless price_data_provider.present?
response = price_data_provider.fetch_security_price(
symbol: ticker,
exchange_operating_mic: exchange_operating_mic,
date: date
@@ -79,7 +216,7 @@ module Security::Provided
end
def import_provider_details(clear_cache: false)
unless provider.present?
unless price_data_provider.present?
Rails.logger.warn("No provider configured for Security.import_provider_details")
return
end
@@ -88,19 +225,21 @@ module Security::Provided
return
end
response = provider.fetch_security_info(
response = price_data_provider.fetch_security_info(
symbol: ticker,
exchange_operating_mic: exchange_operating_mic
)
if response.success?
update(
name: response.data.name,
logo_url: response.data.logo_url,
website_url: response.data.links
)
# Only overwrite fields the provider actually returned, so providers that
# don't support metadata (e.g. Alpha Vantage) won't blank existing values.
attrs = {}
attrs[:name] = response.data.name if response.data.name.present?
attrs[:logo_url] = response.data.logo_url if response.data.logo_url.present?
attrs[:website_url] = response.data.links if response.data.links.present?
update(attrs) if attrs.any?
else
Rails.logger.warn("Failed to fetch security info for #{ticker} from #{provider.class.name}: #{response.error.message}")
Rails.logger.warn("Failed to fetch security info for #{ticker} from #{price_data_provider.class.name}: #{response.error.message}")
Sentry.capture_exception(SecurityInfoMissingError.new("Failed to get security info"), level: :warning) do |scope|
scope.set_tags(security_id: self.id)
scope.set_context("security", { id: self.id, provider_error: response.error.message })
@@ -109,23 +248,18 @@ module Security::Provided
end
def import_provider_prices(start_date:, end_date:, clear_cache: false)
unless provider.present?
unless price_data_provider.present?
Rails.logger.warn("No provider configured for Security.import_provider_prices")
return 0
end
importer = Security::Price::Importer.new(
security: self,
security_provider: provider,
security_provider: price_data_provider,
start_date: start_date,
end_date: end_date,
clear_cache: clear_cache
)
[ importer.import_provider_prices, importer.provider_error ]
end
private
def provider
self.class.provider
end
end

61
app/models/security/resolver.rb
View File

@@ -1,8 +1,9 @@
class Security::Resolver
def initialize(symbol, exchange_operating_mic: nil, country_code: nil)
def initialize(symbol, exchange_operating_mic: nil, country_code: nil, price_provider: nil)
@symbol = validate_symbol!(symbol)
@exchange_operating_mic = exchange_operating_mic
@country_code = country_code
@price_provider = validated_price_provider(price_provider)
end
# Attempts several paths to resolve a security:
@@ -20,13 +21,22 @@ class Security::Resolver
end
private
attr_reader :symbol, :exchange_operating_mic, :country_code
attr_reader :symbol, :exchange_operating_mic, :country_code, :price_provider
def validate_symbol!(symbol)
raise ArgumentError, "Symbol is required and cannot be blank" if symbol.blank?
symbol.strip.upcase
end
# Only accept price_provider values that are known and currently enabled.
# Prevents tampered combobox values from persisting invalid provider names.
def validated_price_provider(value)
return nil if value.blank?
return nil unless Security.valid_price_providers.include?(value.to_s)
return nil unless Setting.enabled_securities_providers.include?(value.to_s)
value.to_s
end
def offline_security
security = Security.find_or_initialize_by(
ticker: symbol,
@@ -44,13 +54,26 @@ class Security::Resolver
end
def exact_match_from_db
Security.find_by(
security = Security.find_by(
{
ticker: symbol,
exchange_operating_mic: exchange_operating_mic,
country_code: country_code.presence
}.compact
)
return nil unless security
# When the caller provides an explicit provider (e.g. user selected from
# search results), honor that choice. Automated syncs (Plaid, SimpleFIN)
# pass price_provider: nil and will not overwrite.
if price_provider.present? && security.price_provider != price_provider
security.update!(price_provider: price_provider)
end
reactivate_if_provider_available!(security)
security
end
# If provided a ticker + exchange (and optionally, a country code), we can find exact matches
@@ -59,8 +82,8 @@ class Security::Resolver
return nil unless exchange_operating_mic.present?
match = provider_search_result.find do |s|
ticker_matches = s.ticker.upcase.to_s == symbol.upcase.to_s
exchange_matches = s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s
ticker_matches = s.ticker&.upcase.to_s == symbol.upcase.to_s
exchange_matches = s.exchange_operating_mic&.upcase.to_s == exchange_operating_mic.upcase.to_s
if country_code && exchange_operating_mic
ticker_matches && exchange_matches && s.country_code&.upcase.to_s == country_code.upcase.to_s
@@ -88,8 +111,8 @@ class Security::Resolver
# 4. Rank by exchange_operating_mic relevance (lower index in the list is more relevant)
sorted_candidates = filtered_candidates.sort_by do |s|
[
s.ticker.upcase.to_s == symbol.upcase.to_s ? 0 : 1,
exchange_operating_mic.present? && s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
s.ticker&.upcase.to_s == symbol.upcase.to_s ? 0 : 1,
exchange_operating_mic.present? && s.exchange_operating_mic&.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
sorted_country_codes_by_relevance.index(s.country_code&.upcase.to_s) || sorted_country_codes_by_relevance.length,
sorted_exchange_operating_mics_by_relevance.index(s.exchange_operating_mic&.upcase.to_s) || sorted_exchange_operating_mics_by_relevance.length
]
@@ -109,11 +132,35 @@ class Security::Resolver
)
security.country_code = match.country_code
# Set provider when explicitly provided (user selection) or when the
# record is new / has no provider yet. Automated syncs pass nil and
# will not overwrite an existing choice.
effective_provider = price_provider.presence ||
(match.respond_to?(:price_provider) ? match.price_provider.presence : nil)
if effective_provider.present?
security.price_provider = effective_provider
end
security.save!
reactivate_if_provider_available!(security)
security
end
# If a security was marked offline (e.g. its provider was temporarily
# removed in settings) but now has a valid, enabled provider, bring it
# back online so the MarketDataImporter picks it up again.
def reactivate_if_provider_available!(security)
return unless security.offline?
return unless security.offline_reason == "provider_disabled"
return unless security.price_data_provider.present?
security.update!(offline: false, offline_reason: nil, failed_fetch_count: 0, failed_fetch_at: nil)
end
def provider_search_result
params = {
exchange_operating_mic: exchange_operating_mic,