mirror of
https://github.com/we-promise/sure.git
synced 2026-09-05 23:01:26 +00:00
Faldy Ikhwan Fadila
792047b82e
Add JKT → XIDX exchange MIC mapping, .JK symbol suffix normalization, IDR default currency, and ID country code for Jakarta exchange. Yahoo Finance returns Indonesian stocks (e.g. BBCA.JK) with exchange code 'JKT'. Without this mapping, the provider cannot resolve the exchange to the XIDX MIC already defined in config/exchanges.yml, and normalize_symbol cannot append the .JK suffix for price lookups. Tested manually: Yahoo Finance search and chart endpoints return valid results for IDX tickers (BBCA.JK, currency=IDR, timezone=WIB).
1160 lines
39 KiB
Ruby
1160 lines
39 KiB
Ruby
require "set"
|
|
|
|
class Provider::YahooFinance < Provider
|
|
include ExchangeRateConcept, SecurityConcept
|
|
extend SslConfigurable
|
|
|
|
# Subclass so errors caught in this provider are raised as Provider::YahooFinance::Error
|
|
Error = Class.new(Provider::Error)
|
|
InvalidSecurityPriceError = Class.new(Error)
|
|
RateLimitError = Class.new(Error)
|
|
AuthenticationError = Class.new(Error)
|
|
InvalidSymbolError = Class.new(Error)
|
|
MarketClosedError = Class.new(Error)
|
|
|
|
# Cache duration for repeated requests (5 minutes)
|
|
CACHE_DURATION = 5.minutes
|
|
|
|
# Maximum cache duration for cookie/crumb authentication
|
|
# Even if cookie has longer expiry, cap it to avoid stale crumbs
|
|
MAX_CRUMB_CACHE_DURATION = 1.hour
|
|
|
|
HEALTH_STATUS_FRESHNESS = {
|
|
healthy: 15.minutes,
|
|
rate_limited: 30.minutes,
|
|
unavailable: 5.minutes
|
|
}.freeze
|
|
HEALTH_STATUS_RETENTION = 1.hour
|
|
HEALTH_LOCK_DURATION = 15.seconds
|
|
HEALTH_STATUS_CACHE_KEY = "yahoo_finance_health_status"
|
|
HEALTH_LOCK_CACHE_KEY = "yahoo_finance_health_status_lock"
|
|
|
|
INVALID_CRUMBS = Set.new([ "too many requests" ]).freeze
|
|
|
|
# Maximum lookback window for historical data (configurable)
|
|
MAX_LOOKBACK_WINDOW = 10.years
|
|
|
|
def max_history_days
|
|
(MAX_LOOKBACK_WINDOW / 1.day).to_i
|
|
end
|
|
|
|
# Minimum delay between requests to avoid rate limiting (in seconds)
|
|
MIN_REQUEST_INTERVAL = 0.5
|
|
|
|
# Pool of modern browser user-agents to rotate through
|
|
# Based on https://github.com/ranaroussi/yfinance/pull/2277
|
|
# UPDATED user-agents string on 2026-02-27 with current versions of browsers (Chrome 145, Firefox 148, Safari 26)
|
|
USER_AGENTS = [
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"
|
|
].freeze
|
|
|
|
def initialize
|
|
# Yahoo Finance doesn't require an API key but we may want to add proxy support later
|
|
@cache_prefix = "yahoo_finance"
|
|
end
|
|
|
|
def health_status
|
|
assessment = read_health_cache(HEALTH_STATUS_CACHE_KEY)
|
|
return assessment[:status] if assessment_fresh?(assessment)
|
|
|
|
YahooFinanceHealthCheckJob.perform_later
|
|
assessment&.dig(:status) || :unknown
|
|
rescue HealthCacheError => e
|
|
record_health_cache_failure(e)
|
|
:unknown
|
|
end
|
|
|
|
def refresh_health_status
|
|
assessment = read_health_cache(HEALTH_STATUS_CACHE_KEY)
|
|
return assessment[:status] if assessment_fresh?(assessment)
|
|
|
|
lock_token = SecureRandom.uuid
|
|
lock_acquired = write_health_cache(
|
|
HEALTH_LOCK_CACHE_KEY,
|
|
lock_token,
|
|
expires_in: HEALTH_LOCK_DURATION,
|
|
unless_exist: true
|
|
)
|
|
return assessment&.dig(:status) || :unknown unless lock_acquired
|
|
return :unknown unless health_lock_owned?(lock_token)
|
|
|
|
result = perform_health_check
|
|
completed_assessment = result.merge(checked_at: Time.current)
|
|
unless publish_health_assessment(lock_token, completed_assessment)
|
|
return read_health_cache(HEALTH_STATUS_CACHE_KEY)&.dig(:status) || :unknown
|
|
end
|
|
|
|
record_health_transition(assessment, result)
|
|
status = result.fetch(:status)
|
|
released = release_health_lock(lock_token)
|
|
lock_token = nil
|
|
released ? status : :unknown
|
|
rescue HealthCacheError => e
|
|
record_health_cache_failure(e)
|
|
:unknown
|
|
ensure
|
|
release_health_lock(lock_token) if lock_token.present?
|
|
end
|
|
|
|
def healthy?
|
|
health_status == :healthy
|
|
end
|
|
|
|
def usage
|
|
# Yahoo Finance doesn't expose usage data, so we return a mock structure
|
|
with_provider_response do
|
|
usage_data = UsageData.new(
|
|
used: 0,
|
|
limit: 2000, # Estimated daily limit based on community knowledge
|
|
utilization: 0,
|
|
plan: "Free"
|
|
)
|
|
|
|
usage_data
|
|
end
|
|
end
|
|
|
|
# ================================
|
|
# Exchange Rates
|
|
# ================================
|
|
|
|
def fetch_exchange_rate(from:, to:, date:)
|
|
with_provider_response do
|
|
# Return 1.0 if same currency
|
|
if from == to
|
|
Rate.new(date: date, from: from, to: to, rate: 1.0)
|
|
else
|
|
cache_key = "exchange_rate_#{from}_#{to}_#{date}"
|
|
if cached_result = get_cached_result(cache_key)
|
|
cached_result
|
|
else
|
|
# For a single date, we'll fetch a range and find the closest match
|
|
end_date = date
|
|
start_date = date - 10.days # Extended range for better coverage
|
|
|
|
rates_response = fetch_exchange_rates(
|
|
from: from,
|
|
to: to,
|
|
start_date: start_date,
|
|
end_date: end_date
|
|
)
|
|
|
|
raise Error, "Failed to fetch exchange rates: #{rates_response.error.message}" unless rates_response.success?
|
|
|
|
rates = rates_response.data
|
|
if rates.length == 1
|
|
rates.first
|
|
else
|
|
# Find the exact date or the closest previous date
|
|
target_rate = rates.find { |r| r.date == date } ||
|
|
rates.select { |r| r.date <= date }.max_by(&:date)
|
|
|
|
raise Error, "No exchange rate found for #{from}/#{to} on or before #{date}" unless target_rate
|
|
|
|
cache_result(cache_key, target_rate)
|
|
target_rate
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def fetch_exchange_rates(from:, to:, start_date:, end_date:)
|
|
with_provider_response do
|
|
validate_date_range!(start_date, end_date)
|
|
# Return 1.0 rates if same currency
|
|
if from == to
|
|
generate_same_currency_rates(from, to, start_date, end_date)
|
|
else
|
|
cache_key = "exchange_rates_#{from}_#{to}_#{start_date}_#{end_date}"
|
|
if cached_result = get_cached_result(cache_key)
|
|
cached_result
|
|
else
|
|
# Try both direct and inverse currency pairs
|
|
rates = fetch_currency_pair_data(from, to, start_date, end_date) ||
|
|
fetch_inverse_currency_pair_data(from, to, start_date, end_date)
|
|
|
|
raise Error, "No chart data found for currency pair #{from}/#{to}" unless rates&.any?
|
|
|
|
cache_result(cache_key, rates)
|
|
rates
|
|
end
|
|
end
|
|
rescue JSON::ParserError => e
|
|
raise Error, "Invalid response format: #{e.message}"
|
|
end
|
|
end
|
|
|
|
# ================================
|
|
# Securities
|
|
# ================================
|
|
|
|
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
|
|
with_provider_response do
|
|
cache_key = "search_#{symbol}_#{country_code}_#{exchange_operating_mic}"
|
|
if cached_result = get_cached_result(cache_key)
|
|
cached_result
|
|
else
|
|
throttle_request
|
|
response = client.get("#{base_url}/v1/finance/search") do |req|
|
|
req.params["q"] = symbol.strip.upcase
|
|
req.params["quotesCount"] = 25
|
|
end
|
|
|
|
data = JSON.parse(response.body)
|
|
quotes = data.dig("quotes") || []
|
|
|
|
securities = quotes.filter_map do |quote|
|
|
mic = map_exchange_mic(quote["exchange"])
|
|
|
|
Security.new(
|
|
symbol: quote["symbol"],
|
|
name: quote["longname"] || quote["shortname"] || quote["symbol"],
|
|
logo_url: nil, # Yahoo search doesn't provide logos
|
|
exchange_operating_mic: mic,
|
|
country_code: ::Security::EXCHANGES.dig(mic, "country") || map_country_code(quote["exchDisp"])
|
|
)
|
|
end
|
|
|
|
securities = deduplicate_dual_listings(securities) unless exchange_operating_mic.present?
|
|
|
|
cache_result(cache_key, securities)
|
|
securities
|
|
end
|
|
rescue JSON::ParserError => e
|
|
raise Error, "Invalid search response format: #{e.message}"
|
|
end
|
|
end
|
|
|
|
def fetch_security_info(symbol:, exchange_operating_mic:)
|
|
with_provider_response do
|
|
symbol = normalize_symbol(symbol, exchange_operating_mic)
|
|
|
|
# quoteSummary endpoint requires cookie/crumb authentication
|
|
throttle_request
|
|
cookie, crumb = fetch_cookie_and_crumb
|
|
|
|
response = authenticated_client(cookie).get("#{base_url}/v10/finance/quoteSummary/#{symbol}") do |req|
|
|
req.params["modules"] = "assetProfile,price,quoteType"
|
|
req.params["crumb"] = crumb
|
|
end
|
|
|
|
data = JSON.parse(response.body)
|
|
|
|
# Check for auth errors in response body
|
|
if data.dig("quoteSummary", "error", "code") == "Unauthorized"
|
|
# Clear cached crumb and retry once
|
|
clear_crumb_cache
|
|
cookie, crumb = fetch_cookie_and_crumb
|
|
response = authenticated_client(cookie).get("#{base_url}/v10/finance/quoteSummary/#{symbol}") do |req|
|
|
req.params["modules"] = "assetProfile,price,quoteType"
|
|
req.params["crumb"] = crumb
|
|
end
|
|
data = JSON.parse(response.body)
|
|
if data.dig("quoteSummary", "error", "code") == "Unauthorized"
|
|
raise AuthenticationError, "Yahoo Finance authentication failed after crumb refresh"
|
|
end
|
|
end
|
|
|
|
result = data.dig("quoteSummary", "result", 0)
|
|
|
|
raise Error, "No security info found for #{symbol}" unless result
|
|
|
|
asset_profile = result["assetProfile"] || {}
|
|
price_info = result["price"] || {}
|
|
quote_type = result["quoteType"] || {}
|
|
|
|
security_info = SecurityInfo.new(
|
|
symbol: symbol,
|
|
name: price_info["longName"] || price_info["shortName"] || quote_type["longName"] || quote_type["shortName"],
|
|
links: asset_profile["website"],
|
|
logo_url: nil, # Yahoo doesn't provide reliable logo URLs
|
|
description: asset_profile["longBusinessSummary"],
|
|
kind: map_security_type(quote_type["quoteType"]),
|
|
exchange_operating_mic: exchange_operating_mic
|
|
)
|
|
|
|
security_info
|
|
rescue JSON::ParserError => e
|
|
raise Error, "Invalid response format: #{e.message}"
|
|
end
|
|
end
|
|
|
|
def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
|
|
with_provider_response do
|
|
symbol = normalize_symbol(symbol, exchange_operating_mic)
|
|
cache_key = "security_price_#{symbol}_#{exchange_operating_mic}_#{date}"
|
|
if cached_result = get_cached_result(cache_key)
|
|
cached_result
|
|
else
|
|
# For a single date, we'll fetch a range and find the closest match
|
|
end_date = date
|
|
start_date = date - 10.days # Extended range for better coverage
|
|
|
|
prices_response = fetch_security_prices(
|
|
symbol: symbol,
|
|
exchange_operating_mic: exchange_operating_mic,
|
|
start_date: start_date,
|
|
end_date: end_date
|
|
)
|
|
|
|
raise Error, "Failed to fetch security prices: #{prices_response.error.message}" unless prices_response.success?
|
|
|
|
prices = prices_response.data
|
|
if prices.length == 1
|
|
target_price = prices.first
|
|
else
|
|
# Find the exact date or the closest previous date
|
|
target_price = prices.find { |p| p.date == date } ||
|
|
prices.select { |p| p.date <= date }.max_by(&:date)
|
|
|
|
raise Error, "No price found for #{symbol} on or before #{date}" unless target_price
|
|
end
|
|
|
|
cache_result(cache_key, target_price)
|
|
target_price
|
|
end
|
|
end
|
|
end
|
|
|
|
def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
|
|
with_provider_response do
|
|
symbol = normalize_symbol(symbol, exchange_operating_mic)
|
|
validate_date_params!(start_date, end_date)
|
|
# Convert dates to Unix timestamps using UTC to ensure consistent epoch boundaries across timezones
|
|
period1 = start_date.to_time.utc.to_i
|
|
period2 = end_date.end_of_day.to_time.utc.to_i
|
|
|
|
throttle_request
|
|
data = fetch_authenticated_chart(symbol, {
|
|
"period1" => period1,
|
|
"period2" => period2,
|
|
"interval" => "1d",
|
|
"includeAdjustedClose" => true
|
|
})
|
|
|
|
chart_data = data.dig("chart", "result", 0)
|
|
|
|
raise Error, "No chart data found for #{symbol}" unless chart_data
|
|
|
|
timestamps = chart_data.dig("timestamp") || []
|
|
quotes = chart_data.dig("indicators", "quote", 0) || {}
|
|
closes = quotes["close"] || []
|
|
|
|
# Get currency from metadata
|
|
meta_exchange = chart_data.dig("meta", "exchangeName") || ""
|
|
raw_currency = chart_data.dig("meta", "currency")
|
|
raw_currency ||= default_currency_for_exchange(meta_exchange) || "USD"
|
|
|
|
prices = []
|
|
timestamps.each_with_index do |timestamp, index|
|
|
close_price = closes[index]
|
|
next if close_price.nil? # Skip days with no data (weekends, holidays)
|
|
|
|
# Normalize currency and price to handle minor units
|
|
normalized_currency, normalized_price = normalize_currency_and_price(raw_currency, close_price.to_f)
|
|
|
|
prices << Price.new(
|
|
symbol: symbol,
|
|
date: Time.at(timestamp).utc.to_date,
|
|
price: normalized_price,
|
|
currency: normalized_currency,
|
|
exchange_operating_mic: exchange_operating_mic
|
|
)
|
|
end
|
|
|
|
sorted_prices = prices.sort_by(&:date)
|
|
sorted_prices
|
|
rescue JSON::ParserError => e
|
|
raise Error, "Invalid response format: #{e.message}"
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
HealthCacheError = Class.new(StandardError)
|
|
|
|
def perform_health_check
|
|
stage = :cookie
|
|
|
|
cookie, crumb, stage = fetch_health_cookie_and_crumb
|
|
|
|
stage = :chart
|
|
chart_response = health_authenticated_client(cookie).get("#{base_url}/v8/finance/chart/AAPL") do |req|
|
|
req.params["interval"] = "1d"
|
|
req.params["range"] = "1d"
|
|
req.params["crumb"] = crumb
|
|
end
|
|
return health_result(:rate_limited, stage:, http_status: 429) if chart_response.status == 429
|
|
return health_result(:unavailable, stage:, http_status: chart_response.status) unless chart_response.success?
|
|
|
|
data = JSON.parse(chart_response.body)
|
|
if data.dig("chart", "error", "code") == "Unauthorized"
|
|
delete_health_cache("#{@cache_prefix}_auth_crumb")
|
|
return health_result(:unavailable, stage:, http_status: chart_response.status)
|
|
end
|
|
return health_result(:unavailable, stage:, http_status: chart_response.status) if data.dig("chart", "error").present?
|
|
|
|
results = data.dig("chart", "result")
|
|
health_result(results.present? ? :healthy : :unavailable, stage:, http_status: chart_response.status)
|
|
rescue HealthCacheError
|
|
raise
|
|
rescue Faraday::Error, JSON::ParserError => e
|
|
health_result(:unavailable, stage:, exception_class: e.class.name, http_status: faraday_status(e))
|
|
rescue RateLimitError => e
|
|
health_result(:rate_limited, stage: :crumb, exception_class: e.class.name, http_status: e.details&.dig(:status))
|
|
rescue AuthenticationError => e
|
|
health_result(:unavailable, stage:, exception_class: e.class.name, http_status: e.details&.dig(:status))
|
|
rescue => e
|
|
health_result(:unavailable, stage:, exception_class: e.class.name)
|
|
end
|
|
|
|
def health_result(status, stage:, exception_class: nil, http_status: nil)
|
|
{
|
|
status: status,
|
|
stage: stage,
|
|
exception_class: exception_class,
|
|
http_status: http_status
|
|
}.compact
|
|
end
|
|
|
|
def fetch_health_cookie_and_crumb
|
|
cache_key = "#{@cache_prefix}_auth_crumb"
|
|
cached = read_health_cache(cache_key)
|
|
if cached.present?
|
|
return [ cached.first, cached.second, :authentication_cache ] if valid_crumb?(cached.second)
|
|
|
|
delete_health_cache(cache_key)
|
|
end
|
|
|
|
cookie, crumb, cache_duration = request_cookie_and_crumb(health_auth_client)
|
|
write_health_cache!(cache_key, [ cookie, crumb ], expires_in: cache_duration)
|
|
[ cookie, crumb, :crumb ]
|
|
end
|
|
|
|
def faraday_status(error)
|
|
error.response&.dig(:status) if error.respond_to?(:response)
|
|
end
|
|
|
|
def health_auth_client
|
|
@health_auth_client ||= Faraday.new(ssl: self.class.faraday_ssl_options) do |faraday|
|
|
configure_health_client(faraday)
|
|
faraday.headers["Accept"] = "*/*"
|
|
end
|
|
end
|
|
|
|
def health_authenticated_client(cookie)
|
|
Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
|
|
configure_health_client(faraday)
|
|
faraday.request :json
|
|
faraday.headers["Accept"] = "application/json"
|
|
faraday.headers["Cookie"] = cookie
|
|
end
|
|
end
|
|
|
|
def configure_health_client(faraday)
|
|
faraday.headers["User-Agent"] = random_user_agent
|
|
faraday.headers["Accept-Language"] = "en-US,en;q=0.9"
|
|
faraday.headers["Cache-Control"] = "no-cache"
|
|
faraday.headers["Pragma"] = "no-cache"
|
|
faraday.options.timeout = 5
|
|
faraday.options.open_timeout = 3
|
|
end
|
|
|
|
def read_health_cache(key)
|
|
Rails.cache.read(key)
|
|
rescue => e
|
|
raise HealthCacheError, e.class.name
|
|
end
|
|
|
|
def write_health_cache(key, value, **options)
|
|
Rails.cache.write(key, value, **options)
|
|
rescue => e
|
|
raise HealthCacheError, e.class.name
|
|
end
|
|
|
|
def write_health_cache!(key, value, **options)
|
|
return true if write_health_cache(key, value, **options)
|
|
|
|
raise HealthCacheError, "CacheWriteFailed"
|
|
end
|
|
|
|
def delete_health_cache(key)
|
|
Rails.cache.delete(key)
|
|
rescue => e
|
|
raise HealthCacheError, e.class.name
|
|
end
|
|
|
|
def publish_health_assessment(lock_token, assessment)
|
|
return false unless health_lock_owned?(lock_token)
|
|
|
|
write_health_cache!(HEALTH_STATUS_CACHE_KEY, assessment, expires_in: HEALTH_STATUS_RETENTION)
|
|
return true if health_lock_owned?(lock_token)
|
|
|
|
delete_health_cache(HEALTH_STATUS_CACHE_KEY) if read_health_cache(HEALTH_STATUS_CACHE_KEY) == assessment
|
|
false
|
|
end
|
|
|
|
def release_health_lock(lock_token)
|
|
return true unless health_lock_owned?(lock_token)
|
|
|
|
Rails.cache.delete(HEALTH_LOCK_CACHE_KEY)
|
|
true
|
|
rescue HealthCacheError => e
|
|
record_health_cache_failure(e)
|
|
false
|
|
rescue => e
|
|
record_health_cache_failure(HealthCacheError.new(e.class.name))
|
|
false
|
|
end
|
|
|
|
def health_lock_owned?(lock_token)
|
|
read_health_cache(HEALTH_LOCK_CACHE_KEY) == lock_token
|
|
end
|
|
|
|
def record_health_transition(previous_assessment, result)
|
|
previous_status = previous_assessment&.dig(:status)
|
|
return if previous_status == result[:status]
|
|
|
|
status = result.fetch(:status)
|
|
DebugLogEntry.capture(
|
|
category: "provider_health",
|
|
level: status == :healthy ? "info" : "warn",
|
|
message: "Yahoo Finance Provider Health Status changed to #{status}",
|
|
source: self.class.name,
|
|
provider_key: "yahoo_finance",
|
|
metadata: {
|
|
previous_state: previous_status || :unknown,
|
|
new_state: status,
|
|
health_check_stage: result[:stage],
|
|
exception_class: result[:exception_class],
|
|
http_status: result[:http_status]
|
|
}.compact
|
|
)
|
|
end
|
|
|
|
def record_health_cache_failure(error)
|
|
DebugLogEntry.capture(
|
|
category: "provider_health_cache",
|
|
level: "warn",
|
|
message: "Yahoo Finance Provider Health Status cache is unavailable",
|
|
source: self.class.name,
|
|
provider_key: "yahoo_finance",
|
|
metadata: { exception_class: error.message }
|
|
)
|
|
end
|
|
|
|
def assessment_fresh?(assessment)
|
|
return false unless assessment
|
|
|
|
freshness = HEALTH_STATUS_FRESHNESS[assessment[:status]]
|
|
freshness && assessment[:checked_at] >= freshness.ago
|
|
end
|
|
|
|
def base_url
|
|
ENV["YAHOO_FINANCE_URL"] || "https://query1.finance.yahoo.com"
|
|
end
|
|
|
|
# ================================
|
|
# Currency Normalization
|
|
# ================================
|
|
|
|
# Per-exchange configuration for Yahoo Finance. Each entry maps an ISO
|
|
# MIC code to its Yahoo-specific symbol suffix, the default currency when
|
|
# Yahoo omits one, and an optional dual-listing group with a preference
|
|
# rank (lower = preferred). Adding a new market is a one-line hash entry.
|
|
EXCHANGE_CONFIG = {
|
|
"XNSE" => { yahoo_suffix: ".NS", default_currency: "INR", dual_list_group: :india, preference_rank: 0 },
|
|
"XBOM" => { yahoo_suffix: ".BO", default_currency: "INR", dual_list_group: :india, preference_rank: 1 },
|
|
"XBOG" => { yahoo_suffix: ".CL", default_currency: "COP" },
|
|
"XIDX" => { yahoo_suffix: ".JK", default_currency: "IDR" }
|
|
}.freeze
|
|
|
|
# Yahoo Finance sometimes returns currencies in minor units (pence, cents)
|
|
# This is not part of ISO 4217 but is a convention used by financial data providers
|
|
# Mapping of Yahoo Finance minor unit codes to standard currency codes and conversion multipliers
|
|
MINOR_CURRENCY_CONVERSIONS = {
|
|
"GBp" => { currency: "GBP", multiplier: 0.01 }, # British pence to pounds (eg. https://finance.yahoo.com/quote/IITU.L/)
|
|
"ZAc" => { currency: "ZAR", multiplier: 0.01 } # South African cents to rand (eg. https://finance.yahoo.com/quote/JSE.JO)
|
|
}.freeze
|
|
|
|
# Normalizes Yahoo Finance currency codes and prices
|
|
# Returns [currency_code, price] with currency converted to standard ISO code
|
|
# and price converted from minor units to major units if applicable
|
|
def normalize_currency_and_price(currency, price)
|
|
if conversion = MINOR_CURRENCY_CONVERSIONS[currency]
|
|
[ conversion[:currency], price * conversion[:multiplier] ]
|
|
else
|
|
[ currency, price ]
|
|
end
|
|
end
|
|
|
|
# Appends the Yahoo Finance symbol suffix for exchanges that require one
|
|
# (e.g. XNSE → ".NS", XBOM → ".BO"). Already-suffixed symbols pass through.
|
|
def normalize_symbol(symbol, exchange_operating_mic)
|
|
suffix = EXCHANGE_CONFIG.dig(exchange_operating_mic, :yahoo_suffix)
|
|
return symbol if suffix.nil? || symbol.end_with?(suffix)
|
|
"#{symbol}#{suffix}"
|
|
end
|
|
|
|
# Returns the default currency for a Yahoo exchange name (e.g. "NSE" → "INR")
|
|
# by resolving through map_exchange_mic → EXCHANGE_CONFIG. Returns nil for
|
|
# unknown exchanges so callers can fall back to their own default.
|
|
def default_currency_for_exchange(yahoo_exchange_name)
|
|
mic = map_exchange_mic(yahoo_exchange_name)
|
|
EXCHANGE_CONFIG.dig(mic, :default_currency)
|
|
end
|
|
|
|
# De-duplicates dual-listed securities that share the same company name
|
|
# and dual_list_group (e.g. NSE + BSE for India), keeping the exchange
|
|
# with the lowest preference_rank. Preserves Yahoo's original relevance
|
|
# ordering by removing duplicates in-place rather than reordering.
|
|
def deduplicate_dual_listings(securities)
|
|
dominated = Set.new
|
|
|
|
securities
|
|
.select { |s| EXCHANGE_CONFIG.dig(s.exchange_operating_mic, :dual_list_group) }
|
|
.group_by { |s| [ EXCHANGE_CONFIG[s.exchange_operating_mic][:dual_list_group], s.name.to_s.strip.downcase ] }
|
|
.each_value do |group|
|
|
next unless group.size > 1
|
|
preferred = group.min_by { |s| EXCHANGE_CONFIG[s.exchange_operating_mic][:preference_rank] }
|
|
group.each { |s| dominated << s.object_id unless s.equal?(preferred) }
|
|
end
|
|
|
|
return securities if dominated.empty?
|
|
securities.reject { |s| dominated.include?(s.object_id) }
|
|
end
|
|
|
|
# ================================
|
|
# Validation
|
|
# ================================
|
|
|
|
|
|
def validate_date_range!(start_date, end_date)
|
|
raise Error, "Start date cannot be after end date" if start_date > end_date
|
|
raise Error, "Date range too large (max 5 years)" if end_date > start_date + 5.years
|
|
end
|
|
|
|
def validate_date_params!(start_date, end_date)
|
|
# Validate presence and coerce to dates
|
|
validated_start_date = validate_and_coerce_date!(start_date, "start_date")
|
|
validated_end_date = validate_and_coerce_date!(end_date, "end_date")
|
|
|
|
# Ensure start_date <= end_date
|
|
if validated_start_date > validated_end_date
|
|
error_msg = "Start date (#{validated_start_date}) cannot be after end date (#{validated_end_date})"
|
|
raise ArgumentError, error_msg
|
|
end
|
|
|
|
# Ensure end_date is not in the future
|
|
today = Date.current
|
|
if validated_end_date > today
|
|
error_msg = "End date (#{validated_end_date}) cannot be in the future"
|
|
raise ArgumentError, error_msg
|
|
end
|
|
|
|
# Optional: Enforce max lookback window (configurable via constant)
|
|
max_lookback = MAX_LOOKBACK_WINDOW.ago.to_date
|
|
if validated_start_date < max_lookback
|
|
error_msg = "Start date (#{validated_start_date}) exceeds maximum lookback window (#{max_lookback})"
|
|
raise ArgumentError, error_msg
|
|
end
|
|
end
|
|
|
|
def validate_and_coerce_date!(date_param, param_name)
|
|
# Check presence
|
|
if date_param.blank?
|
|
error_msg = "#{param_name} cannot be blank"
|
|
raise ArgumentError, error_msg
|
|
end
|
|
|
|
# Try to coerce to date
|
|
begin
|
|
if date_param.respond_to?(:to_date)
|
|
date_param.to_date
|
|
else
|
|
Date.parse(date_param.to_s)
|
|
end
|
|
rescue ArgumentError => e
|
|
error_msg = "Invalid #{param_name}: #{date_param} (#{e.message})"
|
|
raise ArgumentError, error_msg
|
|
end
|
|
end
|
|
|
|
# ================================
|
|
# Caching
|
|
# ================================
|
|
|
|
def get_cached_result(key)
|
|
full_key = "#{@cache_prefix}_#{key}"
|
|
data = Rails.cache.read(full_key)
|
|
data
|
|
end
|
|
|
|
def cache_result(key, data)
|
|
full_key = "#{@cache_prefix}_#{key}"
|
|
Rails.cache.write(full_key, data, expires_in: CACHE_DURATION)
|
|
end
|
|
|
|
|
|
|
|
# ================================
|
|
# Helper Methods
|
|
# ================================
|
|
|
|
def generate_same_currency_rates(from, to, start_date, end_date)
|
|
(start_date..end_date).map do |date|
|
|
Rate.new(date: date, from: from, to: to, rate: 1.0)
|
|
end
|
|
end
|
|
|
|
def fetch_currency_pair_data(from, to, start_date, end_date)
|
|
symbol = "#{from}#{to}=X"
|
|
fetch_chart_data(symbol, start_date, end_date) do |timestamp, close_rate|
|
|
Rate.new(
|
|
date: Time.at(timestamp).utc.to_date,
|
|
from: from,
|
|
to: to,
|
|
rate: close_rate.to_f
|
|
)
|
|
end
|
|
end
|
|
|
|
def fetch_inverse_currency_pair_data(from, to, start_date, end_date)
|
|
symbol = "#{to}#{from}=X"
|
|
rates = fetch_chart_data(symbol, start_date, end_date) do |timestamp, close_rate|
|
|
Rate.new(
|
|
date: Time.at(timestamp).utc.to_date,
|
|
from: from,
|
|
to: to,
|
|
rate: (BigDecimal("1") / BigDecimal(close_rate.to_s)).round(12)
|
|
)
|
|
end
|
|
|
|
rates
|
|
end
|
|
|
|
# Makes a single authenticated GET to /v8/finance/chart/:symbol.
|
|
# If Yahoo returns a stale-crumb error (200 OK with Unauthorized body),
|
|
# clears the crumb cache and retries once with fresh credentials.
|
|
def fetch_authenticated_chart(symbol, params)
|
|
cookie, crumb = fetch_cookie_and_crumb
|
|
response = authenticated_client(cookie).get("#{base_url}/v8/finance/chart/#{symbol}") do |req|
|
|
params.each { |k, v| req.params[k] = v }
|
|
req.params["crumb"] = crumb
|
|
end
|
|
data = JSON.parse(response.body)
|
|
|
|
if data.dig("chart", "error", "code") == "Unauthorized"
|
|
clear_crumb_cache
|
|
cookie, crumb = fetch_cookie_and_crumb
|
|
response = authenticated_client(cookie).get("#{base_url}/v8/finance/chart/#{symbol}") do |req|
|
|
params.each { |k, v| req.params[k] = v }
|
|
req.params["crumb"] = crumb
|
|
end
|
|
data = JSON.parse(response.body)
|
|
if data.dig("chart", "error", "code") == "Unauthorized"
|
|
raise AuthenticationError, "Yahoo Finance authentication failed after crumb refresh"
|
|
end
|
|
end
|
|
|
|
data
|
|
end
|
|
|
|
def fetch_chart_data(symbol, start_date, end_date, &block)
|
|
period1 = start_date.to_time.utc.to_i
|
|
period2 = end_date.end_of_day.to_time.utc.to_i
|
|
|
|
begin
|
|
throttle_request
|
|
data = fetch_authenticated_chart(symbol, {
|
|
"period1" => period1,
|
|
"period2" => period2,
|
|
"interval" => "1d",
|
|
"includeAdjustedClose" => true
|
|
})
|
|
|
|
# Check for Yahoo Finance errors
|
|
if data.dig("chart", "error")
|
|
return nil
|
|
end
|
|
|
|
chart_data = data.dig("chart", "result", 0)
|
|
return nil unless chart_data
|
|
|
|
timestamps = chart_data.dig("timestamp") || []
|
|
quotes = chart_data.dig("indicators", "quote", 0) || {}
|
|
closes = quotes["close"] || []
|
|
|
|
results = []
|
|
timestamps.each_with_index do |timestamp, index|
|
|
close_value = closes[index]
|
|
next if close_value.nil? || close_value <= 0
|
|
|
|
results << block.call(timestamp, close_value)
|
|
end
|
|
|
|
results.sort_by(&:date)
|
|
rescue Faraday::Error, JSON::ParserError => e
|
|
nil
|
|
end
|
|
end
|
|
|
|
def client
|
|
@client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
|
|
faraday.request(:retry, {
|
|
max: max_retries,
|
|
interval: retry_interval,
|
|
interval_randomness: 0.5,
|
|
backoff_factor: 2,
|
|
retry_statuses: [ 429 ],
|
|
exceptions: [ Faraday::ConnectionFailed, Faraday::TimeoutError ]
|
|
})
|
|
|
|
faraday.request :json
|
|
faraday.response :raise_error
|
|
|
|
# Yahoo Finance requires common browser headers to avoid blocking
|
|
# Rotate user-agents to reduce rate limiting (based on yfinance PR #2277)
|
|
faraday.headers["User-Agent"] = random_user_agent
|
|
faraday.headers["Accept"] = "application/json"
|
|
faraday.headers["Accept-Language"] = "en-US,en;q=0.9"
|
|
faraday.headers["Cache-Control"] = "no-cache"
|
|
faraday.headers["Pragma"] = "no-cache"
|
|
|
|
# Set reasonable timeouts
|
|
faraday.options.timeout = 10
|
|
faraday.options.open_timeout = 5
|
|
end
|
|
end
|
|
|
|
def random_user_agent
|
|
USER_AGENTS.sample
|
|
end
|
|
|
|
def max_retries
|
|
ENV.fetch("YAHOO_FINANCE_MAX_RETRIES", 5).to_i
|
|
end
|
|
|
|
def retry_interval
|
|
ENV.fetch("YAHOO_FINANCE_RETRY_INTERVAL", 1.0).to_f
|
|
end
|
|
|
|
def min_request_interval
|
|
ENV.fetch("YAHOO_FINANCE_MIN_REQUEST_INTERVAL", MIN_REQUEST_INTERVAL).to_f
|
|
end
|
|
|
|
def throttle_request
|
|
@last_request_time ||= Time.at(0)
|
|
elapsed = Time.current - @last_request_time
|
|
sleep_time = min_request_interval - elapsed
|
|
sleep(sleep_time) if sleep_time > 0
|
|
@last_request_time = Time.current
|
|
end
|
|
|
|
# ================================
|
|
# Cookie/Crumb Authentication
|
|
# ================================
|
|
|
|
# Fetches and caches the Yahoo Finance cookie and crumb for authenticated endpoints
|
|
# The crumb is a CSRF token required by some Yahoo Finance endpoints (e.g., quoteSummary)
|
|
def fetch_cookie_and_crumb
|
|
cache_key = "#{@cache_prefix}_auth_crumb"
|
|
cached = Rails.cache.read(cache_key)
|
|
if cached.present?
|
|
return cached if valid_crumb?(cached.second)
|
|
|
|
Rails.cache.delete(cache_key)
|
|
end
|
|
|
|
cookie, crumb, cache_duration = request_cookie_and_crumb(auth_client)
|
|
result = [ cookie, crumb ]
|
|
Rails.cache.write(cache_key, result, expires_in: cache_duration)
|
|
result
|
|
rescue Faraday::TooManyRequestsError => e
|
|
raise RateLimitError.new(
|
|
"Yahoo Finance rate limit exceeded",
|
|
details: { status: e.response&.dig(:status) }
|
|
)
|
|
rescue Faraday::Error => e
|
|
raise AuthenticationError, "Failed to authenticate with Yahoo Finance: #{e.message}"
|
|
end
|
|
|
|
def request_cookie_and_crumb(authentication_client)
|
|
cookie_response = authentication_client.get("https://fc.yahoo.com")
|
|
if cookie_response.respond_to?(:status) && cookie_response.status == 429
|
|
raise RateLimitError.new(
|
|
"Yahoo Finance rate limit exceeded",
|
|
details: { status: cookie_response.status }
|
|
)
|
|
end
|
|
|
|
cookie = extract_cookie(cookie_response)
|
|
cookie_max_age = extract_cookie_max_age(cookie_response)
|
|
raise AuthenticationError, "Failed to obtain Yahoo Finance cookie" if cookie.blank?
|
|
|
|
crumb_response = authentication_client.get("#{base_url}/v1/test/getcrumb") do |req|
|
|
req.headers["Cookie"] = cookie
|
|
end
|
|
if crumb_response.status == 429
|
|
raise RateLimitError.new(
|
|
"Yahoo Finance rate limit exceeded",
|
|
details: { status: crumb_response.status }
|
|
)
|
|
end
|
|
unless crumb_response.success?
|
|
raise AuthenticationError.new(
|
|
"Failed to obtain Yahoo Finance crumb",
|
|
details: { status: crumb_response.status }
|
|
)
|
|
end
|
|
|
|
crumb = crumb_response.body.to_s.strip
|
|
unless valid_crumb?(crumb)
|
|
error_class = INVALID_CRUMBS.include?(crumb.downcase) ? RateLimitError : AuthenticationError
|
|
raise error_class.new(
|
|
"Failed to obtain Yahoo Finance crumb",
|
|
details: { status: crumb_response.status }
|
|
)
|
|
end
|
|
|
|
cache_duration = [ cookie_max_age || MAX_CRUMB_CACHE_DURATION, MAX_CRUMB_CACHE_DURATION ].min
|
|
[ cookie, crumb, cache_duration ]
|
|
end
|
|
|
|
def valid_crumb?(crumb)
|
|
crumb.present? && !INVALID_CRUMBS.include?(crumb.to_s.strip.downcase)
|
|
end
|
|
|
|
def clear_crumb_cache
|
|
Rails.cache.delete("#{@cache_prefix}_auth_crumb")
|
|
end
|
|
|
|
# Extract the authentication cookie from Yahoo Finance response
|
|
def extract_cookie(response)
|
|
set_cookie = response.headers["set-cookie"]
|
|
return nil if set_cookie.blank?
|
|
|
|
# Extract the cookie value (format: "A3=d=xxx&S=xxx; Max-Age=31557600; ...")
|
|
# We only need the part before the first semicolon
|
|
set_cookie.split(";").first
|
|
end
|
|
|
|
# Extract Max-Age from cookie header and convert to seconds
|
|
# Format: "...; Max-Age=31557600; ..."
|
|
def extract_cookie_max_age(response)
|
|
set_cookie = response.headers["set-cookie"]
|
|
return nil if set_cookie.blank?
|
|
|
|
max_age_match = set_cookie.match(/Max-Age=(\d+)/i)
|
|
return nil unless max_age_match
|
|
|
|
max_age_match[1].to_i.seconds
|
|
end
|
|
|
|
# Client for authentication requests (no error raising - fc.yahoo.com returns 404 but sets cookie)
|
|
def auth_client
|
|
@auth_client ||= Faraday.new(ssl: self.class.faraday_ssl_options) do |faraday|
|
|
faraday.headers["User-Agent"] = random_user_agent
|
|
faraday.headers["Accept"] = "*/*"
|
|
faraday.headers["Accept-Language"] = "en-US,en;q=0.9"
|
|
faraday.options.timeout = 10
|
|
faraday.options.open_timeout = 5
|
|
end
|
|
end
|
|
|
|
# Client for authenticated requests (includes cookie header)
|
|
def authenticated_client(cookie)
|
|
Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
|
|
faraday.request(:retry, {
|
|
max: max_retries,
|
|
interval: retry_interval,
|
|
interval_randomness: 0.5,
|
|
backoff_factor: 2,
|
|
retry_statuses: [ 429 ],
|
|
exceptions: [ Faraday::ConnectionFailed, Faraday::TimeoutError ]
|
|
})
|
|
|
|
faraday.request :json
|
|
faraday.response :raise_error
|
|
|
|
faraday.headers["User-Agent"] = random_user_agent
|
|
faraday.headers["Accept"] = "application/json"
|
|
faraday.headers["Accept-Language"] = "en-US,en;q=0.9"
|
|
faraday.headers["Cache-Control"] = "no-cache"
|
|
faraday.headers["Pragma"] = "no-cache"
|
|
faraday.headers["Cookie"] = cookie
|
|
|
|
faraday.options.timeout = 10
|
|
faraday.options.open_timeout = 5
|
|
end
|
|
end
|
|
|
|
def map_country_code(exchange_name)
|
|
return nil if exchange_name.blank?
|
|
|
|
# Map common exchange names to country codes
|
|
case exchange_name.upcase.strip
|
|
when /NASDAQ|NYSE|AMEX|BATS|IEX/
|
|
"US"
|
|
when /TSX|TSXV|CSE/
|
|
"CA"
|
|
when /LSE|LONDON|AIM/
|
|
"GB"
|
|
when /TOKYO|TSE|NIKKEI|JASDAQ/
|
|
"JP"
|
|
when /ASX|AUSTRALIA/
|
|
"AU"
|
|
when /EURONEXT|PARIS|AMSTERDAM|BRUSSELS|LISBON/
|
|
case exchange_name.upcase
|
|
when /PARIS/ then "FR"
|
|
when /AMSTERDAM/ then "NL"
|
|
when /BRUSSELS/ then "BE"
|
|
when /LISBON/ then "PT"
|
|
else "FR" # Default to France for Euronext
|
|
end
|
|
when /FRANKFURT|XETRA|GETTEX/
|
|
"DE"
|
|
when /SIX|ZURICH/
|
|
"CH"
|
|
when /BME|MADRID/
|
|
"ES"
|
|
when /BORSA|MILAN/
|
|
"IT"
|
|
when /OSLO|OSE/
|
|
"NO"
|
|
when /STOCKHOLM|OMX/
|
|
"SE"
|
|
when /COPENHAGEN/
|
|
"DK"
|
|
when /HELSINKI/
|
|
"FI"
|
|
when /VIENNA/
|
|
"AT"
|
|
when /WARSAW|GPW/
|
|
"PL"
|
|
when /PRAGUE/
|
|
"CZ"
|
|
when /BUDAPEST/
|
|
"HU"
|
|
when /SHANGHAI|SHENZHEN/
|
|
"CN"
|
|
when /HONG\s*KONG|HKG/
|
|
"HK"
|
|
when /KOREA|KRX/
|
|
"KR"
|
|
when /SINGAPORE|SGX/
|
|
"SG"
|
|
when /MUMBAI|NSE|BSE/
|
|
"IN"
|
|
when /SAO\s*PAULO|BOVESPA/
|
|
"BR"
|
|
when /MEXICO|BMV/
|
|
"MX"
|
|
when /JSE|JOHANNESBURG/
|
|
"ZA"
|
|
when /JAKARTA|IDX/
|
|
"ID"
|
|
else
|
|
nil
|
|
end
|
|
end
|
|
|
|
def map_exchange_mic(exchange_code)
|
|
return nil if exchange_code.blank?
|
|
|
|
# Map Yahoo exchange codes to MIC codes
|
|
case exchange_code.upcase.strip
|
|
when "NMS"
|
|
"XNAS" # NASDAQ Global Select
|
|
when "NGM"
|
|
"XNAS" # NASDAQ Global Market
|
|
when "NCM"
|
|
"XNAS" # NASDAQ Capital Market
|
|
when "NYQ"
|
|
"XNYS" # NYSE
|
|
when "PCX", "PSX"
|
|
"ARCX" # NYSE Arca
|
|
when "ASE", "AMX"
|
|
"XASE" # NYSE American
|
|
when "YHD"
|
|
"XNAS" # Yahoo default, assume NASDAQ
|
|
when "TSE", "TOR"
|
|
"XTSE" # Toronto Stock Exchange
|
|
when "CVE"
|
|
"XTSX" # TSX Venture Exchange
|
|
when "LSE", "LON"
|
|
"XLON" # London Stock Exchange
|
|
when "FRA"
|
|
"XFRA" # Frankfurt Stock Exchange
|
|
when "PAR"
|
|
"XPAR" # Euronext Paris
|
|
when "AMS"
|
|
"XAMS" # Euronext Amsterdam
|
|
when "BRU"
|
|
"XBRU" # Euronext Brussels
|
|
when "SWX"
|
|
"XSWX" # SIX Swiss Exchange
|
|
when "HKG"
|
|
"XHKG" # Hong Kong Stock Exchange
|
|
when "TYO"
|
|
"XJPX" # Japan Exchange Group
|
|
when "ASX"
|
|
"XASX" # Australian Securities Exchange
|
|
when "NSE", "NSI"
|
|
"XNSE" # National Stock Exchange of India
|
|
when "BSE", "BOM"
|
|
"XBOM" # BSE (Bombay Stock Exchange)
|
|
when "BVC"
|
|
"XBOG" # Colombian Securities Exchange
|
|
when "JKT"
|
|
"XIDX" # Indonesia Stock Exchange (IDX)
|
|
else
|
|
exchange_code.upcase
|
|
end
|
|
end
|
|
|
|
def map_security_type(quote_type)
|
|
case quote_type&.downcase
|
|
when "equity"
|
|
"common stock"
|
|
when "etf"
|
|
"etf"
|
|
when "mutualfund"
|
|
"mutual fund"
|
|
when "index"
|
|
"index"
|
|
else
|
|
quote_type&.downcase
|
|
end
|
|
end
|
|
|
|
# Override default error transformer to handle Yahoo Finance specific errors
|
|
def default_error_transformer(error)
|
|
case error
|
|
when Faraday::TooManyRequestsError
|
|
RateLimitError.new("Yahoo Finance rate limit exceeded", details: error.response&.dig(:body))
|
|
when Faraday::UnauthorizedError
|
|
# 401 indicates missing or invalid crumb/cookie authentication
|
|
AuthenticationError.new("Yahoo Finance authentication failed (invalid crumb)", details: error.response&.dig(:body))
|
|
when AuthenticationError
|
|
# Already an authentication error, return as is
|
|
error
|
|
when Faraday::Error
|
|
Error.new(
|
|
error.message,
|
|
details: error.response&.dig(:body)
|
|
)
|
|
when Error
|
|
# Already a Yahoo Finance error, return as is
|
|
error
|
|
else
|
|
Error.new(error.message)
|
|
end
|
|
end
|
|
end
|