sure/app/models/provider/tiingo.rb

class Provider::Tiingo < Provider
  include SecurityConcept, RateLimitable
  extend SslConfigurable

  # Subclass so errors caught in this provider are raised as Provider::Tiingo::Error
  Error = Class.new(Provider::Error)
  InvalidSecurityPriceError = Class.new(Error)
  RateLimitError = Class.new(Error)

  # Minimum delay between requests to avoid rate limiting (in seconds)
  MIN_REQUEST_INTERVAL = 1.5

  # Maximum unique symbols per month (Tiingo free tier limit)
  MAX_SYMBOLS_PER_MONTH = 500

  # Maximum requests per hour
  MAX_REQUESTS_PER_HOUR = 1000

  # Tiingo exchange names to MIC codes
  TIINGO_EXCHANGE_TO_MIC = {
    "NASDAQ" => "XNAS",
    "NYSE" => "XNYS",
    "NYSE ARCA" => "XARC",
    "NYSE MKT" => "XASE",
    "BATS" => "BATS",
    "LSE" => "XLON",
    "SHE" => "XSHE",
    "SHG" => "XSHG",
    "OTCMKTS" => "XOTC",
    "OTCD" => "XOTC",
    "PINK" => "XOTC"
  }.freeze

  # Tiingo asset types to normalized kinds
  TIINGO_ASSET_TYPE_MAP = {
    "Stock" => "common stock",
    "ETF" => "etf",
    "Mutual Fund" => "mutual fund"
  }.freeze

  def initialize(api_key)
    @api_key = api_key # pipelock:ignore
  end

  def healthy?
    with_provider_response do
      response = client.get("#{base_url}/tiingo/daily/AAPL")
      parsed = JSON.parse(response.body)
      parsed.dig("ticker").present?
    end
  end

  def usage
    with_provider_response do
      count_key = "tiingo:symbol_count:#{Date.current.strftime('%Y-%m')}"
      symbols_used = Rails.cache.read(count_key).to_i

      UsageData.new(
        used: symbols_used,
        limit: MAX_SYMBOLS_PER_MONTH,
        utilization: (symbols_used.to_f / MAX_SYMBOLS_PER_MONTH * 100).round(1),
        plan: "Free"
      )
    end
  end

  # ================================
  #           Securities
  # ================================

  def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
    with_provider_response do
      throttle_request

      response = client.get("#{base_url}/tiingo/utilities/search") do |req|
        req.params["query"] = symbol
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)

      unless parsed.is_a?(Array)
        raise Error, "Unexpected response format from search endpoint"
      end

      # Tiingo's daily-price endpoints are looked up by ticker alone, so every
      # result sharing a ticker resolves to the same priced entry (see
      # best_match_for_ticker) and therefore the same currency. Resolve it once
      # per unique ticker and reuse it both for caching (so fetch_security_prices
      # can use it without a second search request) and for the Security objects
      # below, so what's shown in search results always matches what
      # fetch_security_prices will later return.
      matches_by_ticker = parsed.filter_map { |security| security["ticker"] }.map(&:upcase).uniq.index_with do |ticker|
        best_match_for_ticker(parsed, ticker)
      end

      currency_by_ticker = matches_by_ticker.transform_values { |match| currency_for_country(match&.dig("countryCode")) }

      currency_by_ticker.each do |ticker, currency|
        next if currency.blank?

        cache_key = "tiingo:currency:#{ticker}"

        # A ticker's daily-price endpoint is US-centric (see best_match_for_ticker),
        # so a currency derived from a US match is authoritative. But a search
        # query can return a result set that happens not to include the US
        # cross-listing for an already-cached ticker (Tiingo's relevance ranking
        # varies by query), and this loop runs on every search_securities call.
        # Only overwrite a cached value when this result set's match is US, or
        # nothing is cached yet - never downgrade a previously US-derived
        # currency to one derived from a non-US match.
        next if matches_by_ticker[ticker]&.dig("countryCode") != "US" && Rails.cache.read(cache_key).present?

        Rails.cache.write(cache_key, currency, expires_in: 24.hours)
      end

      parsed.first(25).map do |security|
        ticker = security["ticker"]
        currency = currency_by_ticker[ticker&.upcase]

        Security.new(
          symbol: ticker,
          name: security["name"],
          logo_url: nil,
          exchange_operating_mic: map_exchange_to_mic(security["exchange"]),
          country_code: security["countryCode"].presence || country_code,
          currency: currency
        )
      end
    end
  end

  def fetch_security_info(symbol:, exchange_operating_mic:)
    with_provider_response do
      throttle_request
      track_symbol(symbol)

      response = client.get("#{base_url}/tiingo/daily/#{CGI.escape(symbol)}")

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)

      # The daily metadata endpoint returns exchangeCode (e.g., "NYSE ARCA", "OTCD")
      resolved_mic = exchange_operating_mic.presence || map_exchange_to_mic(parsed["exchangeCode"])

      SecurityInfo.new(
        symbol: parsed["ticker"] || symbol,
        name: parsed["name"],
        links: nil,
        logo_url: nil,
        description: parsed["description"],
        kind: nil,
        exchange_operating_mic: resolved_mic
      )
    end
  end

  def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
    with_provider_response do
      historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)

      raise historical_data.error if historical_data.error.present?
      raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?

      historical_data.data.first
    end
  end

  def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
    with_provider_response do
      throttle_request
      track_symbol(symbol)

      response = client.get("#{base_url}/tiingo/daily/#{CGI.escape(symbol)}/prices") do |req|
        req.params["startDate"] = start_date.to_s
        req.params["endDate"] = end_date.to_s
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)

      unless parsed.is_a?(Array)
        error_message = parsed.is_a?(Hash) ? (parsed["detail"] || "Unexpected response format") : "Unexpected response format"
        raise InvalidSecurityPriceError, "API error: #{error_message}"
      end

      # Prefer cached currency from search results to avoid a second API call
      cache_key = "tiingo:currency:#{symbol.upcase}"
      currency = Rails.cache.read(cache_key) || fetch_currency_for_symbol(symbol)

      parsed.map do |resp|
        price = resp["close"]
        date = resp["date"]

        if price.nil? || price.to_f <= 0
          Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}.  Price data: #{price.inspect}")
          next
        end

        Price.new(
          symbol: symbol,
          date: Date.parse(date),
          price: price,
          currency: currency,
          exchange_operating_mic: exchange_operating_mic
        )
      end.compact
    end
  end

  private
    attr_reader :api_key

    def base_url
      ENV["TIINGO_URL"] || "https://api.tiingo.com"
    end

    def client
      @client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
        faraday.request(:retry, {
          max: 3,
          interval: 1.0,
          interval_randomness: 0.5,
          backoff_factor: 2,
          exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
        })

        faraday.request :json
        faraday.response :raise_error
        faraday.headers["Authorization"] = "Token #{api_key}"
        faraday.headers["Content-Type"] = "application/json"
      end
    end

    # Adds hourly request counter on top of the interval throttle from RateLimitable.
    def throttle_request
      super

      # Global per-hour request counter via cache (Redis).
      # Atomic increment-then-check avoids the TOCTOU of read-check-increment.
      hour_key = "tiingo:requests:#{Time.current.to_i / 3600}"
      new_count = Rails.cache.increment(hour_key, 1, expires_in: 7200.seconds).to_i

      if new_count >= max_requests_per_hour
        raise RateLimitError, "Tiingo hourly request limit reached (#{new_count}/#{max_requests_per_hour})"
      end
    end

    # Tracks unique symbols queried per month to stay within Tiingo's 500 symbols/month limit.
    # Uses atomic set-if-absent (Redis SETNX) to eliminate the read-then-write race
    # where two concurrent workers could both see the symbol as untracked and both
    # increment the counter.
    def track_symbol(symbol)
      symbol_key = "tiingo:symbol:#{Date.current.strftime('%Y-%m')}:#{symbol.upcase}"
      count_key  = "tiingo:symbol_count:#{Date.current.strftime('%Y-%m')}"

      # Atomic write-if-absent: returns false when the key already exists (Redis SETNX).
      # Only the first worker to claim this symbol will proceed to increment the counter.
      return unless Rails.cache.write(symbol_key, true, expires_in: 35.days, unless_exist: true)

      new_count = Rails.cache.increment(count_key, 1, expires_in: 35.days).to_i

      if new_count >= MAX_SYMBOLS_PER_MONTH
        Rails.cache.decrement(count_key, 1)
        Rails.cache.delete(symbol_key)
        raise RateLimitError, "Tiingo unique symbol limit reached (#{MAX_SYMBOLS_PER_MONTH} per month)"
      end
    end

    # min_request_interval provided by RateLimitable

    def max_requests_per_hour
      ENV.fetch("TIINGO_MAX_REQUESTS_PER_HOUR", MAX_REQUESTS_PER_HOUR).to_i
    end

    # Fetches the price currency for a symbol via the search endpoint.
    # Only called as a fallback when the cache (populated by search_securities)
    # doesn't have the currency. Raises on failure to avoid silently mislabeling
    # non-USD instruments as USD.
    def fetch_currency_for_symbol(symbol)
      throttle_request

      response = client.get("#{base_url}/tiingo/utilities/search") do |req|
        req.params["query"] = symbol
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)

      if parsed.is_a?(Array)
        match = best_match_for_ticker(parsed, symbol)
        currency = currency_for_country(match&.dig("countryCode"))

        if currency.present?
          Rails.cache.write("tiingo:currency:#{symbol.upcase}", currency, expires_in: 24.hours)
          return currency
        end
      end

      raise Error, "Could not determine currency for #{symbol} from Tiingo search"
    end

    def map_exchange_to_mic(exchange_name)
      return nil if exchange_name.blank?
      TIINGO_EXCHANGE_TO_MIC[exchange_name.strip] || exchange_name.strip
    end

    # Tiingo's search/utilities response never includes a priceCurrency field
    # (confirmed against the live API), only countryCode. Resolve the currency
    # via the countries gem's ISO 4217 data (already used for country
    # resolution in Provider::TwelveData) instead of hand-maintaining a
    # per-provider allowlist.
    def currency_for_country(country_code)
      return nil if country_code.blank?
      ISO3166::Country.new(country_code.strip)&.currency_code
    end

    # Tiingo's search endpoint can return multiple entries sharing the exact
    # same ticker - e.g. a US primary listing alongside a foreign
    # cross-listing (confirmed live: searching "AAPL" returns both a US entry
    # and a CA entry). The /tiingo/daily price endpoints this resolves
    # currency for are US-centric (also confirmed live: a Canadian ticker
    # like VFV has search metadata but no daily price history at all), so
    # when a US entry exists for the ticker, that's the one actually backing
    # the price data. Fall back to the first match otherwise.
    def best_match_for_ticker(results, ticker)
      return nil if ticker.blank?

      matches = results.select { |s| s["ticker"]&.upcase == ticker.upcase }
      matches.find { |s| s["countryCode"] == "US" } || matches.first
    end

    def check_api_error!(parsed)
      return unless parsed.is_a?(Hash) && parsed["detail"].present?

      detail = parsed["detail"]

      if detail.downcase.include?("rate limit") || detail.downcase.include?("too many")
        raise RateLimitError, detail
      end

      raise Error, "API error: #{detail}"
    end
end