sure/app/models/provider/twelve_data.rb

class Provider::TwelveData < Provider
  include ExchangeRateConcept, SecurityConcept
  extend SslConfigurable

  # Subclass so errors caught in this provider are raised as Provider::TwelveData::Error
  Error = Class.new(Provider::Error)
  InvalidExchangeRateError = Class.new(Error)
  InvalidSecurityPriceError = Class.new(Error)
  RateLimitError = Class.new(Error)

  # Minimum delay between requests to avoid rate limiting (in seconds)
  MIN_REQUEST_INTERVAL = 1.0

  # Pattern to detect plan upgrade errors in API responses
  PLAN_UPGRADE_PATTERN = /available starting with (\w+)/i

  # Returns true if the error message indicates a plan upgrade is required
  def self.plan_upgrade_required?(error_message)
    return false if error_message.blank?
    PLAN_UPGRADE_PATTERN.match?(error_message)
  end

  # Extracts the required plan name from an error message, or nil if not found
  def self.extract_required_plan(error_message)
    return nil if error_message.blank?
    match = error_message.match(PLAN_UPGRADE_PATTERN)
    match ? match[1] : nil
  end

  def initialize(api_key)
    @api_key = api_key
  end

  def healthy?
    with_provider_response do
      response = client.get("#{base_url}/api_usage")
      JSON.parse(response.body).dig("plan_category").present?
    end
  end

  def usage
    with_provider_response do
      response = client.get("#{base_url}/api_usage")

      parsed = JSON.parse(response.body)

      limit = parsed.dig("plan_daily_limit")
      used = parsed.dig("daily_usage")
      remaining = limit - used

      UsageData.new(
        used: used,
        limit: limit,
        utilization: used / limit * 100,
        plan: parsed.dig("plan_category"),
      )
    end
  end

  # ================================
  #          Exchange Rates
  # ================================

  def fetch_exchange_rate(from:, to:, date:)
    with_provider_response do
      throttle_request
      response = client.get("#{base_url}/exchange_rate") do |req|
        req.params["symbol"] = "#{from}/#{to}"
        req.params["date"] = date.to_s
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)

      Rate.new(date: date.to_date, from:, to:, rate: parsed.dig("rate"))
    end
  end

  def fetch_exchange_rates(from:, to:, start_date:, end_date:)
    with_provider_response do
      # Try to fetch the currency pair via the time_series API (consumes 1 credit) - this might not return anything as the API does not provide time series data for all possible currency pairs
      throttle_request
      response = client.get("#{base_url}/time_series") do |req|
        req.params["symbol"] = "#{from}/#{to}"
        req.params["start_date"] = start_date.to_s
        req.params["end_date"] = end_date.to_s
        req.params["interval"] = "1day"
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)
      data = parsed.dig("values")

      # If currency pair is not available, try to fetch via the time_series/cross API (consumes 5 credits)
      if data.nil?
        Rails.logger.info("#{self.class.name}: Currency pair #{from}/#{to} not available, fetching via time_series/cross API")
        throttle_request(credits: 5)
        response = client.get("#{base_url}/time_series/cross") do |req|
          req.params["base"] = from
          req.params["quote"] = to
          req.params["start_date"] = start_date.to_s
          req.params["end_date"] = end_date.to_s
          req.params["interval"] = "1day"
        end

        parsed = JSON.parse(response.body)
        check_api_error!(parsed)
        data = parsed.dig("values")
      end

      if data.nil?
        error_message = parsed.dig("message") || "No data returned"
        error_code = parsed.dig("code") || "unknown"
        raise InvalidExchangeRateError, "API error (code: #{error_code}): #{error_message}"
      end

      data.map do |resp|
        rate = resp.dig("close")
        date = resp.dig("datetime")
        if rate.nil? || rate.to_f <= 0
          Rails.logger.warn("#{self.class.name} returned invalid rate data for pair from: #{from} to: #{to} on: #{date}.  Rate data: #{rate.inspect}")
          next
        end

        Rate.new(date: date.to_date, from:, to:, rate:)
      end.compact
    end
  end

  # ================================
  #           Securities
  # ================================

  def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
    with_provider_response do
      throttle_request
      response = client.get("#{base_url}/symbol_search") do |req|
        req.params["symbol"] = symbol
        req.params["outputsize"] = 25
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)
      data = parsed.dig("data")

      if data.nil?
        error_message = parsed.dig("message") || "No data returned"
        error_code = parsed.dig("code") || "unknown"
        raise Error, "API error (code: #{error_code}): #{error_message}"
      end

      data.reject { |row| crypto_row?(row) }.map do |security|
        country = ISO3166::Country.find_country_by_any_name(security.dig("country"))

        Security.new(
          symbol: security.dig("symbol"),
          name: security.dig("instrument_name"),
          logo_url: nil,
          exchange_operating_mic: security.dig("mic_code"),
          country_code: country ? country.alpha2 : nil,
          currency: security.dig("currency")
        )
      end
    end
  end

  def fetch_security_info(symbol:, exchange_operating_mic:)
    with_provider_response do
      throttle_request
      response = client.get("#{base_url}/profile") do |req|
        req.params["symbol"] = symbol
        req.params["mic_code"] = exchange_operating_mic
      end

      profile = JSON.parse(response.body)
      check_api_error!(profile)

      throttle_request
      response = client.get("#{base_url}/logo") do |req|
        req.params["symbol"] = symbol
        req.params["mic_code"] = exchange_operating_mic
      end

      logo = JSON.parse(response.body)
      check_api_error!(logo)

      SecurityInfo.new(
        symbol: symbol,
        name: profile.dig("name"),
        links: profile.dig("website"),
        logo_url: logo.dig("url"),
        description: profile.dig("description"),
        kind: profile.dig("type"),
        exchange_operating_mic: exchange_operating_mic
      )
    end
  end

  def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
    with_provider_response do
      historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)

      raise historical_data.error if historical_data.error.present?
      raise InvalidSecurityPriceError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.blank?

      historical_data.data.first
    end
  end

  def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
    with_provider_response do
      throttle_request
      response = client.get("#{base_url}/time_series") do |req|
        req.params["symbol"] = symbol
        req.params["mic_code"] = exchange_operating_mic
        req.params["start_date"] = start_date.to_s
        req.params["end_date"] = end_date.to_s
        req.params["interval"] = "1day"
      end

      parsed = JSON.parse(response.body)
      check_api_error!(parsed)
      values = parsed.dig("values")

      if values.nil?
        error_message = parsed.dig("message") || "No data returned"
        error_code = parsed.dig("code") || "unknown"
        raise InvalidSecurityPriceError, "API error (code: #{error_code}): #{error_message}"
      end

      values.map do |resp|
        price = resp.dig("close")
        date = resp.dig("datetime")
        if price.nil? || price.to_f <= 0
          Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}.  Price data: #{price.inspect}")
          next
        end

        Price.new(
          symbol: symbol,
          date: date.to_date,
          price: price,
          currency: parsed.dig("meta", "currency") || parsed.dig("currency"),
          exchange_operating_mic: exchange_operating_mic
        )
      end.compact
    end
  end

  private
    attr_reader :api_key

    # TwelveData tags crypto symbols with `instrument_type: "Digital Currency"` and
    # `mic_code: "DIGITAL_CURRENCY"`, and returns an empty `currency` field for them.
    # We exclude them so crypto is handled exclusively by Provider::BinancePublic —
    # TD's empty currency would otherwise cascade into Security::Price rows defaulting
    # to USD, silently mispricing EUR/GBP crypto holdings.
    def crypto_row?(row)
      row["instrument_type"].to_s.casecmp?("Digital Currency") ||
        row["mic_code"].to_s.casecmp?("DIGITAL_CURRENCY")
    end

    def base_url
      ENV["TWELVE_DATA_URL"] || "https://api.twelvedata.com"
    end

    def client
      @client ||= Faraday.new(url: base_url, ssl: self.class.faraday_ssl_options) do |faraday|
        faraday.request(:retry, {
          max: 3,
          interval: 1.0,
          interval_randomness: 0.5,
          backoff_factor: 2,
          exceptions: Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [ Faraday::ConnectionFailed ]
        })

        faraday.request :json
        faraday.response :raise_error
        faraday.headers["Authorization"] = "apikey #{api_key}"
      end
    end

    # Paces API requests to stay within TwelveData's rate limits. Sleeps inline
    # because the API physically cannot be called faster — this is unavoidable
    # with a rate-limited provider. The 5-minute cache lock TTL in
    # ExchangeRate::Provided accounts for worst-case throttle waits.
    def throttle_request(credits: 1)
      # Layer 1: Per-instance minimum interval between calls
      @last_request_time ||= Time.at(0)
      elapsed = Time.current - @last_request_time
      sleep_time = min_request_interval - elapsed
      sleep(sleep_time) if sleep_time > 0

      # Layer 2: Global per-minute credit counter via cache (Redis in prod).
      # Read current usage first — if adding these credits would exceed the limit,
      # wait for the next minute BEFORE incrementing. This ensures credits are
      # charged to the minute the request actually fires in, not a stale minute
      # we slept through (which would undercount the new minute's usage).
      minute_key = "twelve_data:credits:#{Time.current.to_i / 60}"
      current_count = Rails.cache.read(minute_key).to_i

      if current_count + credits > max_requests_per_minute
        wait_seconds = 60 - (Time.current.to_i % 60) + 1
        Rails.logger.info("TwelveData: #{current_count + credits}/#{max_requests_per_minute} credits this minute, waiting #{wait_seconds}s")
        sleep(wait_seconds)
      end

      # Charge credits to the minute the request actually fires in
      active_minute_key = "twelve_data:credits:#{Time.current.to_i / 60}"
      Rails.cache.increment(active_minute_key, credits, expires_in: 120.seconds)

      # Set timestamp after all waits so the next call's 1s pacing is measured
      # from when this request actually fires, not from before the minute wait.
      @last_request_time = Time.current
    end

    def min_request_interval
      ENV.fetch("TWELVE_DATA_MIN_REQUEST_INTERVAL", MIN_REQUEST_INTERVAL).to_f
    end

    def max_requests_per_minute
      ENV.fetch("TWELVE_DATA_MAX_REQUESTS_PER_MINUTE", 7).to_i
    end

    def check_api_error!(parsed)
      return unless parsed.is_a?(Hash) && parsed["code"].present?

      if parsed["code"] == 429
        raise RateLimitError, parsed["message"] || "Rate limit exceeded"
      end

      raise Error, "API error (code: #{parsed["code"]}): #{parsed["message"] || "Unknown error"}"
    end

    def default_error_transformer(error)
      case error
      when RateLimitError
        error
      when Faraday::TooManyRequestsError
        RateLimitError.new("TwelveData rate limit exceeded", details: error.response&.dig(:body))
      when Faraday::Error
        self.class::Error.new(error.message, details: error.response&.dig(:body))
      else
        self.class::Error.new(error.message)
      end
    end
end