sure/app/models/security/price/importer.rb

class Security::Price::Importer
  MissingSecurityPriceError = Class.new(StandardError)
  MissingStartPriceError    = Class.new(StandardError)

  PROVISIONAL_LOOKBACK_DAYS = 7

  def initialize(security:, security_provider:, start_date:, end_date:, clear_cache: false)
    @security          = security
    @security_provider = security_provider
    @start_date        = start_date
    @end_date          = normalize_end_date(end_date)
    @clear_cache       = clear_cache
  end

  # Constructs a daily series of prices for a single security over the date range.
  # Returns the number of rows upserted.
  def import_provider_prices
    if !clear_cache && all_prices_exist?
      Rails.logger.info("No new prices to sync for #{security.ticker} between #{start_date} and #{end_date}, skipping")
      return 0
    end

    if provider_prices.empty?
      Rails.logger.warn("Could not fetch prices for #{security.ticker} between #{start_date} and #{end_date} because provider returned no prices")
      return 0
    end

    prev_price_value = start_price_value
    prev_currency = prev_price_currency || db_price_currency || "USD"

    unless prev_price_value.present?
      Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{start_date}")

      Sentry.capture_exception(MissingStartPriceError.new("Could not determine start price for ticker")) do |scope|
        scope.set_tags(security_id: security.id)
        scope.set_context("security", {
          id: security.id,
          start_date: start_date
        })
      end

      return 0
    end

    gapfilled_prices = effective_start_date.upto(end_date).map do |date|
      db_price             = db_prices[date]
      db_price_value       = db_price&.price
      provider_price       = provider_prices[date]
      provider_price_value = provider_price&.price
      provider_currency    = provider_price&.currency

      has_provider_price = provider_price_value.present? && provider_price_value.to_f > 0
      has_db_price = db_price_value.present? && db_price_value.to_f > 0
      is_provisional = db_price&.provisional

      # Choose price and currency from the same source to avoid mismatches
      chosen_price, chosen_currency = if clear_cache || is_provisional
        # For provisional/cache clear: only use provider price, let gap-fill handle missing
        # This ensures stale DB values don't persist when provider has no weekend data
        [ provider_price_value, provider_currency ]
      elsif has_db_price
        # For non-provisional with valid DB price: preserve existing value (user edits)
        [ db_price_value, db_price&.currency ]
      else
        # Fill gaps with provider data
        [ provider_price_value, provider_currency ]
      end

      # Gap-fill using LOCF (last observation carried forward)
      # Treat nil or zero prices as invalid and use previous price/currency
      used_locf = false
      if chosen_price.nil? || chosen_price.to_f <= 0
        chosen_price = prev_price_value
        chosen_currency = prev_currency
        used_locf = true
      end
      prev_price_value = chosen_price
      prev_currency = chosen_currency || prev_currency

      provisional = determine_provisional_status(
        date: date,
        has_provider_price: has_provider_price,
        used_locf: used_locf,
        existing_provisional: db_price&.provisional
      )

      {
        security_id: security.id,
        date:        date,
        price:       chosen_price,
        currency:    chosen_currency || "USD",
        provisional: provisional
      }
    end

    upsert_rows(gapfilled_prices)
  end

  private
    attr_reader :security, :security_provider, :start_date, :end_date, :clear_cache

    def provider_prices
      @provider_prices ||= begin
        provider_fetch_start_date = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days

        response = security_provider.fetch_security_prices(
          symbol: security.ticker,
          exchange_operating_mic: security.exchange_operating_mic,
          start_date: provider_fetch_start_date,
          end_date:   end_date
        )

        if response.success?
          response.data.index_by(&:date)
        else
          Rails.logger.warn("#{security_provider.class.name} could not fetch prices for #{security.ticker} between #{provider_fetch_start_date} and #{end_date}. Provider error: #{response.error.message}")
          Sentry.capture_exception(MissingSecurityPriceError.new("Could not fetch prices for ticker"), level: :warning) do |scope|
            scope.set_tags(security_id: security.id)
            scope.set_context("security", { id: security.id, start_date: start_date, end_date: end_date })
          end

          {}
        end
      end
    end

    def db_prices
      @db_prices ||= Security::Price.where(security_id: security.id, date: start_date..end_date)
                                    .order(:date)
                                    .to_a
                                    .index_by(&:date)
    end

    def all_prices_exist?
      return false if has_refetchable_provisional_prices?
      db_prices.count == expected_count
    end

    def has_refetchable_provisional_prices?
      Security::Price.where(security_id: security.id, date: start_date..end_date)
                     .refetchable_provisional(lookback_days: PROVISIONAL_LOOKBACK_DAYS)
                     .exists?
    end

    def expected_count
      (start_date..end_date).count
    end

    # Skip over ranges that already exist unless clearing cache
    # Also includes dates with refetchable provisional prices
    def effective_start_date
      return start_date if clear_cache

      refetchable_dates = Security::Price.where(security_id: security.id, date: start_date..end_date)
                                         .refetchable_provisional(lookback_days: PROVISIONAL_LOOKBACK_DAYS)
                                         .pluck(:date)
                                         .to_set

      (start_date..end_date).detect do |d|
        !db_prices.key?(d) || refetchable_dates.include?(d)
      end || end_date
    end

    def start_price_value
      # When processing full range (first sync), use original behavior
      if effective_start_date == start_date
        provider_price_value = provider_prices.select { |date, _| date <= start_date }
                                              .max_by { |date, _| date }
                                              &.last&.price
        db_price_value = db_prices[start_date]&.price

        return provider_price_value if provider_price_value.present? && provider_price_value.to_f > 0
        return db_price_value if db_price_value.present? && db_price_value.to_f > 0
        return nil
      end

      # For partial range (effective_start_date > start_date), use recent data
      # This prevents stale prices from old trade dates propagating to current gap-fills
      cutoff_date = effective_start_date

      # First try provider prices (most recent before cutoff)
      provider_price_value = provider_prices
        .select { |date, _| date < cutoff_date }
        .max_by { |date, _| date }
        &.last&.price

      return provider_price_value if provider_price_value.present? && provider_price_value.to_f > 0

      # Fall back to most recent DB price before cutoff
      currency = prev_price_currency || db_price_currency
      Security::Price
        .where(security_id: security.id)
        .where("date < ?", cutoff_date)
        .where("price > 0")
        .where(provisional: false)
        .then { |q| currency.present? ? q.where(currency: currency) : q }
        .order(date: :desc)
        .limit(1)
        .pick(:price)
    end

    def determine_provisional_status(date:, has_provider_price:, used_locf:, existing_provisional:)
      # Provider returned real price => NOT provisional
      return false if has_provider_price

      # Gap-filled (LOCF) => provisional if recent (including weekends)
      # Weekend prices inherit uncertainty from Friday and get fixed via cascade
      # when the next weekday sync fetches correct Friday price
      if used_locf
        is_recent = date >= PROVISIONAL_LOOKBACK_DAYS.days.ago.to_date
        return is_recent
      end

      # Otherwise preserve existing status
      existing_provisional || false
    end

    def upsert_rows(rows)
      batch_size         = 200
      total_upsert_count = 0
      now = Time.current

      rows_with_timestamps = rows.map { |row| row.merge(updated_at: now) }

      rows_with_timestamps.each_slice(batch_size) do |batch|
        ids = Security::Price.upsert_all(
          batch,
          unique_by: %i[security_id date currency],
          returning: [ "id" ]
        )
        total_upsert_count += ids.count
      end

      total_upsert_count
    end

    def db_price_currency
      db_prices.values.first&.currency
    end

    def prev_price_currency
      @prev_price_currency ||= provider_prices.values.first&.currency
    end

    # Clamp to today (EST) so we never call our price API for a future date (our API is in EST/EDT timezone)
    def normalize_end_date(requested_end_date)
      today_est = Date.current.in_time_zone("America/New_York").to_date
      [ requested_end_date, today_est ].min
    end
end