sure/app/models/mercury_item/importer.rb

class MercuryItem::Importer
  attr_reader :mercury_item, :mercury_provider

  def initialize(mercury_item, mercury_provider:)
    @mercury_item = mercury_item
    @mercury_provider = mercury_provider
  end

  def import
    Rails.logger.info "MercuryItem::Importer - Starting import for item #{mercury_item.id}"

    # Step 1: Fetch all accounts from Mercury
    accounts_data = fetch_accounts_data
    unless accounts_data
      Rails.logger.error "MercuryItem::Importer - Failed to fetch accounts data for item #{mercury_item.id}"
      return { success: false, error: "Failed to fetch accounts data", accounts_imported: 0, transactions_imported: 0 }
    end

    # Store raw payload
    begin
      mercury_item.upsert_mercury_snapshot!(accounts_data)
    rescue => e
      Rails.logger.error "MercuryItem::Importer - Failed to store accounts snapshot: #{e.message}"
      # Continue with import even if snapshot storage fails
    end

    # Step 2: Update linked accounts and create records for new accounts from API
    accounts_updated = 0
    accounts_created = 0
    accounts_failed = 0

    if accounts_data[:accounts].present?
      # Get linked mercury account IDs (ones actually imported/used by the user)
      linked_account_ids = mercury_item.mercury_accounts
                                       .joins(:account_provider)
                                       .pluck(:account_id)
                                       .map(&:to_s)

      # Get all existing mercury account IDs (linked or not)
      all_existing_ids = mercury_item.mercury_accounts.pluck(:account_id).map(&:to_s)

      accounts_data[:accounts].each do |account_data|
        account_id = account_data[:id]&.to_s
        next unless account_id.present?

        # Mercury uses 'name' or 'nickname' for account name
        account_name = account_data[:nickname].presence || account_data[:name].presence || account_data[:legalBusinessName].presence
        next if account_name.blank?

        if linked_account_ids.include?(account_id)
          # Update existing linked accounts
          begin
            import_account(account_data)
            accounts_updated += 1
          rescue => e
            accounts_failed += 1
            Rails.logger.error "MercuryItem::Importer - Failed to update account #{account_id}: #{e.message}"
          end
        elsif !all_existing_ids.include?(account_id)
          # Create new unlinked mercury_account records for accounts we haven't seen before
          # This allows users to link them later via "Setup new accounts"
          begin
            mercury_account = mercury_item.mercury_accounts.build(
              account_id: account_id,
              name: account_name,
              currency: "USD"  # Mercury is US-only, always USD
            )
            mercury_account.upsert_mercury_snapshot!(account_data)
            accounts_created += 1
            Rails.logger.info "MercuryItem::Importer - Created new unlinked account record for #{account_id}"
          rescue => e
            accounts_failed += 1
            Rails.logger.error "MercuryItem::Importer - Failed to create account #{account_id}: #{e.message}"
          end
        end
      end
    end

    Rails.logger.info "MercuryItem::Importer - Updated #{accounts_updated} accounts, created #{accounts_created} new (#{accounts_failed} failed)"

    # Step 3: Fetch transactions only for linked accounts with active status
    transactions_imported = 0
    transactions_failed = 0

    mercury_item.mercury_accounts.joins(:account).merge(Account.visible).each do |mercury_account|
      begin
        result = fetch_and_store_transactions(mercury_account)
        if result[:success]
          transactions_imported += result[:transactions_count]
        else
          transactions_failed += 1
        end
      rescue => e
        transactions_failed += 1
        Rails.logger.error "MercuryItem::Importer - Failed to fetch/store transactions for account #{mercury_account.account_id}: #{e.message}"
        # Continue with other accounts even if one fails
      end
    end

    Rails.logger.info "MercuryItem::Importer - Completed import for item #{mercury_item.id}: #{accounts_updated} accounts updated, #{accounts_created} new accounts discovered, #{transactions_imported} transactions"

    {
      success: accounts_failed == 0 && transactions_failed == 0,
      accounts_updated: accounts_updated,
      accounts_created: accounts_created,
      accounts_failed: accounts_failed,
      transactions_imported: transactions_imported,
      transactions_failed: transactions_failed
    }
  end

  private

    def fetch_accounts_data
      begin
        accounts_data = mercury_provider.get_accounts
      rescue Provider::Mercury::MercuryError => e
        # Handle authentication errors by marking item as requiring update
        if e.error_type == :unauthorized || e.error_type == :access_forbidden
          begin
            mercury_item.update!(status: :requires_update)
          rescue => update_error
            Rails.logger.error "MercuryItem::Importer - Failed to update item status: #{update_error.message}"
          end
        end
        Rails.logger.error "MercuryItem::Importer - Mercury API error: #{e.message}"
        return nil
      rescue JSON::ParserError => e
        Rails.logger.error "MercuryItem::Importer - Failed to parse Mercury API response: #{e.message}"
        return nil
      rescue => e
        Rails.logger.error "MercuryItem::Importer - Unexpected error fetching accounts: #{e.class} - #{e.message}"
        Rails.logger.error e.backtrace.join("\n")
        return nil
      end

      # Validate response structure
      unless accounts_data.is_a?(Hash)
        Rails.logger.error "MercuryItem::Importer - Invalid accounts_data format: expected Hash, got #{accounts_data.class}"
        return nil
      end

      # Handle errors if present in response
      if accounts_data[:error].present?
        handle_error(accounts_data[:error])
        return nil
      end

      accounts_data
    end

    def import_account(account_data)
      # Validate account data structure
      unless account_data.is_a?(Hash)
        Rails.logger.error "MercuryItem::Importer - Invalid account_data format: expected Hash, got #{account_data.class}"
        raise ArgumentError, "Invalid account data format"
      end

      account_id = account_data[:id]

      # Validate required account_id
      if account_id.blank?
        Rails.logger.warn "MercuryItem::Importer - Skipping account with missing ID"
        raise ArgumentError, "Account ID is required"
      end

      # Only find existing accounts, don't create new ones during sync
      mercury_account = mercury_item.mercury_accounts.find_by(
        account_id: account_id.to_s
      )

      # Skip if account wasn't previously selected
      unless mercury_account
        Rails.logger.debug "MercuryItem::Importer - Skipping unselected account #{account_id}"
        return
      end

      begin
        mercury_account.upsert_mercury_snapshot!(account_data)
        mercury_account.save!
      rescue ActiveRecord::RecordInvalid => e
        Rails.logger.error "MercuryItem::Importer - Failed to save mercury_account: #{e.message}"
        raise StandardError.new("Failed to save account: #{e.message}")
      end
    end

    def fetch_and_store_transactions(mercury_account)
      start_date = determine_sync_start_date(mercury_account)
      Rails.logger.info "MercuryItem::Importer - Fetching transactions for account #{mercury_account.account_id} from #{start_date}"

      begin
        # Fetch transactions
        transactions_data = mercury_provider.get_account_transactions(
          mercury_account.account_id,
          start_date: start_date
        )

        # Validate response structure
        unless transactions_data.is_a?(Hash)
          Rails.logger.error "MercuryItem::Importer - Invalid transactions_data format for account #{mercury_account.account_id}"
          return { success: false, transactions_count: 0, error: "Invalid response format" }
        end

        transactions_count = transactions_data[:transactions]&.count || 0
        Rails.logger.info "MercuryItem::Importer - Fetched #{transactions_count} transactions for account #{mercury_account.account_id}"

        # Store transactions in the account
        if transactions_data[:transactions].present?
          begin
            existing_transactions = mercury_account.raw_transactions_payload.to_a

            # Build a map of existing transaction IDs for efficient lookup
            existing_by_id = existing_transactions.index_by { |tx| tx.with_indifferent_access[:id] }

            new_transactions = []
            updated_transactions = []

            transactions_data[:transactions].each do |tx|
              next unless tx.is_a?(Hash)

              tx_data = tx.with_indifferent_access
              tx_id   = tx_data[:id]
              next unless tx_id.present?

              if existing_by_id.key?(tx_id)
                existing = existing_by_id[tx_id].with_indifferent_access
                # Mercury reuses the same transaction ID when a pending entry posts.
                # Replace the stored copy so the processor clears the pending flag.
                if existing[:status] == "pending" && tx_data[:status] != "pending"
                  existing_by_id[tx_id] = tx
                  updated_transactions << tx_id
                end
              else
                new_transactions << tx
              end
            end

            if new_transactions.any? || updated_transactions.any?
              merged = existing_by_id.values + new_transactions
              DebugLogEntry.capture(
                category: "provider_sync",
                level: "info",
                message: "Storing #{new_transactions.count} new, #{updated_transactions.count} status-updated transactions for account #{mercury_account.account_id}",
                source: self.class.name,
                provider_key: "mercury",
                family: mercury_item.family,
                metadata: { account_id: mercury_account.account_id, new_count: new_transactions.count, updated_count: updated_transactions.count }
              )
              mercury_account.upsert_mercury_transactions_snapshot!(merged)
            else
              DebugLogEntry.capture(
                category: "provider_sync",
                level: "info",
                message: "No new or updated transactions for account #{mercury_account.account_id}",
                source: self.class.name,
                provider_key: "mercury",
                family: mercury_item.family,
                metadata: { account_id: mercury_account.account_id }
              )
            end
          rescue => e
            Rails.logger.error "MercuryItem::Importer - Failed to store transactions for account #{mercury_account.account_id}: #{e.message}"
            return { success: false, transactions_count: 0, error: "Failed to store transactions: #{e.message}" }
          end
        else
          Rails.logger.info "MercuryItem::Importer - No transactions to store for account #{mercury_account.account_id}"
        end

        { success: true, transactions_count: transactions_count }
      rescue Provider::Mercury::MercuryError => e
        Rails.logger.error "MercuryItem::Importer - Mercury API error for account #{mercury_account.id}: #{e.message}"
        { success: false, transactions_count: 0, error: e.message }
      rescue JSON::ParserError => e
        Rails.logger.error "MercuryItem::Importer - Failed to parse transaction response for account #{mercury_account.id}: #{e.message}"
        { success: false, transactions_count: 0, error: "Failed to parse response" }
      rescue => e
        Rails.logger.error "MercuryItem::Importer - Unexpected error fetching transactions for account #{mercury_account.id}: #{e.class} - #{e.message}"
        Rails.logger.error e.backtrace.join("\n")
        { success: false, transactions_count: 0, error: "Unexpected error: #{e.message}" }
      end
    end

    def determine_sync_start_date(mercury_account)
      # Check if this account has any stored transactions
      # If not, treat it as a first sync for this account even if the item has been synced before
      has_stored_transactions = mercury_account.raw_transactions_payload.to_a.any?

      if has_stored_transactions
        # Account has been synced before, use item-level logic with buffer
        # For subsequent syncs, fetch from last sync date with a buffer
        if mercury_item.last_synced_at
          mercury_item.last_synced_at - 7.days
        else
          # Fallback if item hasn't been synced but account has transactions
          90.days.ago
        end
      else
        # Account has no stored transactions - this is a first sync for this account
        # Use account creation date or a generous historical window
        account_baseline = mercury_account.created_at || Time.current
        first_sync_window = [ account_baseline - 7.days, 90.days.ago ].max

        # Use the more recent of: (account created - 7 days) or (90 days ago)
        # This caps old accounts at 90 days while respecting recent account creation dates
        first_sync_window
      end
    end

    def handle_error(error_message)
      # Mark item as requiring update for authentication-related errors
      error_msg_lower = error_message.to_s.downcase
      needs_update = error_msg_lower.include?("authentication") ||
                     error_msg_lower.include?("unauthorized") ||
                     error_msg_lower.include?("api key") ||
                     error_msg_lower.include?("api token")

      if needs_update
        begin
          mercury_item.update!(status: :requires_update)
        rescue => e
          Rails.logger.error "MercuryItem::Importer - Failed to update item status: #{e.message}"
        end
      end

      Rails.logger.error "MercuryItem::Importer - API error: #{error_message}"
      raise Provider::Mercury::MercuryError.new(
        "Mercury API error: #{error_message}",
        :api_error
      )
    end
end