feat: process pending transactions from lunchflow (#731)

* feat(config): add Lunchflow runtime configuration flags

* feat(api): add include_pending parameter to Lunchflow API

* feat(processor): add pending metadata support to Lunchflow processor

* feat(processor): generate temporary IDs for pending transactions

* feat(importer): integrate pending transaction support in sync

* fix(importer): improve deduplication for transactions without IDs

* feat(model): add Lunchflow pending support to Transaction scopes

* test: add Lunchflow processor pending metadata tests

* docs: update AGENTS.md for Lunchflow pending support

* chore: remove unused variable

* fix: simplify key check

* fix: dotenv-linter key order

* fix: avoid collapsing distinct pending transactions

* fix: prevent unbounded raw payload growth for blank IDs

test/models/lunchflow_entry/processor_test.rb

@@ -0,0 +1,202 @@
+require "test_helper"
+
+class LunchflowEntry::ProcessorTest < ActiveSupport::TestCase
+  setup do
+    @family = families(:empty)
+    @lunchflow_item = LunchflowItem.create!(
+      name: "Test Lunchflow Connection",
+      api_key: "test_key",
+      family: @family
+    )
+    @lunchflow_account = LunchflowAccount.create!(
+      lunchflow_item: @lunchflow_item,
+      name: "Test Account",
+      currency: "USD",
+      account_id: "lf_acc_123"
+    )
+
+    # Create a real account and link it
+    @account = Account.create!(
+      family: @family,
+      name: "Test Checking",
+      accountable: Depository.new(subtype: "checking"),
+      balance: 1000,
+      currency: "USD"
+    )
+
+    AccountProvider.create!(
+      account: @account,
+      provider: @lunchflow_account
+    )
+
+    @lunchflow_account.reload
+  end
+
+  test "stores pending metadata when isPending is true" do
+    transaction_data = {
+      id: "lf_txn_123",
+      accountId: 456,
+      amount: -50.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "Test Merchant",
+      description: "Test transaction",
+      isPending: true
+    }
+
+    result = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result
+    transaction = result.entryable
+    assert_kind_of Transaction, transaction
+    assert_equal true, transaction.pending?
+    assert_equal true, transaction.extra.dig("lunchflow", "pending")
+  end
+
+  test "stores pending false when isPending is false" do
+    transaction_data = {
+      id: "lf_txn_124",
+      accountId: 456,
+      amount: -50.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "Test Merchant",
+      description: "Test transaction",
+      isPending: false
+    }
+
+    result = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result
+    transaction = result.entryable
+    assert_equal false, transaction.pending?
+    assert_equal false, transaction.extra.dig("lunchflow", "pending")
+  end
+
+  test "does not store pending metadata when isPending is absent" do
+    transaction_data = {
+      id: "lf_txn_125",
+      accountId: 456,
+      amount: -50.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "Test Merchant",
+      description: "Test transaction"
+    }
+
+    result = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result
+    transaction = result.entryable
+    assert_not transaction.pending?
+    assert_nil transaction.extra.dig("lunchflow", "pending")
+    assert_nil transaction.extra.dig("lunchflow")
+  end
+
+  test "handles string true value for isPending" do
+    transaction_data = {
+      id: "lf_txn_126",
+      accountId: 456,
+      amount: -50.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "Test Merchant",
+      description: "Test transaction",
+      isPending: "true"
+    }
+
+    result = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result
+    transaction = result.entryable
+    assert_equal true, transaction.pending?
+    assert_equal true, transaction.extra.dig("lunchflow", "pending")
+  end
+
+  test "generates temporary ID for pending transactions with blank ID" do
+    transaction_data = {
+      id: "",
+      accountId: 456,
+      amount: -50.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "Test Merchant",
+      description: "Pending transaction",
+      isPending: true
+    }
+
+    # Process transaction with blank ID
+    result = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result
+    transaction = result.entryable
+    assert_kind_of Transaction, transaction
+    assert transaction.pending?
+
+    # Verify the entry has a generated external_id (since we can't have blank IDs)
+    assert result.external_id.present?
+    assert_match /^lunchflow_pending_[a-f0-9]{32}$/, result.external_id
+
+    # Note: Calling the processor again with identical data will trigger collision
+    # detection and create a SECOND entry (with _1 suffix). In real syncs, the
+    # importer's deduplication prevents this. For true idempotency testing,
+    # use the importer, not the processor directly.
+  end
+
+  test "generates unique IDs for multiple pending transactions with identical attributes" do
+    # Two pending transactions with same merchant, amount, date (e.g., two Uber rides)
+    transaction_data = {
+      id: "",
+      accountId: 456,
+      amount: -15.00,
+      currency: "USD",
+      date: "2025-01-15",
+      merchant: "UBER",
+      description: "Ride",
+      isPending: true
+    }
+
+    # Process first transaction
+    result1 = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result1
+    assert_match /^lunchflow_pending_[a-f0-9]{32}$/, result1.external_id
+
+    # Process second transaction with IDENTICAL attributes
+    result2 = LunchflowEntry::Processor.new(
+      transaction_data,
+      lunchflow_account: @lunchflow_account
+    ).process
+
+    assert_not_nil result2
+
+    # Should create a DIFFERENT entry (not update the first one)
+    assert_not_equal result1.id, result2.id, "Should create separate entries for distinct pending transactions"
+
+    # Second should have a counter appended to avoid collision
+    assert_match /^lunchflow_pending_[a-f0-9]{32}_\d+$/, result2.external_id
+    assert_not_equal result1.external_id, result2.external_id, "Should generate different external_ids to avoid collision"
+
+    # Verify both transactions exist
+    entries = @account.entries.where(source: "lunchflow", "entries.date": "2025-01-15")
+    assert_equal 2, entries.count, "Should have created 2 separate entries"
+  end
+end

test/models/lunchflow_item/importer_blank_id_test.rb

@@ -0,0 +1,218 @@
+require "test_helper"
+
+class LunchflowItem::ImporterBlankIdTest < ActiveSupport::TestCase
+  setup do
+    @family = families(:dylan_family)
+    @item = LunchflowItem.create!(
+      family: @family,
+      name: "Test Lunchflow",
+      api_key: "test_key_123",
+      status: :good
+    )
+    @lunchflow_account = @item.lunchflow_accounts.create!(
+      account_id: "test_account_123",
+      name: "Test Account",
+      currency: "GBP"
+    )
+    @sure_account = @family.accounts.create!(
+      name: "Test Account",
+      balance: 1000,
+      currency: "GBP",
+      accountable: Depository.new(subtype: "checking")
+    )
+    AccountProvider.create!(
+      account: @sure_account,
+      provider: @lunchflow_account
+    )
+    @lunchflow_account.reload
+  end
+
+  test "prevents unbounded growth when same blank-ID transaction synced multiple times" do
+    # Simulate a pending transaction with blank ID that Lunchflow keeps returning
+    pending_transaction = {
+      "id" => "",
+      "accountId" => @lunchflow_account.account_id,
+      "amount" => -15.50,
+      "currency" => "GBP",
+      "date" => Date.today.to_s,
+      "merchant" => "UBER",
+      "description" => "Ride to office",
+      "isPending" => true
+    }
+
+    # Mock the API to return this transaction
+    mock_provider = mock()
+    mock_provider.stubs(:get_account_transactions)
+      .with(@lunchflow_account.account_id, anything)
+      .returns({
+        transactions: [ pending_transaction ],
+        count: 1
+      })
+    mock_provider.stubs(:get_account_balance)
+      .with(@lunchflow_account.account_id)
+      .returns({ balance: 100.0, currency: "GBP" })
+
+    # First sync - should store the transaction
+    importer = LunchflowItem::Importer.new(@item, lunchflow_provider: mock_provider)
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+
+    assert result[:success]
+    first_payload_size = @lunchflow_account.reload.raw_transactions_payload.size
+    assert_equal 1, first_payload_size, "First sync should store 1 transaction"
+
+    # Second sync - same transaction returned again
+    # Without fix: would append again (size = 2)
+    # With fix: should detect duplicate via content hash (size = 1)
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+
+    assert result[:success]
+    second_payload_size = @lunchflow_account.reload.raw_transactions_payload.size
+    assert_equal 1, second_payload_size, "Second sync should NOT append duplicate blank-ID transaction"
+
+    # Third sync - verify it stays at 1
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+
+    assert result[:success]
+    third_payload_size = @lunchflow_account.reload.raw_transactions_payload.size
+    assert_equal 1, third_payload_size, "Third sync should NOT append duplicate blank-ID transaction"
+  end
+
+  test "allows multiple DISTINCT blank-ID transactions to be stored" do
+    # Two different pending transactions, both with blank IDs
+    transaction1 = {
+      "id" => "",
+      "accountId" => @lunchflow_account.account_id,
+      "amount" => -10.00,
+      "currency" => "GBP",
+      "date" => Date.today.to_s,
+      "merchant" => "UBER",
+      "description" => "Morning ride",
+      "isPending" => true
+    }
+
+    transaction2 = {
+      "id" => "",
+      "accountId" => @lunchflow_account.account_id,
+      "amount" => -15.50,
+      "currency" => "GBP",
+      "date" => Date.today.to_s,
+      "merchant" => "UBER",
+      "description" => "Evening ride",  # Different description
+      "isPending" => true
+    }
+
+    # First sync with transaction1
+    mock_provider = mock()
+    mock_provider.stubs(:get_account_transactions)
+      .with(@lunchflow_account.account_id, anything)
+      .returns({
+        transactions: [ transaction1 ],
+        count: 1
+      })
+    mock_provider.stubs(:get_account_balance)
+      .with(@lunchflow_account.account_id)
+      .returns({ balance: 100.0, currency: "GBP" })
+
+    importer = LunchflowItem::Importer.new(@item, lunchflow_provider: mock_provider)
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+
+    assert result[:success]
+    assert_equal 1, @lunchflow_account.reload.raw_transactions_payload.size
+
+    # Second sync with BOTH transactions
+    mock_provider.stubs(:get_account_transactions)
+      .with(@lunchflow_account.account_id, anything)
+      .returns({
+        transactions: [ transaction1, transaction2 ],
+        count: 2
+      })
+
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+
+    assert result[:success]
+    payload = @lunchflow_account.reload.raw_transactions_payload
+    assert_equal 2, payload.size, "Should store both distinct transactions despite blank IDs"
+
+    # Verify both transactions are different
+    descriptions = payload.map { |tx| tx.with_indifferent_access[:description] }
+    assert_includes descriptions, "Morning ride"
+    assert_includes descriptions, "Evening ride"
+  end
+
+  test "content hash uses same attributes as processor for consistency" do
+    pending_transaction = {
+      "id" => "",
+      "accountId" => @lunchflow_account.account_id,
+      "amount" => -10.00,
+      "currency" => "GBP",
+      "date" => Date.today.to_s,
+      "merchant" => "TEST",
+      "description" => "Test transaction",
+      "isPending" => true
+    }
+
+    # Store the transaction
+    @lunchflow_account.upsert_lunchflow_transactions_snapshot!([ pending_transaction ])
+
+    # Process it through the processor to generate external_id
+    processor = LunchflowEntry::Processor.new(
+      pending_transaction.with_indifferent_access,
+      lunchflow_account: @lunchflow_account
+    )
+    processor.process
+
+    # Check that the entry was created
+    entry = @sure_account.entries.last
+    assert entry.present?
+    assert entry.external_id.start_with?("lunchflow_pending_")
+
+    # Extract the hash from the external_id (remove prefix and any collision suffix)
+    external_id_hash = entry.external_id.sub("lunchflow_pending_", "").split("_").first
+
+    # Generate content hash using importer method
+    mock_provider = mock()
+    importer = LunchflowItem::Importer.new(@item, lunchflow_provider: mock_provider)
+    content_hash = importer.send(:content_hash_for_transaction, pending_transaction.with_indifferent_access)
+
+    # They should match (processor adds prefix, importer is just the hash)
+    assert_equal content_hash, external_id_hash, "Importer content hash should match processor's MD5 hash"
+  end
+
+  test "transactions with IDs are not affected by content hash logic" do
+    # Transaction with a proper ID
+    transaction_with_id = {
+      "id" => "txn_123",
+      "accountId" => @lunchflow_account.account_id,
+      "amount" => -20.00,
+      "currency" => "GBP",
+      "date" => Date.today.to_s,
+      "merchant" => "TESCO",
+      "description" => "Groceries",
+      "isPending" => false
+    }
+
+    # Sync twice - should only store once based on ID
+    mock_provider = mock()
+    mock_provider.stubs(:get_account_transactions)
+      .with(@lunchflow_account.account_id, anything)
+      .returns({
+        transactions: [ transaction_with_id ],
+        count: 1
+      })
+    mock_provider.stubs(:get_account_balance)
+      .with(@lunchflow_account.account_id)
+      .returns({ balance: 100.0, currency: "GBP" })
+
+    importer = LunchflowItem::Importer.new(@item, lunchflow_provider: mock_provider)
+
+    # First sync
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+    assert result[:success]
+    assert_equal 1, @lunchflow_account.reload.raw_transactions_payload.size
+
+    # Second sync - should not duplicate
+    result = importer.send(:fetch_and_store_transactions, @lunchflow_account)
+    assert result[:success]
+    assert_equal 1, @lunchflow_account.reload.raw_transactions_payload.size
+  end
+end

test/models/transaction_test.rb

@@ -13,6 +13,12 @@ class TransactionTest < ActiveSupport::TestCase
     assert transaction.pending?
   end
 
+  test "pending? is true when extra.lunchflow.pending is truthy" do
+    transaction = Transaction.new(extra: { "lunchflow" => { "pending" => true } })
+
+    assert transaction.pending?
+  end
+
   test "pending? is false when no provider pending metadata is present" do
     transaction = Transaction.new(extra: { "plaid" => { "pending" => false } })